Skip to content

Commit 837733e

Browse files
johnjcaseyjohnjcasey
authored
Add Gradle sub-projects to enable parallel running of Kafka tests (#26153)
* add parallel number to gradle.properties * remove unused config, attempt to configure higher parallelism for kafka tests * Create generic task to represent Integration tests. Then, create sub-projects to enable running tests in parallel * extend test timeout to reduce flakyness * add comment for running locally * run spotless * factor kafka integration tests up a level
1 parent 7919c3f commit 837733e

File tree

14 files changed

+343
-49
lines changed

14 files changed

+343
-49
lines changed

‎buildSrc/src/main/groovy/org/apache/beam/gradle/kafka/KafkaTestUtilities.groovy

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* License); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an AS IS BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.beam.gradle.kafka
19+
20+
import org.gradle.api.Project
21+
import org.gradle.api.artifacts.ConfigurationContainer
22+
import org.gradle.api.tasks.testing.Test
23+
24+
import javax.inject.Inject
25+
26+
class KafkaTestUtilities {
27+
abstract static class KafkaBatchIT extends Test {
28+
29+
@Inject
30+
KafkaBatchIT(String delimited, String undelimited, Boolean sdfCompatible, ConfigurationContainer configurations, Project runningProject){
31+
group = "Verification"
32+
description = "Runs KafkaIO IT tests with Kafka clients API $delimited"
33+
outputs.upToDateWhen { false }
34+
testClassesDirs = runningProject.findProject(":sdks:java:io:kafka").sourceSets.test.output.classesDirs
35+
classpath = configurations."kafkaVersion$undelimited" + runningProject.sourceSets.test.runtimeClasspath + runningProject.findProject(":sdks:java:io:kafka").sourceSets.test.runtimeClasspath
36+
37+
def pipelineOptions = [
38+
'--sourceOptions={' +
39+
'"numRecords": "1000",' +
40+
'"keySizeBytes": "10",' +
41+
'"valueSizeBytes": "90"' +
42+
'}',
43+
"--readTimeout=120",
44+
"--kafkaTopic=beam",
45+
"--withTestcontainers=true",
46+
"--kafkaContainerVersion=5.5.2",
47+
]
48+
49+
systemProperty "beamTestPipelineOptions", groovy.json.JsonOutput.toJson(pipelineOptions)
50+
include '**/KafkaIOIT.class'
51+
52+
filter {
53+
excludeTestsMatching "*InStreaming"
54+
if (!sdfCompatible) {
55+
excludeTestsMatching "*DynamicPartitions" //admin client create partitions does not exist in kafka 0.11.0.3 and kafka sdf does not appear to work for kafka versions <2.0.1
56+
excludeTestsMatching "*SDFResumesCorrectly" //Kafka SDF does not work for kafka versions <2.0.1
57+
excludeTestsMatching "*StopReadingFunction" //Kafka SDF does not work for kafka versions <2.0.1
58+
excludeTestsMatching "*WatermarkUpdateWithSparseMessages" //Kafka SDF does not work for kafka versions <2.0.1
59+
}
60+
}
61+
}
62+
}
63+
}

‎sdks/java/io/kafka/build.gradle

+13-46
Original file line numberDiff line numberDiff line change
@@ -43,15 +43,6 @@ def kafkaVersions = [
4343
'251': "2.5.1",
4444
]
4545

46-
def sdfKafkaVersions = [
47-
'201',
48-
'211',
49-
'222',
50-
'231',
51-
'241',
52-
'251'
53-
]
54-
5546
kafkaVersions.each{k,v -> configurations.create("kafkaVersion$k")}
5647

5748
dependencies {
@@ -125,48 +116,24 @@ kafkaVersions.each {kv ->
125116
}
126117
}
127118

128-
kafkaVersions.each {kv ->
129-
task "kafkaVersion${kv.key}BatchIT"(type: Test) {
130-
group = "Verification"
131-
description = "Runs KafkaIO IT tests with Kafka clients API $kv.value"
132-
outputs.upToDateWhen { false }
133-
testClassesDirs = sourceSets.test.output.classesDirs
134-
classpath = configurations."kafkaVersion${kv.key}" + sourceSets.test.runtimeClasspath
135-
136-
def pipelineOptions = [
137-
'--sourceOptions={' +
138-
'"numRecords": "1000",' +
139-
'"keySizeBytes": "10",' +
140-
'"valueSizeBytes": "90"' +
141-
'}',
142-
"--readTimeout=120",
143-
"--kafkaTopic=beam",
144-
"--withTestcontainers=true",
145-
"--kafkaContainerVersion=5.5.2",
146-
]
147-
148-
systemProperty "beamTestPipelineOptions", groovy.json.JsonOutput.toJson(pipelineOptions)
149-
include '**/KafkaIOIT.class'
150-
151-
filter {
152-
excludeTestsMatching "*InStreaming"
153-
if (!(kv.key in sdfKafkaVersions)) {
154-
excludeTestsMatching "*DynamicPartitions" //admin client create partitions does not exist in kafka 0.11.0.3 and kafka sdf does not appear to work for kafka versions <2.0.1
155-
excludeTestsMatching "*SDFResumesCorrectly" //Kafka SDF does not work for kafka versions <2.0.1
156-
excludeTestsMatching "*StopReadingFunction" //Kafka SDF does not work for kafka versions <2.0.1
157-
excludeTestsMatching "*WatermarkUpdateWithSparseMessages" //Kafka SDF does not work for kafka versions <2.0.1
158-
}
159-
}
160-
}
161-
}
162-
119+
//Because this runs many integration jobs in parallel, each of which use a
120+
//container, it can fail locally due to performance limitations on a desktop.
121+
//To avoid this, use --max-workers=N, where N is less than half your CPUs.
122+
//4 is a good start for parallelism without overloading your computer.
163123
task kafkaVersionsCompatibilityTest {
164124
group = "Verification"
165125
description = 'Runs KafkaIO with different Kafka client APIs'
166126
def testNames = createTestList(kafkaVersions, "Test")
167-
def batchItTestNames = createTestList(kafkaVersions, "BatchIT")
168127
dependsOn testNames
169-
dependsOn batchItTestNames
128+
dependsOn (":sdks:java:io:kafka:kafka-01103:kafkaVersion01103BatchIT")
129+
dependsOn (":sdks:java:io:kafka:kafka-100:kafkaVersion100BatchIT")
130+
dependsOn (":sdks:java:io:kafka:kafka-111:kafkaVersion111BatchIT")
131+
dependsOn (":sdks:java:io:kafka:kafka-201:kafkaVersion201BatchIT")
132+
dependsOn (":sdks:java:io:kafka:kafka-211:kafkaVersion211BatchIT")
133+
dependsOn (":sdks:java:io:kafka:kafka-222:kafkaVersion222BatchIT")
134+
dependsOn (":sdks:java:io:kafka:kafka-231:kafkaVersion231BatchIT")
135+
dependsOn (":sdks:java:io:kafka:kafka-241:kafkaVersion241BatchIT")
136+
dependsOn (":sdks:java:io:kafka:kafka-251:kafkaVersion251BatchIT")
170137
}
171138

172139
static def createTestList(Map<String, String> prefixMap, String suffix) {

‎sdks/java/io/kafka/kafka-01103/build.gradle

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* License); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an AS IS BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
project.ext {
19+
delimited="0.11.0.3"
20+
undelimited="01103"
21+
}
22+
23+
apply from: "../kafka-integration-test.gradle"

‎sdks/java/io/kafka/kafka-100/build.gradle

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* License); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an AS IS BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
project.ext {
19+
delimited="1.0.0"
20+
undelimited="100"
21+
}
22+
23+
apply from: "../kafka-integration-test.gradle"

‎sdks/java/io/kafka/kafka-111/build.gradle

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* License); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an AS IS BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
project.ext {
19+
delimited="1.1.1"
20+
undelimited="111"
21+
}
22+
23+
apply from: "../kafka-integration-test.gradle"

‎sdks/java/io/kafka/kafka-201/build.gradle

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* License); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an AS IS BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
project.ext {
19+
delimited="2.0.1"
20+
undelimited="201"
21+
}
22+
23+
apply from: "../kafka-integration-test.gradle"

‎sdks/java/io/kafka/kafka-211/build.gradle

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* License); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an AS IS BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
project.ext {
19+
delimited="2.1.1"
20+
undelimited="211"
21+
}
22+
23+
apply from: "../kafka-integration-test.gradle"

‎sdks/java/io/kafka/kafka-222/build.gradle

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* License); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an AS IS BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
project.ext {
19+
delimited="2.2.2"
20+
undelimited="222"
21+
}
22+
23+
apply from: "../kafka-integration-test.gradle"

‎sdks/java/io/kafka/kafka-231/build.gradle

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* License); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an AS IS BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
project.ext {
19+
delimited="2.3.1"
20+
undelimited="231"
21+
}
22+
23+
apply from: "../kafka-integration-test.gradle"

‎sdks/java/io/kafka/kafka-241/build.gradle

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* License); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an AS IS BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
project.ext {
19+
delimited="2.4.1"
20+
undelimited="241"
21+
}
22+
23+
apply from: "../kafka-integration-test.gradle"

‎sdks/java/io/kafka/kafka-251/build.gradle

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* License); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an AS IS BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
project.ext {
19+
delimited="2.5.1"
20+
undelimited="251"
21+
}
22+
23+
apply from: "../kafka-integration-test.gradle"

0 commit comments

Comments
 (0)