Skip to content

Commit a842740

Browse files
authored
WIP Kubenetes parallel build (corda#5396)
* Split integration tests * add simple example of printing all methods annotated with @test * add docker plugin to root project remove docker plugin from child projects add Dockerfile for image to use when testing add task to build testing image to root project * add comment describing proposed testing workflow * simple attempt at running tests in docker container * add my first k8s interaction script * add fabric8 as dependnency to buildSrc * before adding classpath * collect reports from containers and run through testReports * re-enable kubes backed testing * for each project 1. add a list tests task 2. use this list tests task to modify the included tests 3. add a parallel version of the test task * tweak logic for downloading test report XML files * use output of parallel testing tasks in report tasks to determine build resultCode * prepare for jenkins test * prepare for jenkins test * make docker reg password system property * add logging to print out docker reg creds * enable docker build * fix gradle build file * gather xml files into root project * change log level for gradle modification * stop printing gradle docker push passwd * tidy up report generation * fix compilation errors * split signature constraints test into two * change Sig constraint tests type hierarchy * tidy up build.gradle * try method based test includes * add unit test for test listing * fix bug with test slicing * stop filtering ignored tests to make the numbers match existing runs * change log level to ensure print out * move all plugin logic to buildSrc files * tidy up test modification add comments to explain what DistributedTesting plugin does * move new plugins into properly named packages * tidy up runConfigs * fix compile errors due to merge with slow-integration-test work * add system parameter to enable / disable build modification * add -Dkubenetise to build command * address review comments * type safe declaration of parameters in KubesTest
1 parent 99f4e4a commit a842740

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+1545
-356
lines changed

.dockerignore

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
.git
2+
.cache
3+
.idea
4+
.ci
5+
.github
6+
.bootstrapper
7+
**/*.class

.gitignore

-1
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@ lib/quasar.jar
4141

4242

4343
# Include the -parameters compiler option by default in IntelliJ required for serialization.
44-
!.idea/compiler.xml
4544
!.idea/codeStyleSettings.xml
4645

4746
# if you remove the above rule, at least ignore the following:

build.gradle

+37-29
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import net.corda.testing.DistributedTesting
2+
13
buildscript {
24
// For sharing constants between builds
35
Properties constants = new Properties()
@@ -16,25 +18,25 @@ buildscript {
1618
ext.quasar_group = 'co.paralleluniverse'
1719
ext.quasar_version = constants.getProperty("quasarVersion")
1820
ext.quasar_exclusions = [
19-
'co.paralleluniverse**',
20-
'groovy**',
21-
'com.esotericsoftware.**',
22-
'jdk**',
23-
'junit**',
24-
'kotlin**',
25-
'net.rubygrapefruit.**',
26-
'org.gradle.**',
27-
'org.apache.**',
28-
'org.jacoco.**',
29-
'org.junit**',
30-
'org.slf4j**',
31-
'worker.org.gradle.**',
32-
'com.nhaarman.mockito_kotlin**',
33-
'org.assertj**',
34-
'org.hamcrest**',
35-
'org.mockito**',
36-
'org.opentest4j**'
37-
]
21+
'co.paralleluniverse**',
22+
'groovy**',
23+
'com.esotericsoftware.**',
24+
'jdk**',
25+
'junit**',
26+
'kotlin**',
27+
'net.rubygrapefruit.**',
28+
'org.gradle.**',
29+
'org.apache.**',
30+
'org.jacoco.**',
31+
'org.junit**',
32+
'org.slf4j**',
33+
'worker.org.gradle.**',
34+
'com.nhaarman.mockito_kotlin**',
35+
'org.assertj**',
36+
'org.hamcrest**',
37+
'org.mockito**',
38+
'org.opentest4j**'
39+
]
3840

3941
// gradle-capsule-plugin:1.0.2 contains capsule:1.0.1 by default.
4042
// We must configure it manually to use the latest capsule version.
@@ -98,7 +100,7 @@ buildscript {
98100
ext.jsch_version = '0.1.55'
99101
ext.protonj_version = '0.33.0' // Overide Artemis version
100102
ext.snappy_version = '0.4'
101-
ext.class_graph_version = '4.8.41'
103+
ext.class_graph_version = constants.getProperty('classgraphVersion')
102104
ext.jcabi_manifests_version = '1.1'
103105
ext.picocli_version = '3.9.6'
104106
ext.commons_io_version = '2.6'
@@ -113,7 +115,14 @@ buildscript {
113115
// Updates [131, 161] also have zip compression bugs on MacOS (High Sierra).
114116
// when the java version in NodeStartup.hasMinimumJavaVersion() changes, so must this check
115117
ext.java8_minUpdateVersion = constants.getProperty('java8MinUpdateVersion')
116-
118+
ext.corda_revision = {
119+
try {
120+
"git rev-parse HEAD".execute().text.trim()
121+
} catch (Exception ignored) {
122+
logger.warn("git is unavailable in build environment")
123+
"unknown"
124+
}
125+
}()
117126
repositories {
118127
mavenLocal()
119128
mavenCentral()
@@ -152,10 +161,10 @@ plugins {
152161
// Add the shadow plugin to the plugins classpath for the entire project.
153162
id 'com.github.johnrengelman.shadow' version '2.0.4' apply false
154163
id "com.gradle.build-scan" version "2.2.1"
164+
id 'com.bmuschko.docker-remote-api'
155165
}
156166

157167
ext {
158-
corda_revision = "git rev-parse HEAD".execute().text.trim()
159168
}
160169

161170
apply plugin: 'project-report'
@@ -172,7 +181,6 @@ apply plugin: 'java'
172181
sourceCompatibility = 1.8
173182
targetCompatibility = 1.8
174183

175-
176184
allprojects {
177185
apply plugin: 'kotlin'
178186
apply plugin: 'jacoco'
@@ -250,14 +258,14 @@ allprojects {
250258
ex.append = false
251259
}
252260

253-
maxParallelForks = (System.env.CORDA_TESTING_FORKS == null) ? 1 : "$System.env.CORDA_TESTING_FORKS".toInteger()
261+
maxParallelForks = (System.env.CORDA_TESTING_FORKS == null) ? 1 : "$System.env.CORDA_TESTING_FORKS".toInteger()
254262

255263
systemProperty 'java.security.egd', 'file:/dev/./urandom'
256264
}
257265

258-
tasks.withType(Test){
259-
if (name.contains("integrationTest")){
260-
maxParallelForks = (System.env.CORDA_INT_TESTING_FORKS == null) ? 1 : "$System.env.CORDA_INT_TESTING_FORKS".toInteger()
266+
tasks.withType(Test) {
267+
if (name.contains("integrationTest")) {
268+
maxParallelForks = (System.env.CORDA_INT_TESTING_FORKS == null) ? 1 : "$System.env.CORDA_INT_TESTING_FORKS".toInteger()
261269
}
262270
}
263271

@@ -496,8 +504,6 @@ if (file('corda-docs-only-build').exists() || (System.getenv('CORDA_DOCS_ONLY_BU
496504
}
497505
}
498506

499-
500-
501507
wrapper {
502508
gradleVersion = "5.4.1"
503509
distributionType = Wrapper.DistributionType.ALL
@@ -507,3 +513,5 @@ buildScan {
507513
termsOfServiceUrl = 'https://gradle.com/terms-of-service'
508514
termsOfServiceAgree = 'yes'
509515
}
516+
517+
apply plugin: DistributedTesting

buildSrc/build.gradle

+9
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ buildscript {
44

55
ext {
66
guava_version = constants.getProperty("guavaVersion")
7+
class_graph_version = constants.getProperty('classgraphVersion')
78
assertj_version = '3.9.1'
89
junit_version = '4.12'
910
}
@@ -12,6 +13,7 @@ buildscript {
1213
repositories {
1314
mavenLocal()
1415
mavenCentral()
16+
jcenter()
1517
}
1618

1719
allprojects {
@@ -30,4 +32,11 @@ dependencies {
3032
runtime project.childProjects.collect { n, p ->
3133
project(p.path)
3234
}
35+
compile gradleApi()
36+
compile "io.fabric8:kubernetes-client:4.4.1"
37+
compile 'org.apache.commons:commons-compress:1.19'
38+
compile 'commons-codec:commons-codec:1.13'
39+
compile "io.github.classgraph:classgraph:$class_graph_version"
40+
compile "com.bmuschko:gradle-docker-plugin:5.0.0"
41+
testCompile "junit:junit:$junit_version"
3342
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
package net.corda.testing
2+
3+
4+
import com.bmuschko.gradle.docker.tasks.image.DockerPushImage
5+
import org.gradle.api.Plugin
6+
import org.gradle.api.Project
7+
import org.gradle.api.Task
8+
import org.gradle.api.tasks.testing.Test
9+
10+
/**
11+
This plugin is responsible for wiring together the various components of test task modification
12+
*/
13+
class DistributedTesting implements Plugin<Project> {
14+
15+
static def getPropertyAsInt(Project proj, String property, Integer defaultValue) {
16+
return proj.hasProperty(property) ? Integer.parseInt(proj.property(property).toString()) : defaultValue
17+
}
18+
19+
@Override
20+
void apply(Project project) {
21+
if (System.getProperty("kubenetize") != null) {
22+
ensureImagePluginIsApplied(project)
23+
ImageBuilding imagePlugin = project.plugins.getPlugin(ImageBuilding)
24+
DockerPushImage imageBuildingTask = imagePlugin.pushTask
25+
26+
//in each subproject
27+
//1. add the task to determine all tests within the module
28+
//2. modify the underlying testing task to use the output of the listing task to include a subset of tests for each fork
29+
//3. KubesTest will invoke these test tasks in a parallel fashion on a remote k8s cluster
30+
project.subprojects { Project subProject ->
31+
subProject.tasks.withType(Test) { Test task ->
32+
ListTests testListerTask = createTestListingTasks(task, subProject)
33+
Test modifiedTestTask = modifyTestTaskForParallelExecution(subProject, task, testListerTask)
34+
KubesTest parallelTestTask = generateParallelTestingTask(subProject, task, imageBuildingTask)
35+
}
36+
}
37+
38+
//now we are going to create "super" groupings of these KubesTest tasks, so that it is possible to invoke all submodule tests with a single command
39+
//group all kubes tests by their underlying target task (test/integrationTest/smokeTest ... etc)
40+
Map<String, List<KubesTest>> allKubesTestingTasksGroupedByType = project.subprojects.collect { prj -> prj.getAllTasks(false).values() }
41+
.flatten()
42+
.findAll { task -> task instanceof KubesTest }
43+
.groupBy { task -> task.taskToExecuteName }
44+
45+
//first step is to create a single task which will invoke all the submodule tasks for each grouping
46+
//ie allParallelTest will invoke [node:test, core:test, client:rpc:test ... etc]
47+
//ie allIntegrationTest will invoke [node:integrationTest, core:integrationTest, client:rpc:integrationTest ... etc]
48+
createGroupedParallelTestTasks(allKubesTestingTasksGroupedByType, project, imageBuildingTask)
49+
}
50+
}
51+
52+
private List<Task> createGroupedParallelTestTasks(Map<String, List<KubesTest>> allKubesTestingTasksGroupedByType, Project project, DockerPushImage imageBuildingTask) {
53+
allKubesTestingTasksGroupedByType.entrySet().collect { entry ->
54+
def taskType = entry.key
55+
def allTasksOfType = entry.value
56+
def allParallelTask = project.rootProject.tasks.create("allParallel" + taskType.capitalize(), KubesTest) {
57+
dependsOn imageBuildingTask
58+
printOutput = true
59+
fullTaskToExecutePath = allTasksOfType.collect { task -> task.fullTaskToExecutePath }.join(" ")
60+
taskToExecuteName = taskType
61+
doFirst {
62+
dockerTag = imageBuildingTask.imageName.get() + ":" + imageBuildingTask.tag.get()
63+
}
64+
}
65+
66+
//second step is to create a task to use the reports output by the parallel test task
67+
def reportOnAllTask = project.rootProject.tasks.create("reportAllParallel${taskType.capitalize()}", KubesReporting) {
68+
dependsOn allParallelTask
69+
destinationDir new File(project.rootProject.getBuildDir(), "allResults${taskType.capitalize()}")
70+
doFirst {
71+
destinationDir.deleteDir()
72+
podResults = allParallelTask.containerResults
73+
reportOn(allParallelTask.testOutput)
74+
}
75+
}
76+
77+
//invoke this report task after parallel testing
78+
allParallelTask.finalizedBy(reportOnAllTask)
79+
project.logger.info "Created task: ${allParallelTask.getPath()} to enable testing on kubenetes for tasks: ${allParallelTask.fullTaskToExecutePath}"
80+
project.logger.info "Created task: ${reportOnAllTask.getPath()} to generate test html output for task ${allParallelTask.getPath()}"
81+
return allParallelTask
82+
83+
}
84+
}
85+
86+
private KubesTest generateParallelTestingTask(Project projectContainingTask, Test task, DockerPushImage imageBuildingTask) {
87+
def taskName = task.getName()
88+
def capitalizedTaskName = task.getName().capitalize()
89+
90+
KubesTest createdParallelTestTask = projectContainingTask.tasks.create("parallel" + capitalizedTaskName, KubesTest) {
91+
dependsOn imageBuildingTask
92+
printOutput = true
93+
fullTaskToExecutePath = task.getPath()
94+
taskToExecuteName = taskName
95+
doFirst {
96+
dockerTag = imageBuildingTask.imageName.get() + ":" + imageBuildingTask.tag.get()
97+
}
98+
}
99+
projectContainingTask.logger.info "Created task: ${createdParallelTestTask.getPath()} to enable testing on kubenetes for task: ${task.getPath()}"
100+
return createdParallelTestTask as KubesTest
101+
}
102+
103+
private Test modifyTestTaskForParallelExecution(Project subProject, Test task, ListTests testListerTask) {
104+
subProject.logger.info("modifying task: ${task.getPath()} to depend on task ${testListerTask.getPath()}")
105+
def reportsDir = new File(new File(subProject.rootProject.getBuildDir(), "test-reports"), subProject.name + "-" + task.name)
106+
task.configure {
107+
dependsOn testListerTask
108+
binResultsDir new File(reportsDir, "binary")
109+
reports.junitXml.destination new File(reportsDir, "xml")
110+
maxHeapSize = "6g"
111+
doFirst {
112+
filter {
113+
def fork = getPropertyAsInt(subProject, "dockerFork", 0)
114+
def forks = getPropertyAsInt(subProject, "dockerForks", 1)
115+
def shuffleSeed = 42
116+
subProject.logger.info("requesting tests to include in testing task ${task.getPath()} (${fork}, ${forks}, ${shuffleSeed})")
117+
List<String> includes = testListerTask.getTestsForFork(
118+
fork,
119+
forks,
120+
shuffleSeed)
121+
subProject.logger.info "got ${includes.size()} tests to include into testing task ${task.getPath()}"
122+
123+
if (includes.size() == 0) {
124+
subProject.logger.info "Disabling test execution for testing task ${task.getPath()}"
125+
excludeTestsMatching "*"
126+
}
127+
128+
includes.forEach { include ->
129+
subProject.logger.info "including: $include for testing task ${task.getPath()}"
130+
includeTestsMatching include
131+
}
132+
failOnNoMatchingTests false
133+
}
134+
}
135+
}
136+
137+
return task
138+
}
139+
140+
private static void ensureImagePluginIsApplied(Project project) {
141+
project.plugins.apply(ImageBuilding)
142+
}
143+
144+
private ListTests createTestListingTasks(Test task, Project subProject) {
145+
def taskName = task.getName()
146+
def capitalizedTaskName = task.getName().capitalize()
147+
//determine all the tests which are present in this test task.
148+
//this list will then be shared between the various worker forks
149+
def createdListTask = subProject.tasks.create("listTestsFor" + capitalizedTaskName, ListTests) {
150+
//the convention is that a testing task is backed by a sourceSet with the same name
151+
dependsOn subProject.getTasks().getByName("${taskName}Classes")
152+
doFirst {
153+
//we want to set the test scanning classpath to only the output of the sourceSet - this prevents dependencies polluting the list
154+
scanClassPath = task.getTestClassesDirs() ? task.getTestClassesDirs() : Collections.emptyList()
155+
}
156+
}
157+
158+
//convenience task to utilize the output of the test listing task to display to local console, useful for debugging missing tests
159+
def createdPrintTask = subProject.tasks.create("printTestsFor" + capitalizedTaskName) {
160+
dependsOn createdListTask
161+
doLast {
162+
createdListTask.getTestsForFork(
163+
getPropertyAsInt(subProject, "dockerFork", 0),
164+
getPropertyAsInt(subProject, "dockerForks", 1),
165+
42).forEach { testName ->
166+
println testName
167+
}
168+
}
169+
}
170+
171+
subProject.logger.info("created task: " + createdListTask.getPath() + " in project: " + subProject + " it dependsOn: " + createdListTask.dependsOn)
172+
subProject.logger.info("created task: " + createdPrintTask.getPath() + " in project: " + subProject + " it dependsOn: " + createdPrintTask.dependsOn)
173+
174+
return createdListTask as ListTests
175+
}
176+
177+
}

0 commit comments

Comments
 (0)