|
| 1 | +def Machine = 'none' |
| 2 | +def machine = 'none' |
| 3 | +def HOME = 'none' |
| 4 | +def localworkspace = 'none' |
| 5 | +def commonworkspace = 'none' |
| 6 | + |
| 7 | +pipeline { |
| 8 | + agent { label 'built-in' } |
| 9 | + |
| 10 | + options { |
| 11 | + skipDefaultCheckout() |
| 12 | + buildDiscarder(logRotator(numToKeepStr: '2')) |
| 13 | + } |
| 14 | + |
| 15 | + stages { // This initial stage is used to get the Machine name from the GitHub labels on the PR |
| 16 | + // which is used to designate the Nodes in the Jenkins Controler by the agent label |
| 17 | + // Each Jenknis Node is connected to said machine via an JAVA agent via an ssh tunnel |
| 18 | + |
| 19 | + stage('Get Machine') { |
| 20 | + agent { label 'built-in' } |
| 21 | + steps { |
| 22 | + script { |
| 23 | + localworkspace = env.WORKSPACE |
| 24 | + machine = 'none' |
| 25 | + for (label in pullRequest.labels) { |
| 26 | + echo "Label: ${label}" |
| 27 | + if ((label.matches("CI-Hera-Ready"))) { |
| 28 | + machine = 'hera' |
| 29 | + } else if ((label.matches("CI-Orion-Ready"))) { |
| 30 | + machine = 'orion' |
| 31 | + } else if ((label.matches("CI-Hercules-Ready"))) { |
| 32 | + machine = 'hercules' |
| 33 | + } |
| 34 | + } // createing a second machine varible with first letter capital |
| 35 | + // because the first letter of the machine name is captitalized in the GitHub labels |
| 36 | + Machine = machine[0].toUpperCase() + machine.substring(1) |
| 37 | + } |
| 38 | + } |
| 39 | + } |
| 40 | + |
| 41 | + stage('Get Common Workspace') { |
| 42 | + agent { label "${machine}-emc" } |
| 43 | + steps { |
| 44 | + script { |
| 45 | + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in','Hera-EMC','Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) |
| 46 | + HOME = "${WORKSPACE}/TESTDIR" |
| 47 | + commonworkspace = "${WORKSPACE}" |
| 48 | + sh( script: "mkdir -p ${HOME}/RUNTESTS", returnStatus: true) |
| 49 | + pullRequest.addLabel("CI-${Machine}-Building") |
| 50 | + if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Ready") } ) { |
| 51 | + pullRequest.removeLabel("CI-${Machine}-Ready") |
| 52 | + } |
| 53 | + } |
| 54 | + } |
| 55 | + } |
| 56 | + |
| 57 | + stage('Build System') { |
| 58 | + matrix { |
| 59 | + agent { label "${machine}-emc" } |
| 60 | + //options { |
| 61 | + // throttle(['global_matrix_build']) |
| 62 | + //} |
| 63 | + axes { |
| 64 | + axis { |
| 65 | + name "system" |
| 66 | + values "gfs", "gefs" |
| 67 | + } |
| 68 | + } |
| 69 | + stages { |
| 70 | + stage("build system") { |
| 71 | + steps { |
| 72 | + script { |
| 73 | + def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to build the system on per system basis under the common workspace HOME |
| 74 | + sh( script: "mkdir -p ${HOMEgfs}", returnStatus: true) |
| 75 | + ws(HOMEgfs) { |
| 76 | + env.MACHINE_ID = machine // MACHINE_ID is used in the build scripts to determine the machine and is added to the shell environment |
| 77 | + if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs |
| 78 | + sh( script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore |
| 79 | + ws(commonworkspace) { pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}") } |
| 80 | + } else { |
| 81 | + checkout scm |
| 82 | + sh( script: "source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive", returnStatus: true) |
| 83 | + def builds_file = readYaml file: "ci/cases/yamls/build.yaml" |
| 84 | + def build_args_list = builds_file['builds'] |
| 85 | + def build_args = build_args_list[system].join(" ").trim().replaceAll("null", "") |
| 86 | + dir("${HOMEgfs}/sorc") { |
| 87 | + sh( script: "${build_args}", returnStatus: true) |
| 88 | + sh( script: "./link_workflow.sh", returnStatus: true) |
| 89 | + sh( script: "echo ${HOMEgfs} > BUILT_semaphor", returnStatus: true) |
| 90 | + } |
| 91 | + } |
| 92 | + if ( pullRequest.labels.any{ value -> value.matches("CI-${Machine}-Building") } ) { |
| 93 | + pullRequest.removeLabel("CI-${Machine}-Building") |
| 94 | + } |
| 95 | + pullRequest.addLabel("CI-${Machine}-Running") |
| 96 | + } |
| 97 | + } |
| 98 | + } |
| 99 | + } |
| 100 | + } |
| 101 | + } |
| 102 | + } |
| 103 | + |
| 104 | + stage('Run Tests') { |
| 105 | + matrix { |
| 106 | + agent { label "${machine}-emc" } |
| 107 | + axes { |
| 108 | + axis { |
| 109 | + name "Case" |
| 110 | + values "C48_ATM", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar" // TODO add dynamic list of cases from env vars (needs addtional plugins) |
| 111 | + } |
| 112 | + } |
| 113 | + stages { |
| 114 | + stage('Create Experiment') { |
| 115 | + steps { |
| 116 | + script { |
| 117 | + sh( script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp", returnStatus: true) |
| 118 | + def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp" |
| 119 | + system = yaml_case.experiment.system |
| 120 | + def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to populate the XML on per system basis |
| 121 | + env.RUNTESTS = "${HOME}/RUNTESTS" |
| 122 | + sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStatus: true) |
| 123 | + } |
| 124 | + } |
| 125 | + } |
| 126 | + stage('Run Experiments') { |
| 127 | + steps { |
| 128 | + script { |
| 129 | + HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments |
| 130 | + ws(HOMEgfs) { |
| 131 | + pslot = sh( script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true ).trim() |
| 132 | + pullRequest.comment("**Running experiments: ${Case} on ${Machine}**<br>Built against system **${system}** in directory:<br>`${HOMEgfs}`<br>With the experiment in directory:<br>`${HOME}/RUNTESTS/${pslot}`") |
| 133 | + try { |
| 134 | + sh( script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}", returnStatus: true) |
| 135 | + } catch (Exception e) { |
| 136 | + pullRequest.comment("**FAILURE** running experiments: ${Case} on ${Machine}") |
| 137 | + error("Failed to run experiments ${Case} on ${Machine}") |
| 138 | + } |
| 139 | + pullRequest.comment("**SUCCESS** running experiments: ${Case} on ${Machine}") |
| 140 | + } |
| 141 | + } |
| 142 | + } |
| 143 | + post { |
| 144 | + always { |
| 145 | + script { |
| 146 | + ws (HOMEgfs) { |
| 147 | + for (label in pullRequest.labels) { |
| 148 | + if (label.contains("${Machine}")) { |
| 149 | + pullRequest.removeLabel(label) |
| 150 | + } |
| 151 | + } |
| 152 | + } |
| 153 | + } |
| 154 | + } |
| 155 | + success { |
| 156 | + script { |
| 157 | + ws (HOMEgfs) { |
| 158 | + pullRequest.addLabel("CI-${Machine}-Passed") |
| 159 | + def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) |
| 160 | + pullRequest.comment("**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`") |
| 161 | + } |
| 162 | + } |
| 163 | + } |
| 164 | + failure { |
| 165 | + script { |
| 166 | + ws (HOMEgfs) { |
| 167 | + pullRequest.addLabel("CI-${Machine}-Failed") |
| 168 | + def timestamp = new Date().format("MM dd HH:mm:ss", TimeZone.getTimeZone('America/New_York')) |
| 169 | + pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`") |
| 170 | + if (fileExists('${HOME}/RUNTESTS/ci.log')) { |
| 171 | + def fileContent = readFile '${HOME}/RUNTESTS/ci.log' |
| 172 | + fileContent.eachLine { line -> |
| 173 | + if( line.contains(".log")) { |
| 174 | + archiveArtifacts artifacts: "${line}", fingerprint: true |
| 175 | + } |
| 176 | + } |
| 177 | + } |
| 178 | + } |
| 179 | + } |
| 180 | + } |
| 181 | + } |
| 182 | + } |
| 183 | + } |
| 184 | + } |
| 185 | + } |
| 186 | + } |
| 187 | + |
| 188 | +} |
0 commit comments