|
| 1 | +def Machine = 'none' |
| 2 | +def machine = 'none' |
| 3 | +def HOME = 'none' |
| 4 | +def caseList = '' |
| 5 | +def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES'] |
| 6 | + |
| 7 | +pipeline { |
| 8 | + agent { label 'built-in' } |
| 9 | + |
| 10 | + options { |
| 11 | + skipDefaultCheckout() |
| 12 | + parallelsAlwaysFailFast() |
| 13 | + } |
| 14 | + |
| 15 | + stages { // This initial stage is used to get the Machine name from the GitHub labels on the PR |
| 16 | + // which is used to designate the Nodes in the Jenkins Controler by the agent label |
| 17 | + // Each Jenknis Node is connected to said machine via an JAVA agent via an ssh tunnel |
| 18 | + |
| 19 | + stage('Get Machine') { |
| 20 | + agent { label 'built-in' } |
| 21 | + steps { |
| 22 | + script { |
| 23 | + machine = 'none' |
| 24 | + for (label in pullRequest.labels) { |
| 25 | + echo "Label: ${label}" |
| 26 | + if ((label.matches('CI-Hera-Ready'))) { |
| 27 | + machine = 'hera' |
| 28 | + } else if ((label.matches('CI-Orion-Ready'))) { |
| 29 | + machine = 'orion' |
| 30 | + } else if ((label.matches('CI-Hercules-Ready'))) { |
| 31 | + machine = 'hercules' |
| 32 | + } |
| 33 | + } // createing a second machine varible with first letter capital |
| 34 | + // because the first letter of the machine name is captitalized in the GitHub labels |
| 35 | + Machine = machine[0].toUpperCase() + machine.substring(1) |
| 36 | + } |
| 37 | + } |
| 38 | + } |
| 39 | + |
| 40 | + stage('Get Common Workspace') { |
| 41 | + agent { label "${machine}-emc" } |
| 42 | + steps { |
| 43 | + script { |
| 44 | + ws("${custom_workspace[machine]}/${env.CHANGE_ID}") { |
| 45 | + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) |
| 46 | + HOME = "${WORKSPACE}" |
| 47 | + sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs") |
| 48 | + pullRequest.addLabel("CI-${Machine}-Building") |
| 49 | + if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Ready") }) { |
| 50 | + pullRequest.removeLabel("CI-${Machine}-Ready") |
| 51 | + } |
| 52 | + } |
| 53 | + echo "Building and running on ${Machine} in directory ${HOME}" |
| 54 | + } |
| 55 | + } |
| 56 | + } |
| 57 | + |
| 58 | + stage('Build System') { |
| 59 | + matrix { |
| 60 | + agent { label "${machine}-emc" } |
| 61 | + //options { |
| 62 | + // throttle(['global_matrix_build']) |
| 63 | + //} |
| 64 | + axes { |
| 65 | + axis { |
| 66 | + name 'system' |
| 67 | + values 'gfs', 'gefs' |
| 68 | + } |
| 69 | + } |
| 70 | + stages { |
| 71 | + stage('build system') { |
| 72 | + steps { |
| 73 | + script { |
| 74 | + def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to build the system on per system basis under the common workspace HOME |
| 75 | + sh(script: "mkdir -p ${HOMEgfs}") |
| 76 | + ws(HOMEgfs) { |
| 77 | + if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs |
| 78 | + sh(script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore |
| 79 | + checkout scm |
| 80 | + dir('sorc') { |
| 81 | + sh(script: './link_workflow.sh') |
| 82 | + } |
| 83 | + } else { |
| 84 | + checkout scm |
| 85 | + def builds_file = readYaml file: 'ci/cases/yamls/build.yaml' |
| 86 | + def build_args_list = builds_file['builds'] |
| 87 | + def build_args = build_args_list[system].join(' ').trim().replaceAll('null', '') |
| 88 | + dir("${HOMEgfs}/sorc") { |
| 89 | + sh(script: "${build_args}") |
| 90 | + sh(script: './link_workflow.sh') |
| 91 | + sh(script: "echo ${HOMEgfs} > BUILT_semaphor") |
| 92 | + } |
| 93 | + } |
| 94 | + if (env.CHANGE_ID && system == 'gfs') { |
| 95 | + try { |
| 96 | + if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Building") }) { |
| 97 | + pullRequest.removeLabel("CI-${Machine}-Building") |
| 98 | + } |
| 99 | + pullRequest.addLabel("CI-${Machine}-Running") |
| 100 | + } catch (Exception e) { |
| 101 | + echo "Failed to update label from Buildng to Running: ${e.getMessage()}" |
| 102 | + } |
| 103 | + } |
| 104 | + if (system == 'gfs') { |
| 105 | + caseList = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split() |
| 106 | + } |
| 107 | + } |
| 108 | + } |
| 109 | + } |
| 110 | + } |
| 111 | + } |
| 112 | + } |
| 113 | + } |
| 114 | + |
| 115 | + stage('Run Tests') { |
| 116 | + matrix { |
| 117 | + agent { label "${machine}-emc" } |
| 118 | + axes { |
| 119 | + axis { |
| 120 | + name 'Case' |
| 121 | + // TODO add dynamic list of cases from env vars (needs addtional plugins) |
| 122 | + values 'C48C48_ufs_hybatmDA', 'C48_ATM', 'C48_S2SW', 'C48_S2SWA_gefs', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atm3DVar', 'C96_atmsnowDA' |
| 123 | + } |
| 124 | + } |
| 125 | + stages { |
| 126 | + |
| 127 | + stage('Create Experiments') { |
| 128 | + when { |
| 129 | + expression { return caseList.contains(Case) } |
| 130 | + } |
| 131 | + steps { |
| 132 | + script { |
| 133 | + sh(script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp") |
| 134 | + def yaml_case = readYaml file: "${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp" |
| 135 | + system = yaml_case.experiment.system |
| 136 | + def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to populate the XML on per system basis |
| 137 | + env.RUNTESTS = "${HOME}/RUNTESTS" |
| 138 | + sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml") |
| 139 | + } |
| 140 | + } |
| 141 | + } |
| 142 | + |
| 143 | + stage('Run Experiments') { |
| 144 | + when { |
| 145 | + expression { return caseList.contains(Case) } |
| 146 | + } |
| 147 | + steps { |
| 148 | + script { |
| 149 | + HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments |
| 150 | + ws(HOMEgfs) { |
| 151 | + pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim() |
| 152 | + try { |
| 153 | + sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}") |
| 154 | + } catch (Exception e) { |
| 155 | + sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_all_batch_jobs ${HOME}/RUNTESTS") |
| 156 | + ws(HOME) { |
| 157 | + if (fileExists('RUNTESTS/error.logs')) { |
| 158 | + def fileContent = readFile 'RUNTESTS/error.logs' |
| 159 | + def lines = fileContent.readLines() |
| 160 | + for (line in lines) { |
| 161 | + echo "archiving: ${line}" |
| 162 | + archiveArtifacts artifacts: "${line}", fingerprint: true |
| 163 | + } |
| 164 | + } |
| 165 | + } |
| 166 | + error("Failed to run experiments ${Case} on ${Machine}") |
| 167 | + } |
| 168 | + } |
| 169 | + } |
| 170 | + } |
| 171 | + } |
| 172 | + |
| 173 | + } |
| 174 | + } |
| 175 | + } |
| 176 | + } |
| 177 | + |
| 178 | + post { |
| 179 | + always { |
| 180 | + script { |
| 181 | + if(env.CHANGE_ID) { |
| 182 | + try { |
| 183 | + for (label in pullRequest.labels) { |
| 184 | + if (label.contains("${Machine}")) { |
| 185 | + pullRequest.removeLabel(label) |
| 186 | + } |
| 187 | + } |
| 188 | + } catch (Exception e) { |
| 189 | + echo "Failed to remove labels: ${e.getMessage()}" |
| 190 | + } |
| 191 | + } |
| 192 | + } |
| 193 | + } |
| 194 | + success { |
| 195 | + script { |
| 196 | + if(env.CHANGE_ID) { |
| 197 | + try { |
| 198 | + pullRequest.addLabel("CI-${Machine}-Passed") |
| 199 | + def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York')) |
| 200 | + pullRequest.comment("**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`") |
| 201 | + } catch (Exception e) { |
| 202 | + echo "Failed to add success label or comment: ${e.getMessage()}" |
| 203 | + } |
| 204 | + } |
| 205 | + } |
| 206 | + } |
| 207 | + failure { |
| 208 | + script { |
| 209 | + if(env.CHANGE_ID) { |
| 210 | + try { |
| 211 | + pullRequest.addLabel("CI-${Machine}-Failed") |
| 212 | + def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York')) |
| 213 | + pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`") |
| 214 | + } catch (Exception e) { |
| 215 | + echo "Failed to add failure label or comment: ${e.getMessage()}" |
| 216 | + } |
| 217 | + } |
| 218 | + } |
| 219 | + } |
| 220 | + } |
| 221 | +} |
0 commit comments