Dean Birch | 62c4f08 | 2020-01-17 16:13:26 +0000 | [diff] [blame] | 1 | #!/usr/bin/env groovy |
| 2 | //------------------------------------------------------------------------------- |
Xinyu Zhang | 29a57cc | 2023-02-03 10:08:48 +0800 | [diff] [blame] | 3 | // Copyright (c) 2020-2023, Arm Limited and Contributors. All rights reserved. |
Dean Birch | 62c4f08 | 2020-01-17 16:13:26 +0000 | [diff] [blame] | 4 | // |
| 5 | // SPDX-License-Identifier: BSD-3-Clause |
| 6 | // |
| 7 | //------------------------------------------------------------------------------- |
| 8 | |
Dean Birch | d0f9f8c | 2020-03-26 11:10:33 +0000 | [diff] [blame] | 9 | @Library('trustedfirmware') _ |
| 10 | import org.trustedfirmware.Gerrit |
Dean Birch | a6ede7e | 2020-03-13 14:00:33 +0000 | [diff] [blame] | 11 | |
Xinyu Zhang | 0aced4c | 2021-09-03 17:06:21 +0800 | [diff] [blame] | 12 | failure_states = ["FAILURE", "ABORTED", "UNSTABLE", "NOT_BUILT"] |
| 13 | |
Xinyu Zhang | 29a57cc | 2023-02-03 10:08:48 +0800 | [diff] [blame] | 14 | @NonCPS |
| 15 | def isPerPatchJob() { |
| 16 | def upstream_jobs = manager.build.getAction(hudson.model.CauseAction.class).getCauses() |
| 17 | if (upstream_jobs.size() > 0 && upstream_jobs[0] instanceof hudson.model.Cause.UpstreamCause) { |
| 18 | print("Parent project cause: ") |
| 19 | println(upstream_jobs[0].upstreamProject) |
| 20 | if (upstream_jobs[0].upstreamProject.endsWith("tf-m-static")) { |
| 21 | return true |
| 22 | } |
| 23 | } |
| 24 | return false |
| 25 | } |
| 26 | |
Xinyu Zhang | 0aced4c | 2021-09-03 17:06:21 +0800 | [diff] [blame] | 27 | def submit_lava_tests(config, results, build_res, params, params_collection) { |
| 28 | print("Doing LAVA stuff for ${build_res.getAbsoluteUrl()}") |
Xinyu Zhang | 0aced4c | 2021-09-03 17:06:21 +0800 | [diff] [blame] | 29 | params += string(name: 'BUILD_NUMBER', value: "${build_res.number}") |
| 30 | params += string(name: 'BUILD_URL', value: build_res.getAbsoluteUrl()) |
| 31 | params += string(name: 'LAVA_URL', value: env.LAVA_URL) |
Paul Sokolovsky | daa430a | 2022-01-06 09:59:16 +0300 | [diff] [blame] | 32 | params += string(name: 'CI_SCRIPTS_REPO', value: env.CI_SCRIPTS_REPO) |
Xinyu Zhang | 0aced4c | 2021-09-03 17:06:21 +0800 | [diff] [blame] | 33 | params += string(name: 'CI_SCRIPTS_BRANCH', value: env.CI_SCRIPTS_BRANCH) |
| 34 | params += string(name: 'LAVA_CREDENTIALS', value: env.LAVA_CREDENTIALS) |
| 35 | params += string(name: 'CODE_COVERAGE_EN', value: env.CODE_COVERAGE_EN) |
Xinyu Zhang | 0c8d320 | 2022-10-19 14:42:47 +0800 | [diff] [blame] | 36 | params += string(name: 'DEVICE_FILTER', value: env.DEVICE_FILTER) |
Xinyu Zhang | 0aced4c | 2021-09-03 17:06:21 +0800 | [diff] [blame] | 37 | def lava_res = build(job: 'tf-m-lava-submit', parameters: params, propagate: false) |
| 38 | def lava_resubmitted = false |
| 39 | if (lava_res.result in failure_states) { |
| 40 | error("LAVA Create and Submit failed at ${lava_res.getAbsoluteUrl()}") |
| 41 | } else { |
| 42 | lava_des = lava_res.getDescription() |
| 43 | if (lava_des.contains(" Submitted twice!")) { |
| 44 | lava_resubmitted = true |
| 45 | lava_des = lava_des - " Submitted twice!" |
| 46 | } |
| 47 | results['lava_jobs'] += lava_des |
| 48 | } |
| 49 | links = "Build Config: ${config}\n" |
| 50 | links += "Build URL: ${build_res.getAbsoluteUrl()}\n" |
| 51 | links += "LAVA Submit: ${lava_res.getAbsoluteUrl()}" |
| 52 | if (lava_resubmitted) { |
| 53 | links += "\nLAVA Job Re-Submitted!" |
| 54 | } |
| 55 | print(links) |
| 56 | } |
| 57 | |
Xinyu Zhang | 0f78e7a | 2022-10-17 13:55:52 +0800 | [diff] [blame] | 58 | def listConfigs(config_list, filter_group) { |
Xinyu Zhang | 4b10980 | 2022-10-18 17:22:07 +0800 | [diff] [blame] | 59 | def build_config_list_raw = sh(script: |
| 60 | """ |
Xinyu Zhang | 4b10980 | 2022-10-18 17:22:07 +0800 | [diff] [blame] | 61 | python3 ./tf-m-ci-scripts/configs.py -g ${filter_group.replace(" ", " -g ")}\n |
Xinyu Zhang | 4b10980 | 2022-10-18 17:22:07 +0800 | [diff] [blame] | 62 | """, returnStdout: true).trim() |
| 63 | def build_config_list = build_config_list_raw.tokenize('\n') |
| 64 | config_list.addAll(build_config_list) |
Dean Birch | 62c4f08 | 2020-01-17 16:13:26 +0000 | [diff] [blame] | 65 | } |
| 66 | |
Xinyu Zhang | 0f78e7a | 2022-10-17 13:55:52 +0800 | [diff] [blame] | 67 | def obtainBuildParams(config) { |
| 68 | def build_params = [:] |
Xinyu Zhang | 4b10980 | 2022-10-18 17:22:07 +0800 | [diff] [blame] | 69 | build_config_params = sh(script: |
| 70 | """ |
Xinyu Zhang | 4b10980 | 2022-10-18 17:22:07 +0800 | [diff] [blame] | 71 | python3 ./tf-m-ci-scripts/configs.py ${config}\n |
Xinyu Zhang | 4b10980 | 2022-10-18 17:22:07 +0800 | [diff] [blame] | 72 | """, returnStdout: true).trim() |
Dean Birch | 62c4f08 | 2020-01-17 16:13:26 +0000 | [diff] [blame] | 73 | def lines = build_config_params.tokenize('\n') |
| 74 | for (String line : lines) { |
| 75 | def key, value |
| 76 | (key, value) = line.tokenize('=') |
Xinyu Zhang | 0f78e7a | 2022-10-17 13:55:52 +0800 | [diff] [blame] | 77 | build_params[key] = value |
| 78 | } |
| 79 | return build_params |
| 80 | } |
| 81 | |
| 82 | def buildConfig(config, results) { |
| 83 | def params = [] |
| 84 | params_collection = obtainBuildParams(config) |
| 85 | params_collection.each { param -> |
| 86 | params += string(name: param.key, value:param.value) |
Dean Birch | 62c4f08 | 2020-01-17 16:13:26 +0000 | [diff] [blame] | 87 | } |
| 88 | params += string(name: 'GERRIT_BRANCH', value: env.GERRIT_BRANCH) |
Dean Birch | d0f9f8c | 2020-03-26 11:10:33 +0000 | [diff] [blame] | 89 | params += string(name: 'GERRIT_HOST', value: env.GERRIT_HOST) |
| 90 | params += string(name: 'GERRIT_CHANGE_NUMBER', value: env.GERRIT_CHANGE_NUMBER) |
| 91 | params += string(name: 'GERRIT_PATCHSET_REVISION', value: env.GERRIT_PATCHSET_REVISION) |
Dean Birch | 62c4f08 | 2020-01-17 16:13:26 +0000 | [diff] [blame] | 92 | params += string(name: 'GERRIT_REFSPEC', value: env.GERRIT_REFSPEC) |
Karl Zhang | 02d3035 | 2020-08-20 13:48:52 +0800 | [diff] [blame] | 93 | params += string(name: 'MBEDTLS_VERSION', value: env.MBEDTLS_VERSION) |
Dean Birch | 62c4f08 | 2020-01-17 16:13:26 +0000 | [diff] [blame] | 94 | params += string(name: 'CODE_REPO', value: env.CODE_REPO) |
Karl Zhang | f6f467e | 2020-07-10 16:24:45 +0800 | [diff] [blame] | 95 | params += string(name: 'CODE_COVERAGE_EN', value: env.CODE_COVERAGE_EN) |
Paul Sokolovsky | daa430a | 2022-01-06 09:59:16 +0300 | [diff] [blame] | 96 | params += string(name: 'CI_SCRIPTS_REPO', value: env.CI_SCRIPTS_REPO) |
Colin Thorbinson | 58703db | 2020-11-24 12:02:19 +0000 | [diff] [blame] | 97 | params += string(name: 'CI_SCRIPTS_BRANCH', value: env.CI_SCRIPTS_BRANCH) |
Leonardo Sandoval | 7090b2c | 2021-09-17 13:20:44 -0500 | [diff] [blame] | 98 | params += string(name: 'MCUBOOT_REFSPEC', value: env.MCUBOOT_REFSPEC) |
| 99 | params += string(name: 'MCUBOOT_URL', value: env.MCUBOOT_URL) |
| 100 | params += string(name: 'MBEDTLS_VERSION', value: env.MBEDTLS_VERSION) |
Paul Sokolovsky | 8d28871 | 2022-01-13 00:53:03 +0300 | [diff] [blame] | 101 | params += string(name: 'MBEDTLS_URL', value: env.MBEDTLS_URL) |
Bence Balogh | 79fda44 | 2022-10-14 18:01:37 +0200 | [diff] [blame] | 102 | params += string(name: 'TFM_EXTRAS_REFSPEC', value: env.TFM_EXTRAS_REFSPEC) |
| 103 | params += string(name: 'TFM_EXTRAS_URL', value: env.TFM_EXTRAS_URL) |
Leonardo Sandoval | 7090b2c | 2021-09-17 13:20:44 -0500 | [diff] [blame] | 104 | params += string(name: 'TFM_TESTS_REFSPEC', value: env.TFM_TESTS_REFSPEC) |
| 105 | params += string(name: 'TFM_TESTS_URL', value: env.TFM_TESTS_URL) |
| 106 | params += string(name: 'PSA_ARCH_TESTS_VERSION', value: env.PSA_ARCH_TESTS_VERSION) |
| 107 | params += string(name: 'PSA_ARCH_TESTS_URL', value: env.PSA_ARCH_TESTS_URL) |
Xinyu Zhang | c7ad082 | 2022-11-23 17:54:26 +0800 | [diff] [blame] | 108 | params += string(name: 'QCBOR_VERSION', value: env.QCBOR_VERSION) |
| 109 | params += string(name: 'QCBOR_URL', value: env.QCBOR_URL) |
Bence Balogh | 79fda44 | 2022-10-14 18:01:37 +0200 | [diff] [blame] | 110 | params += string(name: 'FREERTOS_KERNEL_REFSPEC', value: env.FREERTOS_KERNEL_REFSPEC) |
| 111 | params += string(name: 'FREERTOS_KERNEL_URL', value: env.FREERTOS_KERNEL_URL) |
Leonardo Sandoval | 7090b2c | 2021-09-17 13:20:44 -0500 | [diff] [blame] | 112 | params += string(name: 'SHARE_FOLDER', value: env.SHARE_FOLDER) |
Xinyu Zhang | 470b3c4 | 2022-09-19 14:41:45 +0800 | [diff] [blame] | 113 | params += string(name: 'SQUAD_CONFIGURATIONS', value: env.SQUAD_CONFIGURATIONS) |
Dean Birch | a6ede7e | 2020-03-13 14:00:33 +0000 | [diff] [blame] | 114 | return { -> results |
| 115 | def build_res = build(job: 'tf-m-build-config', parameters: params, propagate: false) |
Dean Birch | a6ede7e | 2020-03-13 14:00:33 +0000 | [diff] [blame] | 116 | def build_url = build_res.getAbsoluteUrl() |
Xinyu Zhang | 0f78e7a | 2022-10-17 13:55:52 +0800 | [diff] [blame] | 117 | results['builds'][config] = build_res |
Xinyu Zhang | 0aced4c | 2021-09-03 17:06:21 +0800 | [diff] [blame] | 118 | |
Dean Birch | a6ede7e | 2020-03-13 14:00:33 +0000 | [diff] [blame] | 119 | print("${build_res.number}: ${config} ${build_res.result} ${build_url}") |
Xinyu Zhang | f2ba911 | 2021-09-02 13:31:16 +0800 | [diff] [blame] | 120 | |
| 121 | // Filter out configs do not need LAVA tests |
| 122 | |
| 123 | // Configs with build failure do not need LAVA tests |
Xinyu Zhang | 6aa579a | 2022-04-20 11:36:27 +0800 | [diff] [blame] | 124 | if (build_res.result in failure_states) { |
Dean Birch | a6ede7e | 2020-03-13 14:00:33 +0000 | [diff] [blame] | 125 | error("Build failed at ${build_url}") |
Arthur She | 3c0dadd | 2021-11-18 21:17:48 -0800 | [diff] [blame] | 126 | } else { |
| 127 | // Build successful |
Xinyu Zhang | 6aa579a | 2022-04-20 11:36:27 +0800 | [diff] [blame] | 128 | // Job tf-m-extra-build does not need LAVA tests |
| 129 | if (env.JOB_NAME.equals("tf-m-extra-build")) { |
| 130 | print("LAVA is not needed in tf-m-extra-build job.") |
| 131 | } |
Xinyu Zhang | 22a1275 | 2022-10-10 17:21:21 +0800 | [diff] [blame] | 132 | // Submit LAVA tests |
| 133 | else { |
Arthur She | 3c0dadd | 2021-11-18 21:17:48 -0800 | [diff] [blame] | 134 | submit_lava_tests(config, results, build_res, params, params_collection) |
Arthur She | 3c0dadd | 2021-11-18 21:17:48 -0800 | [diff] [blame] | 135 | } |
Dean Birch | 62c4f08 | 2020-01-17 16:13:26 +0000 | [diff] [blame] | 136 | } |
| 137 | } |
| 138 | } |
| 139 | |
Matthew Hart | 06340d7 | 2020-06-15 16:08:20 +0100 | [diff] [blame] | 140 | def buildDocs(results) { |
Dean Birch | 62c4f08 | 2020-01-17 16:13:26 +0000 | [diff] [blame] | 141 | def params = [] |
| 142 | params += string(name: 'GERRIT_BRANCH', value: env.GERRIT_BRANCH) |
Dean Birch | d0f9f8c | 2020-03-26 11:10:33 +0000 | [diff] [blame] | 143 | params += string(name: 'GERRIT_HOST', value: env.GERRIT_HOST) |
| 144 | params += string(name: 'GERRIT_CHANGE_NUMBER', value: env.GERRIT_CHANGE_NUMBER) |
| 145 | params += string(name: 'GERRIT_PATCHSET_REVISION', value: env.GERRIT_PATCHSET_REVISION) |
Dean Birch | 62c4f08 | 2020-01-17 16:13:26 +0000 | [diff] [blame] | 146 | params += string(name: 'GERRIT_REFSPEC', value: env.GERRIT_REFSPEC) |
Dean Birch | 62c4f08 | 2020-01-17 16:13:26 +0000 | [diff] [blame] | 147 | params += string(name: 'CODE_REPO', value: env.CODE_REPO) |
Paul Sokolovsky | daa430a | 2022-01-06 09:59:16 +0300 | [diff] [blame] | 148 | params += string(name: 'CI_SCRIPTS_REPO', value: env.CI_SCRIPTS_REPO) |
Colin Thorbinson | 58703db | 2020-11-24 12:02:19 +0000 | [diff] [blame] | 149 | params += string(name: 'CI_SCRIPTS_BRANCH', value: env.CI_SCRIPTS_BRANCH) |
Leonardo Sandoval | 7090b2c | 2021-09-17 13:20:44 -0500 | [diff] [blame] | 150 | params += string(name: 'SHARE_FOLDER', value: env.SHARE_FOLDER) |
Matthew Hart | 06340d7 | 2020-06-15 16:08:20 +0100 | [diff] [blame] | 151 | return { -> results |
Dean Birch | 62c4f08 | 2020-01-17 16:13:26 +0000 | [diff] [blame] | 152 | def res = build(job: 'tf-m-build-docs', parameters: params, propagate:false) |
| 153 | print("${res.number}: Docs ${res.result} ${res.getAbsoluteUrl()}") |
Matthew Hart | 06340d7 | 2020-06-15 16:08:20 +0100 | [diff] [blame] | 154 | results['docs'] = [res.number, res.result, params] |
Dean Birch | a6ede7e | 2020-03-13 14:00:33 +0000 | [diff] [blame] | 155 | if (res.result in ["FAILURE", "ABORTED", "UNSTABLE", "NOT_BUILT"]) { |
Dean Birch | 62c4f08 | 2020-01-17 16:13:26 +0000 | [diff] [blame] | 156 | error("Build failed at ${res.getAbsoluteUrl()}") |
| 157 | } |
| 158 | } |
| 159 | } |
| 160 | |
Xinyu Zhang | 0f78e7a | 2022-10-17 13:55:52 +0800 | [diff] [blame] | 161 | def generateEmailBody(stage, results) { |
| 162 | // Results format: [CONFIG_NAME: [URL: "", RESULT: "", ...]] |
Xinyu Zhang | 38a1887 | 2020-11-23 16:45:28 +0800 | [diff] [blame] | 163 | body = "Check console output at ${env.BUILD_URL} \n\n" |
| 164 | |
| 165 | body += "Failed Jobs:\n" |
Xinyu Zhang | 0f78e7a | 2022-10-17 13:55:52 +0800 | [diff] [blame] | 166 | results.each { job -> |
| 167 | if (job.value['RESULT'] == 'FAILURE') { |
| 168 | body += "${job.key} ${job.value['URL']}\n" |
| 169 | } |
Xinyu Zhang | 38a1887 | 2020-11-23 16:45:28 +0800 | [diff] [blame] | 170 | } |
| 171 | |
| 172 | body += "\nFor detailed ${stage} results please refer to \ |
| 173 | ${env.BUILD_URL}artifact/${stage}_results.csv \n" |
| 174 | return body |
| 175 | } |
| 176 | |
Xinyu Zhang | 0f78e7a | 2022-10-17 13:55:52 +0800 | [diff] [blame] | 177 | def emailNotification(success, stage, results) { |
Karl Zhang | 0413e97 | 2020-09-18 17:59:26 +0800 | [diff] [blame] | 178 | script { |
Xinyu Zhang | 953dc23 | 2022-09-07 18:27:35 +0800 | [diff] [blame] | 179 | if (env.EMAIL_NOTIFICATION) { |
Xinyu Zhang | 0f78e7a | 2022-10-17 13:55:52 +0800 | [diff] [blame] | 180 | if (success == true) { |
| 181 | print("Skip sending as Success for ${stage}") |
Karl Zhang | 182ecdf | 2020-10-10 09:52:12 +0800 | [diff] [blame] | 182 | } |
| 183 | else { |
| 184 | emailext ( |
Xinyu Zhang | 0f78e7a | 2022-10-17 13:55:52 +0800 | [diff] [blame] | 185 | subject: ("Job ${env.JOB_NAME} ${stage} ${env.BUILD_NUMBER} fail"), |
| 186 | body: generateEmailBody(stage, results), |
Karl Zhang | 182ecdf | 2020-10-10 09:52:12 +0800 | [diff] [blame] | 187 | to: "${EMAIL_NOTIFICATION}" |
| 188 | ) |
| 189 | } |
Karl Zhang | 0413e97 | 2020-09-18 17:59:26 +0800 | [diff] [blame] | 190 | } |
| 191 | } /* script */ |
| 192 | } |
| 193 | |
Xinyu Zhang | 82dab28 | 2022-10-09 16:33:19 +0800 | [diff] [blame] | 194 | def parseTestResults(output) { |
Xinyu Zhang | 82dab28 | 2022-10-09 16:33:19 +0800 | [diff] [blame] | 195 | def test_results = [:] |
| 196 | records = output.split('\nLAVA Test Config:\n') |
| 197 | if (records.size() < 2) { |
| 198 | return test_results |
| 199 | } |
| 200 | records[1..-1].each { record -> |
| 201 | config_name = "" |
| 202 | metadata = [:] |
| 203 | record.split('\n').each { line -> |
| 204 | record_metadata = line.split(': ') |
Paul Sokolovsky | 412d62a | 2022-12-02 21:35:22 +0300 | [diff] [blame] | 205 | // Skip lines which are not "metadata" |
| 206 | if (record_metadata.size() < 2) { |
| 207 | return |
| 208 | } |
Xinyu Zhang | 82dab28 | 2022-10-09 16:33:19 +0800 | [diff] [blame] | 209 | if (record_metadata[0] == 'Config Name') { |
| 210 | config_name = record_metadata[1] |
| 211 | } else { |
| 212 | metadata[record_metadata[0]] = record_metadata[1] |
| 213 | } |
| 214 | } |
Xinyu Zhang | 0f78e7a | 2022-10-17 13:55:52 +0800 | [diff] [blame] | 215 | test_results[config_name] = ['URL': metadata['LAVA link'], |
| 216 | 'RESULT': metadata['Test Result']] |
Xinyu Zhang | 82dab28 | 2022-10-09 16:33:19 +0800 | [diff] [blame] | 217 | } |
Xinyu Zhang | 0f78e7a | 2022-10-17 13:55:52 +0800 | [diff] [blame] | 218 | |
| 219 | writeCsv(test_results, "test_results.csv") |
| 220 | |
Xinyu Zhang | 82dab28 | 2022-10-09 16:33:19 +0800 | [diff] [blame] | 221 | return test_results |
| 222 | } |
| 223 | |
Xinyu Zhang | 57d2d8c | 2022-10-24 14:27:30 +0800 | [diff] [blame] | 224 | def verifyTestStatus(output) { |
| 225 | g = new Gerrit() |
| 226 | if (output.contains('FAILURE')) { |
| 227 | score = -1 |
| 228 | } else { |
| 229 | score = 1 |
| 230 | } |
| 231 | g.verifyStatus(score, "lava_test", "test") |
| 232 | if (score < 0) { |
| 233 | error("Marking job as failed due to failed tests.") |
| 234 | } |
| 235 | } |
| 236 | |
xinyu-tfm | b4fc041 | 2020-08-19 10:49:51 +0800 | [diff] [blame] | 237 | def generateCsvContent(results) { |
Xinyu Zhang | 0f78e7a | 2022-10-17 13:55:52 +0800 | [diff] [blame] | 238 | // Results format: [CONFIG_NAME: [URL: "", RESULT: "", ...]] |
| 239 | // CSV format: CONFIG_NAME, RESULT |
| 240 | csv_header = obtainBuildParams(results.keySet()[0]).keySet().toList() |
| 241 | csv_header.add('RESULT') |
| 242 | csv_content = [csv_header] |
xinyu-tfm | b4fc041 | 2020-08-19 10:49:51 +0800 | [diff] [blame] | 243 | results.each { result -> |
Xinyu Zhang | 0f78e7a | 2022-10-17 13:55:52 +0800 | [diff] [blame] | 244 | build_params = [] |
| 245 | obtainBuildParams(result.key).each { config -> |
| 246 | build_params.add(config.value) |
Xinyu Zhang | 4f2ef5a | 2020-11-09 18:11:43 +0800 | [diff] [blame] | 247 | } |
Xinyu Zhang | 0f78e7a | 2022-10-17 13:55:52 +0800 | [diff] [blame] | 248 | build_params.add(result.value['RESULT']) |
| 249 | csv_content.add(build_params) |
xinyu-tfm | b4fc041 | 2020-08-19 10:49:51 +0800 | [diff] [blame] | 250 | } |
Xinyu Zhang | 0f78e7a | 2022-10-17 13:55:52 +0800 | [diff] [blame] | 251 | return csv_content |
xinyu-tfm | b4fc041 | 2020-08-19 10:49:51 +0800 | [diff] [blame] | 252 | } |
| 253 | |
Xinyu Zhang | 0f78e7a | 2022-10-17 13:55:52 +0800 | [diff] [blame] | 254 | def generateHtmlContent(results) { |
| 255 | // Results format: [CONFIG_NAME: [URL: "", RESULT: "", ...]] |
| 256 | // HTML format: CONFIG_NAME: Job/Logs/Artifacts RESULT |
| 257 | htmlContent = [] |
| 258 | results.each { result -> |
| 259 | htmlContent.add("${result.key}: <a href=\"${result.value['URL']}\">Job</a>/<a href=\"${result.value['URL']}/consoleText\">Logs</a>/<a href=\"${result.value['URL']}/artifact/\">Artifacts</a> ${result.value['RESULT']}<br/>") |
| 260 | } |
| 261 | htmlContent.sort() |
| 262 | return htmlContent.join("\n") |
| 263 | } |
| 264 | |
| 265 | def writeCsv(results, file_name) { |
xinyu-tfm | b4fc041 | 2020-08-19 10:49:51 +0800 | [diff] [blame] | 266 | def csvContent = generateCsvContent(results) |
Xinyu Zhang | 0f78e7a | 2022-10-17 13:55:52 +0800 | [diff] [blame] | 267 | writeCSV file: file_name, records: csvContent, format: CSVFormat.EXCEL |
| 268 | sh(script: """./tf-m-ci-scripts/report_parser/report_csv_helper.py \ |
| 269 | --input-file ${file_name} --output-file ${file_name} \ |
| 270 | """, returnStdout: true) |
| 271 | archiveArtifacts file_name |
xinyu-tfm | b4fc041 | 2020-08-19 10:49:51 +0800 | [diff] [blame] | 272 | } |
Dean Birch | a6ede7e | 2020-03-13 14:00:33 +0000 | [diff] [blame] | 273 | |
Xinyu Zhang | 0f78e7a | 2022-10-17 13:55:52 +0800 | [diff] [blame] | 274 | def writeHTML(results, file_name) { |
| 275 | def buildLinks = generateHtmlContent(results) |
| 276 | writeFile file: file_name, text: buildLinks |
| 277 | archiveArtifacts file_name |
Dean Birch | a6ede7e | 2020-03-13 14:00:33 +0000 | [diff] [blame] | 278 | } |
| 279 | |
Matthew Hart | fb6fd36 | 2020-03-04 21:03:59 +0000 | [diff] [blame] | 280 | def lineInString(string, match) { |
| 281 | def lines = string.split("\n") |
| 282 | def result = lines.findAll { it.contains(match) } |
| 283 | return result[0] |
| 284 | } |
| 285 | |
| 286 | def getResult(string, match) { |
| 287 | line = lineInString(string, match) |
Dean Birch | 1d545c0 | 2020-05-29 14:09:21 +0100 | [diff] [blame] | 288 | a = line.split(match)[1].split(' ') |
| 289 | score = a[0] |
| 290 | if (a.size() > 1) |
| 291 | { |
| 292 | fail_text = a[1..-1].join(" ") |
| 293 | return [score, fail_text] |
| 294 | } |
| 295 | return [score, ""] |
Matthew Hart | fb6fd36 | 2020-03-04 21:03:59 +0000 | [diff] [blame] | 296 | } |
| 297 | |
| 298 | def submitJobsToList(results) { |
| 299 | def all_jobs = [] |
| 300 | for (String result : results){ |
| 301 | jobs_s = result.split('JOBS: ') |
| 302 | if (jobs_s.size() > 1) { |
| 303 | all_jobs += jobs_s[1] |
| 304 | } |
| 305 | } |
| 306 | return(all_jobs) |
| 307 | } |
| 308 | |
Dean Birch | 62c4f08 | 2020-01-17 16:13:26 +0000 | [diff] [blame] | 309 | def configs = [] |
| 310 | def builds = [:] |
Matthew Hart | fb6fd36 | 2020-03-04 21:03:59 +0000 | [diff] [blame] | 311 | def results = [:] |
Dean Birch | 62c4f08 | 2020-01-17 16:13:26 +0000 | [diff] [blame] | 312 | |
Xinyu Zhang | 4cdfd1b | 2021-05-21 15:10:49 +0800 | [diff] [blame] | 313 | timestamps { |
| 314 | node("docker-amd64-tf-m-bionic") { |
| 315 | stage("Init") { |
| 316 | cleanWs() |
| 317 | dir("tf-m-ci-scripts") { |
| 318 | checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]]) |
Xinyu Zhang | 8472b85 | 2021-09-04 00:13:34 +0800 | [diff] [blame] | 319 | sh "git rev-parse --short HEAD" |
Leonardo Sandoval | 7090b2c | 2021-09-17 13:20:44 -0500 | [diff] [blame] | 320 | // Clone TF-M repositories so share folder can be reused by downstream jobs |
| 321 | sh "./clone.sh" |
Xinyu Zhang | 4cdfd1b | 2021-05-21 15:10:49 +0800 | [diff] [blame] | 322 | } |
| 323 | } |
Xinyu Zhang | 6b456c2 | 2022-10-13 15:48:58 +0800 | [diff] [blame] | 324 | |
Xinyu Zhang | 4cdfd1b | 2021-05-21 15:10:49 +0800 | [diff] [blame] | 325 | stage("Configs") { |
Xinyu Zhang | 4b10980 | 2022-10-18 17:22:07 +0800 | [diff] [blame] | 326 | dir(".") { |
| 327 | // Populate configs |
| 328 | listConfigs(configs, env.FILTER_GROUP) |
| 329 | results['builds'] = [:] |
| 330 | results['lava_jobs'] = [] |
| 331 | for (config in configs) { |
| 332 | builds[config] = buildConfig(config, results) |
| 333 | } |
| 334 | if (!env.JOB_NAME.equals("tf-m-extra-build")) { |
| 335 | builds["docs"] = buildDocs(results) |
| 336 | } |
Xinyu Zhang | 351f14c | 2021-11-15 14:30:09 +0800 | [diff] [blame] | 337 | } |
Dean Birch | 62c4f08 | 2020-01-17 16:13:26 +0000 | [diff] [blame] | 338 | } |
Karl Zhang | fec8410 | 2020-06-24 09:56:36 +0800 | [diff] [blame] | 339 | |
Xinyu Zhang | 6b456c2 | 2022-10-13 15:48:58 +0800 | [diff] [blame] | 340 | stage("Builds") { |
| 341 | def verify = 1 |
| 342 | def success = true |
Xinyu Zhang | 4b10980 | 2022-10-18 17:22:07 +0800 | [diff] [blame] | 343 | dir(".") { |
| 344 | try { |
| 345 | parallel(builds) |
| 346 | } catch (Exception e) { |
| 347 | print(e) |
| 348 | manager.buildFailure() |
| 349 | verify = -1 |
| 350 | success = false |
| 351 | } finally { |
| 352 | print("Verifying status") |
| 353 | g = new Gerrit() |
| 354 | g.verifyStatus(verify, 'tf-m-build', 'build') |
| 355 | print("Generating build results summary.") |
| 356 | def build_results_for_summary = [:] |
| 357 | results['builds'].each { build -> |
Xinyu Zhang | 0d85cc3 | 2022-10-19 17:58:32 +0800 | [diff] [blame] | 358 | build_results_for_summary[build.key] = ['URL': build.value.getAbsoluteUrl(), |
| 359 | 'RESULT': build.value.result] |
Xinyu Zhang | 4b10980 | 2022-10-18 17:22:07 +0800 | [diff] [blame] | 360 | } |
| 361 | emailNotification(success, 'build', build_results_for_summary) |
| 362 | writeCsv(build_results_for_summary, "build_results.csv") |
| 363 | writeHTML(build_results_for_summary, "build_links.html") |
Xinyu Zhang | 0f78e7a | 2022-10-17 13:55:52 +0800 | [diff] [blame] | 364 | } |
Xinyu Zhang | 6b456c2 | 2022-10-13 15:48:58 +0800 | [diff] [blame] | 365 | } |
Xinyu Zhang | 4cdfd1b | 2021-05-21 15:10:49 +0800 | [diff] [blame] | 366 | } |
Xinyu Zhang | 4cdfd1b | 2021-05-21 15:10:49 +0800 | [diff] [blame] | 367 | |
Xinyu Zhang | 4cdfd1b | 2021-05-21 15:10:49 +0800 | [diff] [blame] | 368 | stage("Tests") { |
| 369 | dir("tf-m-ci-scripts") { |
| 370 | checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]]) |
| 371 | } |
| 372 | def all_jobs = [] |
| 373 | def success = true |
Xinyu Zhang | 0f78e7a | 2022-10-17 13:55:52 +0800 | [diff] [blame] | 374 | def test_results = [:] |
Xinyu Zhang | 4cdfd1b | 2021-05-21 15:10:49 +0800 | [diff] [blame] | 375 | print("Wait for LAVA results here...") |
| 376 | try { |
| 377 | all_jobs = submitJobsToList(results['lava_jobs']) |
Paul Sokolovsky | 437bc42 | 2022-02-01 17:57:13 +0300 | [diff] [blame] | 378 | output = "" |
Xinyu Zhang | 4cdfd1b | 2021-05-21 15:10:49 +0800 | [diff] [blame] | 379 | if (all_jobs.size() > 0) { |
Paul Sokolovsky | 3d522f7 | 2022-04-13 11:17:00 +0300 | [diff] [blame] | 380 | dir(".") { |
Xinyu Zhang | 29a57cc | 2023-02-03 10:08:48 +0800 | [diff] [blame] | 381 | if (isPerPatchJob()) { |
| 382 | lava_timeout = 2700 // 45min |
| 383 | } else { |
| 384 | lava_timeout = 18000 // 5h |
| 385 | } |
Xinyu Zhang | 4cdfd1b | 2021-05-21 15:10:49 +0800 | [diff] [blame] | 386 | withCredentials([usernamePassword(credentialsId: env.LAVA_CREDENTIALS, passwordVariable: 'LAVA_TOKEN', usernameVariable: 'LAVA_USER')]) { |
Xinyu Zhang | 0b84b63 | 2023-02-08 14:15:08 +0800 | [diff] [blame^] | 387 | sh(script: """./tf-m-ci-scripts/lava_helper/lava_wait_jobs.py --job-ids ${all_jobs.join(",")} \ |
| 388 | --lava-url ${env.LAVA_URL} --lava-user ${LAVA_USER} --lava-token ${LAVA_TOKEN} \ |
| 389 | --artifacts-path cfgs --lava-timeout ${lava_timeout} > output.log |
| 390 | """) |
Paul Sokolovsky | 0136de0 | 2022-04-19 20:21:47 +0300 | [diff] [blame] | 391 | if (env.CODE_COVERAGE_EN == "TRUE") { |
| 392 | println("Producing merged report") |
| 393 | sh(script: """./tf-m-ci-scripts/lava_helper/codecov_merge.sh""") |
| 394 | archiveArtifacts artifacts: 'merged_report/**', allowEmptyArchive: true |
| 395 | } |
Matthew Hart | fb6fd36 | 2020-03-04 21:03:59 +0000 | [diff] [blame] | 396 | } |
| 397 | } |
| 398 | } |
Xinyu Zhang | 4cdfd1b | 2021-05-21 15:10:49 +0800 | [diff] [blame] | 399 | else { |
| 400 | print("There were no LAVA jobs to test.") |
| 401 | } |
Matthew Hart | fb6fd36 | 2020-03-04 21:03:59 +0000 | [diff] [blame] | 402 | } |
Xinyu Zhang | 4cdfd1b | 2021-05-21 15:10:49 +0800 | [diff] [blame] | 403 | catch (Exception e) { |
Paul Sokolovsky | 2cf562a | 2022-12-02 15:10:58 +0300 | [diff] [blame] | 404 | println("ERROR: ${e}") |
| 405 | // We don't print stacktrace, because Jenkins pipeline use CPS conversion |
| 406 | // of the Groovy code, which leads to incomprehensible stacktraces. |
| 407 | //print(hudson.Functions.printThrowable(org.codehaus.groovy.runtime.StackTraceUtils.sanitize(e))) |
Xinyu Zhang | 4cdfd1b | 2021-05-21 15:10:49 +0800 | [diff] [blame] | 408 | success = false |
| 409 | } finally { |
Xinyu Zhang | 0b84b63 | 2023-02-08 14:15:08 +0800 | [diff] [blame^] | 410 | output = readFile("output.log") |
| 411 | println("--- output from lava_wait_jobs.py ---") |
| 412 | println(output) |
| 413 | println("--- end of output from lava_wait_jobs.py ---") |
| 414 | test_results = parseTestResults(output) |
| 415 | archiveArtifacts artifacts: 'test_summary.*', allowEmptyArchive: true |
Paul Sokolovsky | 3d522f7 | 2022-04-13 11:17:00 +0300 | [diff] [blame] | 416 | archiveArtifacts artifacts: 'cfgs/**', allowEmptyArchive: true |
Xinyu Zhang | 4cdfd1b | 2021-05-21 15:10:49 +0800 | [diff] [blame] | 417 | if (all_jobs.size() > 0) { |
Xinyu Zhang | 0f78e7a | 2022-10-17 13:55:52 +0800 | [diff] [blame] | 418 | emailNotification(success, 'test', test_results) |
Xinyu Zhang | 4cdfd1b | 2021-05-21 15:10:49 +0800 | [diff] [blame] | 419 | } |
Xinyu Zhang | 0b84b63 | 2023-02-08 14:15:08 +0800 | [diff] [blame^] | 420 | verifyTestStatus(output) |
Xinyu Zhang | 4cdfd1b | 2021-05-21 15:10:49 +0800 | [diff] [blame] | 421 | cleanWs() |
| 422 | if (!success) { |
| 423 | error("There was an Error waiting for LAVA jobs") |
| 424 | } |
Matthew Hart | fb6fd36 | 2020-03-04 21:03:59 +0000 | [diff] [blame] | 425 | } |
Dean Birch | a6ede7e | 2020-03-13 14:00:33 +0000 | [diff] [blame] | 426 | } |
| 427 | } |
Dean Birch | a6ede7e | 2020-03-13 14:00:33 +0000 | [diff] [blame] | 428 | } |