Jenkins: Introduce a global share repository mechanism
This is a full refactor of the way jobs fetch git data: with this
approach, only the top level job, the upstream job, would set a share
folder populated with all required repositories, including
trusted-firmware-m, tf-m-tests, tf-m-ci-scripts, mbed-tls, mcuboot and
psa-arch-tests, ultimately consumed by downtream jobs,
i.e. tf-m-build-config.
This would reduce considerably the CI build times, avoiding multiple
clones per job thus overloading the TF git servers.
Signed-off-by: Leonardo Sandoval <leonardo.sandoval@linaro.org>
Change-Id: Iec3f0cc345c5052d64f0ea6ca9da01e9149a7b8b
diff --git a/jenkins/build-config.jpl b/jenkins/build-config.jpl
index 4501e3b..fd22714 100644
--- a/jenkins/build-config.jpl
+++ b/jenkins/build-config.jpl
@@ -24,85 +24,11 @@
node(nodeLabel) {
stage("Init") {
cleanWs()
- dir("trusted-firmware-m") {
- checkout(
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: '$GERRIT_BRANCH']],
- extensions: [[$class: 'BuildChooserSetting', buildChooser: [$class: 'GerritTriggerBuildChooser']]],
- userRemoteConfigs: [[
- credentialsId: 'GIT_SSH_KEY',
- refspec: '$GERRIT_REFSPEC', url: '$CODE_REPO'
- ]]
- ])
- sh "git rev-parse --short HEAD"
- }
dir("tf-m-ci-scripts") {
checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]])
sh "git rev-parse --short HEAD"
- }
- dir("mbedtls") {
- checkout(
- changelog: false,
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: 'FETCH_HEAD']],
- userRemoteConfigs: [[
- refspec: 'refs/tags/$MBEDTLS_VERSION',
- url: params.MBEDTLS_URL
- ]]
- ]
- )
- sh "git rev-parse --short HEAD"
- }
- dir("mcuboot") {
- checkout(
- changelog: false,
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: 'FETCH_HEAD']],
- userRemoteConfigs: [[
- refspec: '$MCUBOOT_REFSPEC',
- url: params.MCUBOOT_URL
- ]]
- ]
- )
- sh "git rev-parse --short HEAD"
- }
- dir("tf-m-tests") {
- checkout(
- changelog: false,
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: 'FETCH_HEAD']],
- userRemoteConfigs: [[
- refspec: '$TFM_TESTS_REFSPEC',
- url: params.TFM_TESTS_URL
- ]]
- ]
- )
- sh "git rev-parse --short HEAD"
- }
- if (env.PSA_API_SUITE != "") {
- dir("psa-arch-tests") {
- checkout(
- changelog: false,
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: 'FETCH_HEAD']],
- userRemoteConfigs: [[
- refspec: '$PSA_ARCH_TESTS_VERSION',
- url: params.PSA_ARCH_TESTS_URL
- ]]
- ]
- )
- sh "git rev-parse --short HEAD"
- }
+ // Clone TF-M repositories so share folder can be reused by downstream jobs
+ sh "./clone.sh"
}
}
try {
diff --git a/jenkins/build-docs.jpl b/jenkins/build-docs.jpl
index 8e9e4ba..934f902 100644
--- a/jenkins/build-docs.jpl
+++ b/jenkins/build-docs.jpl
@@ -13,63 +13,10 @@
node("docker-amd64-tf-m-bionic") {
stage("Init") {
cleanWs()
- dir("trusted-firmware-m") {
- checkout(
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: '$GERRIT_BRANCH']],
- extensions: [[$class: 'BuildChooserSetting', buildChooser: [$class: 'GerritTriggerBuildChooser']]],
- userRemoteConfigs: [[
- credentialsId: 'GIT_SSH_KEY',
- refspec: '$GERRIT_REFSPEC', url: '$CODE_REPO'
- ]]
- ])
- }
dir("tf-m-ci-scripts") {
checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]])
- }
- dir("mbedtls") {
- checkout(
- changelog: false,
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: 'FETCH_HEAD']],
- userRemoteConfigs: [[
- refspec: 'refs/tags/$MBEDTLS_VERSION',
- url: params.MBEDTLS_URL
- ]]
- ]
- )
- }
- dir("mcuboot") {
- checkout(
- changelog: false,
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: 'FETCH_HEAD']],
- userRemoteConfigs: [[
- refspec: '$MCUBOOT_REFSPEC',
- url: params.MCUBOOT_URL
- ]]
- ]
- )
- }
- dir("tf-m-tests") {
- checkout(
- changelog: false,
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: 'FETCH_HEAD']],
- userRemoteConfigs: [[
- refspec: '$TFM_TESTS_REFSPEC',
- url: params.TFM_TESTS_URL
- ]]
- ]
- )
+ // Clone TF-M repositories so share folder can be reused by downstream jobs
+ sh "./clone.sh"
}
}
try {
diff --git a/jenkins/checkpatch.jpl b/jenkins/checkpatch.jpl
index 5e8637d..2f87a98 100644
--- a/jenkins/checkpatch.jpl
+++ b/jenkins/checkpatch.jpl
@@ -11,23 +11,12 @@
timestamps {
node("docker-amd64-tf-m-bionic") {
+ // Clone TF-M repositories so share folder can be reused by downstream jobs
stage("Init") {
cleanWs()
- dir("trusted-firmware-m") {
- checkout(
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: '$GERRIT_BRANCH']],
- extensions: [[$class: 'BuildChooserSetting', buildChooser: [$class: 'GerritTriggerBuildChooser']]],
- userRemoteConfigs: [[
- credentialsId: 'GIT_SSH_KEY',
- refspec: '$GERRIT_REFSPEC', url: '$CODE_REPO'
- ]]
- ])
- }
dir("tf-m-ci-scripts") {
checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]])
+ sh "./clone.sh"
}
}
stage("Check") {
diff --git a/jenkins/ci.jpl b/jenkins/ci.jpl
index 534bbe1..81c18a4 100644
--- a/jenkins/ci.jpl
+++ b/jenkins/ci.jpl
@@ -192,6 +192,15 @@
params += string(name: 'CODE_REPO', value: env.CODE_REPO)
params += string(name: 'CODE_COVERAGE_EN', value: env.CODE_COVERAGE_EN)
params += string(name: 'CI_SCRIPTS_BRANCH', value: env.CI_SCRIPTS_BRANCH)
+ params += string(name: 'MCUBOOT_REFSPEC', value: env.MCUBOOT_REFSPEC)
+ params += string(name: 'MCUBOOT_URL', value: env.MCUBOOT_URL)
+ params += string(name: 'MBEDTLS_VERSION', value: env.MBEDTLS_VERSION)
+ params += string(name: 'MCUBOOT_URL', value: env.MBEDTLS_URL)
+ params += string(name: 'TFM_TESTS_REFSPEC', value: env.TFM_TESTS_REFSPEC)
+ params += string(name: 'TFM_TESTS_URL', value: env.TFM_TESTS_URL)
+ params += string(name: 'PSA_ARCH_TESTS_VERSION', value: env.PSA_ARCH_TESTS_VERSION)
+ params += string(name: 'PSA_ARCH_TESTS_URL', value: env.PSA_ARCH_TESTS_URL)
+ params += string(name: 'SHARE_FOLDER', value: env.SHARE_FOLDER)
if (env.JOB_NAME.equals("tf-m-nightly")) { //Setting the Memory footprint gathering.
params += string(name: 'SQUAD_CONFIGURATIONS', value: env.SQUAD_CONFIGURATIONS)
}
@@ -248,6 +257,15 @@
params += string(name: 'MBEDTLS_VERSION', value: env.MBEDTLS_VERSION)
params += string(name: 'CODE_REPO', value: env.CODE_REPO)
params += string(name: 'CI_SCRIPTS_BRANCH', value: env.CI_SCRIPTS_BRANCH)
+ params += string(name: 'MCUBOOT_REFSPEC', value: env.MCUBOOT_REFSPEC)
+ params += string(name: 'MCUBOOT_URL', value: env.MCUBOOT_URL)
+ params += string(name: 'MBEDTLS_VERSION', value: env.MBEDTLS_VERSION)
+ params += string(name: 'MCUBOOT_URL', value: env.MBEDTLS_URL)
+ params += string(name: 'TFM_TESTS_REFSPEC', value: env.TFM_TESTS_REFSPEC)
+ params += string(name: 'TFM_TESTS_URL', value: env.TFM_TESTS_URL)
+ params += string(name: 'PSA_ARCH_TESTS_VERSION', value: env.PSA_ARCH_TESTS_VERSION)
+ params += string(name: 'PSA_ARCH_TESTS_URL', value: env.PSA_ARCH_TESTS_URL)
+ params += string(name: 'SHARE_FOLDER', value: env.SHARE_FOLDER)
return { -> results
def res = build(job: 'tf-m-build-docs', parameters: params, propagate:false)
print("${res.number}: Docs ${res.result} ${res.getAbsoluteUrl()}")
@@ -475,6 +493,8 @@
dir("tf-m-ci-scripts") {
checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]])
sh "git rev-parse --short HEAD"
+ // Clone TF-M repositories so share folder can be reused by downstream jobs
+ sh "./clone.sh"
}
}
stage("Configs") {
diff --git a/jenkins/cppcheck.jpl b/jenkins/cppcheck.jpl
index b88c7f2..69cc9b0 100644
--- a/jenkins/cppcheck.jpl
+++ b/jenkins/cppcheck.jpl
@@ -11,65 +11,12 @@
timestamps {
node("docker-amd64-tf-m-bionic") {
+ // Clone TF-M repositories so share folder can be reused by downstream jobs
stage("Init") {
cleanWs()
- dir("trusted-firmware-m") {
- checkout(
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: '$GERRIT_BRANCH']],
- extensions: [[$class: 'BuildChooserSetting', buildChooser: [$class: 'GerritTriggerBuildChooser']]],
- userRemoteConfigs: [[
- credentialsId: 'GIT_SSH_KEY',
- refspec: '$GERRIT_REFSPEC', url: '$CODE_REPO'
- ]]
- ])
- }
dir("tf-m-ci-scripts") {
checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]])
- }
- dir("mbedtls") {
- checkout(
- changelog: false,
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: 'FETCH_HEAD']],
- userRemoteConfigs: [[
- refspec: 'refs/tags/$MBEDTLS_VERSION',
- url: params.MBEDTLS_URL
- ]]
- ]
- )
- }
- dir("mcuboot") {
- checkout(
- changelog: false,
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: 'FETCH_HEAD']],
- userRemoteConfigs: [[
- refspec: '$MCUBOOT_REFSPEC',
- url: params.MCUBOOT_URL
- ]]
- ]
- )
- }
- dir("tf-m-tests") {
- checkout(
- changelog: false,
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: 'FETCH_HEAD']],
- userRemoteConfigs: [[
- refspec: '$TFM_TESTS_REFSPEC',
- url: params.TFM_TESTS_URL
- ]]
- ]
- )
+ sh "./clone.sh"
}
}
stage("Check") {
diff --git a/jenkins/static-checks.jpl b/jenkins/static-checks.jpl
index 4aacad1..66faecc 100644
--- a/jenkins/static-checks.jpl
+++ b/jenkins/static-checks.jpl
@@ -13,21 +13,10 @@
node("docker-amd64-tf-m-bionic") {
stage("Init") {
cleanWs()
- dir("trusted-firmware-m") {
- checkout(
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: '$GERRIT_BRANCH']],
- extensions: [[$class: 'BuildChooserSetting', buildChooser: [$class: 'GerritTriggerBuildChooser']]],
- userRemoteConfigs: [[
- credentialsId: 'GIT_SSH_KEY',
- refspec: '$GERRIT_REFSPEC', url: '$CODE_REPO'
- ]]
- ])
- }
dir("tf-m-ci-scripts") {
checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]])
+ // Clone TF-M repositories so share folder can be reused by downstream jobs
+ sh "./clone.sh"
}
}
stage("Check") {
diff --git a/jenkins/static.jpl b/jenkins/static.jpl
index a7fe36d..4341b4c 100644
--- a/jenkins/static.jpl
+++ b/jenkins/static.jpl
@@ -17,6 +17,16 @@
params += string(name: 'GERRIT_PATCHSET_REVISION', value: env.GERRIT_PATCHSET_REVISION ?: '')
params += string(name: 'MBEDTLS_VERSION', value: env.MBEDTLS_VERSION)
params += string(name: 'CODE_REPO', value: env.CODE_REPO)
+ params += string(name: 'CI_SCRIPTS_BRANCH', value: env.CI_SCRIPTS_BRANCH)
+ params += string(name: 'MCUBOOT_REFSPEC', value: env.MCUBOOT_REFSPEC)
+ params += string(name: 'MCUBOOT_URL', value: env.MCUBOOT_URL)
+ params += string(name: 'MBEDTLS_VERSION', value: env.MBEDTLS_VERSION)
+ params += string(name: 'MCUBOOT_URL', value: env.MBEDTLS_URL)
+ params += string(name: 'TFM_TESTS_REFSPEC', value: env.TFM_TESTS_REFSPEC)
+ params += string(name: 'TFM_TESTS_URL', value: env.TFM_TESTS_URL)
+ params += string(name: 'PSA_ARCH_TESTS_VERSION', value: env.PSA_ARCH_TESTS_VERSION)
+ params += string(name: 'PSA_ARCH_TESTS_URL', value: env.PSA_ARCH_TESTS_URL)
+ params += string(name: 'SHARE_FOLDER', value: env.SHARE_FOLDER)
build(job: job_name, parameters: params)
}
}
@@ -24,6 +34,16 @@
def status = 1
timestamps {
+ node("docker-amd64-tf-m-bionic") {
+ stage("Init") {
+ cleanWs()
+ dir("tf-m-ci-scripts") {
+ checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]])
+ // Clone TF-M repositories so share folder can be reused by downstream jobs
+ sh "./clone.sh"
+ }
+ }
+ }
stage("Static Checks") {
def checks = [:]
checks["cppcheck"] = trigger("tf-m-cppcheck")