Jenkins: Introduce a global share repository mechanism
This is a full refactor of the way jobs fetch git data: with this
approach, only the top level job, the upstream job, would set a share
folder populated with all required repositories, including
trusted-firmware-m, tf-m-tests, tf-m-ci-scripts, mbed-tls, mcuboot and
psa-arch-tests, ultimately consumed by downtream jobs,
i.e. tf-m-build-config.
This would reduce considerably the CI build times, avoiding multiple
clones per job thus overloading the TF git servers.
Signed-off-by: Leonardo Sandoval <leonardo.sandoval@linaro.org>
Change-Id: Iec3f0cc345c5052d64f0ea6ca9da01e9149a7b8b
diff --git a/clone.sh b/clone.sh
new file mode 100755
index 0000000..009ba66
--- /dev/null
+++ b/clone.sh
@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2021 Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# Clones and checkout TF-M related repositories in case these are not present
+# under SHARE_FOLDER, otherwise copy the share repositories into current folder
+# (workspace)
+
+#
+# The way it works is simple: the top level job sets the SHARE_FOLDER
+# parameter based on its name and number on top of the share
+# volume (/srv/shared/<job name>/<job number>) then it calls the clone
+# script (clone.sh), which in turn it fetches the repositories mentioned
+# above. Jobs triggered on behalf of the latter, share the same
+# SHARE_FOLDER value, and these in turn also call the clone script, but
+# in this case, the script detects that the folder is already populated so
+# its role is to simply copy the repositories into the job's
+# workspace. As seen, all jobs work with repositories on their own
+# workspace, which are just copies of the share folder, so there is no
+# change of a race condition, i.e every job works with its own copy. The
+# worst case scenario is where the down-level job,
+# i.e. tf-m-build-config, uses its default SHARE_FOLDER value, in this
+# case, it would simply clone its own repositories without reusing any
+# file however the current approach prevents the latter unless the job
+# is triggered manually from the buider job itself.
+#
+
+# Global defaults
+GIT_CLONE_PARAMS="--no-checkout"
+
+# Must projects
+TFM_PROJECT="${CODE_REPO:?}"
+TFM_REFSPEC="${GERRIT_REFSPEC:?}"
+TFM_NAME="trusted-firmware-m"
+
+SCRIPTS_PROJECT="${CI_SCRIPTS_REPO:?}"
+SCRIPTS_REFSPEC="${CI_SCRIPTS_BRANCH:?}"
+SCRIPTS_NAME="tf-m-ci-scripts"
+
+# Optional projects
+TFM_TESTS_PROJECT="${TFM_TESTS_URL:-}"
+TFM_TESTS_REFSPEC="${TFM_TESTS_REFSPEC:-}"
+TFM_TESTS_NAME="tf-m-tests"
+
+MBEDTLS_PROJECT="${MBEDTLS_URL:-}"
+MBEDTLS_REFSPEC="${MBEDTLS_VERSION:-}"
+MBEDTLS_NAME="mbedtls"
+
+MCUBOOT_PROJECT="${MCUBOOT_URL:-}"
+MCUBOOT_REFSPEC="${MCUBOOT_REFSPEC:-}"
+MCUBOOT_NAME="mcuboot"
+
+PSA_ARCH_TESTS_PROJECT="${PSA_ARCH_TESTS_URL:-}"
+PSA_ARCH_TESTS_REFSPEC="${PSA_ARCH_TESTS_VERSION:-}"
+PSA_ARCH_TESTS_NAME="psa-arch-tests"
+
+# Array containing "<repo url>;"<repo name>;<refspec>" elements
+repos=(
+ "${TFM_PROJECT};${TFM_NAME};${TFM_REFSPEC}"
+ "${TFM_TESTS_PROJECT};${TFM_TESTS_NAME};${TFM_TESTS_REFSPEC}"
+ "${SCRIPTS_PROJECT};${SCRIPTS_NAME};${SCRIPTS_REFSPEC}"
+ "${MBEDTLS_PROJECT};${MBEDTLS_NAME};${MBEDTLS_REFSPEC}"
+ "${MCUBOOT_PROJECT};${MCUBOOT_NAME};${MCUBOOT_REFSPEC}"
+ "${PSA_ARCH_TESTS_PROJECT};${PSA_ARCH_TESTS_NAME};${PSA_ARCH_TESTS_REFSPEC}"
+)
+
+# Take into consideration non-CI runs where SHARE_FOLDER variable
+# may not be present
+if [ -z "${SHARE_FOLDER}" ]; then
+ # Default Jenkins values
+ SHARE_VOLUME="${SHARE_VOLUME:-$PWD}"
+ JOB_NAME="${JOB_NAME:-local}"
+ BUILD_NUMBER="${BUILD_NUMBER:-0}"
+ SHARE_FOLDER=${SHARE_VOLUME}/${JOB_NAME}/${BUILD_NUMBER}
+fi
+
+echo "Share Folder ${SHARE_FOLDER}"
+
+# clone git repos
+for repo in ${repos[@]}; do
+
+ # parse the repo elements
+ REPO_URL="$(echo "${repo}" | awk -F ';' '{print $1}')"
+ REPO_NAME="$(echo "${repo}" | awk -F ';' '{print $2}')"
+ REPO_REFSPEC="$(echo "${repo}" | awk -F ';' '{print $3}')"
+
+ # in case repository is not define, just skip it
+ if [ -z "${REPO_URL}" ]; then
+ continue
+ fi
+
+ # clone and checkout in case it does not exit
+ if [ ! -d ${SHARE_FOLDER}/${REPO_NAME} ]; then
+ git clone --quiet ${GIT_CLONE_PARAMS} ${REPO_URL} ${SHARE_FOLDER}/${REPO_NAME}
+
+ # fetch and checkout the corresponding refspec
+ cd ${SHARE_FOLDER}/${REPO_NAME}
+
+ git fetch ${REPO_URL} ${REPO_REFSPEC} && git checkout FETCH_HEAD
+ echo -e "\n\nShare Folder ${SHARE_FOLDER}/${REPO_NAME} $(git rev-parse --short HEAD)\n\n"
+ cd $OLDPWD
+
+ else
+ # otherwise just show the head's log
+ cd ${SHARE_FOLDER}/${REPO_NAME}
+ echo -e "\n\nShare Folder ${SHARE_FOLDER}/${REPO_NAME} $(git rev-parse --short HEAD)\n\n"
+ cd $OLDPWD
+ fi
+
+ # copy repository into pwd dir (workspace in CI), so each job would work
+ # on its own workspace
+ cp -a -f ${SHARE_FOLDER}/${REPO_NAME} ${WORKSPACE}/${REPO_NAME}
+
+done
diff --git a/jenkins/build-config.jpl b/jenkins/build-config.jpl
index 4501e3b..fd22714 100644
--- a/jenkins/build-config.jpl
+++ b/jenkins/build-config.jpl
@@ -24,85 +24,11 @@
node(nodeLabel) {
stage("Init") {
cleanWs()
- dir("trusted-firmware-m") {
- checkout(
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: '$GERRIT_BRANCH']],
- extensions: [[$class: 'BuildChooserSetting', buildChooser: [$class: 'GerritTriggerBuildChooser']]],
- userRemoteConfigs: [[
- credentialsId: 'GIT_SSH_KEY',
- refspec: '$GERRIT_REFSPEC', url: '$CODE_REPO'
- ]]
- ])
- sh "git rev-parse --short HEAD"
- }
dir("tf-m-ci-scripts") {
checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]])
sh "git rev-parse --short HEAD"
- }
- dir("mbedtls") {
- checkout(
- changelog: false,
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: 'FETCH_HEAD']],
- userRemoteConfigs: [[
- refspec: 'refs/tags/$MBEDTLS_VERSION',
- url: params.MBEDTLS_URL
- ]]
- ]
- )
- sh "git rev-parse --short HEAD"
- }
- dir("mcuboot") {
- checkout(
- changelog: false,
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: 'FETCH_HEAD']],
- userRemoteConfigs: [[
- refspec: '$MCUBOOT_REFSPEC',
- url: params.MCUBOOT_URL
- ]]
- ]
- )
- sh "git rev-parse --short HEAD"
- }
- dir("tf-m-tests") {
- checkout(
- changelog: false,
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: 'FETCH_HEAD']],
- userRemoteConfigs: [[
- refspec: '$TFM_TESTS_REFSPEC',
- url: params.TFM_TESTS_URL
- ]]
- ]
- )
- sh "git rev-parse --short HEAD"
- }
- if (env.PSA_API_SUITE != "") {
- dir("psa-arch-tests") {
- checkout(
- changelog: false,
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: 'FETCH_HEAD']],
- userRemoteConfigs: [[
- refspec: '$PSA_ARCH_TESTS_VERSION',
- url: params.PSA_ARCH_TESTS_URL
- ]]
- ]
- )
- sh "git rev-parse --short HEAD"
- }
+ // Clone TF-M repositories so share folder can be reused by downstream jobs
+ sh "./clone.sh"
}
}
try {
diff --git a/jenkins/build-docs.jpl b/jenkins/build-docs.jpl
index 8e9e4ba..934f902 100644
--- a/jenkins/build-docs.jpl
+++ b/jenkins/build-docs.jpl
@@ -13,63 +13,10 @@
node("docker-amd64-tf-m-bionic") {
stage("Init") {
cleanWs()
- dir("trusted-firmware-m") {
- checkout(
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: '$GERRIT_BRANCH']],
- extensions: [[$class: 'BuildChooserSetting', buildChooser: [$class: 'GerritTriggerBuildChooser']]],
- userRemoteConfigs: [[
- credentialsId: 'GIT_SSH_KEY',
- refspec: '$GERRIT_REFSPEC', url: '$CODE_REPO'
- ]]
- ])
- }
dir("tf-m-ci-scripts") {
checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]])
- }
- dir("mbedtls") {
- checkout(
- changelog: false,
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: 'FETCH_HEAD']],
- userRemoteConfigs: [[
- refspec: 'refs/tags/$MBEDTLS_VERSION',
- url: params.MBEDTLS_URL
- ]]
- ]
- )
- }
- dir("mcuboot") {
- checkout(
- changelog: false,
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: 'FETCH_HEAD']],
- userRemoteConfigs: [[
- refspec: '$MCUBOOT_REFSPEC',
- url: params.MCUBOOT_URL
- ]]
- ]
- )
- }
- dir("tf-m-tests") {
- checkout(
- changelog: false,
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: 'FETCH_HEAD']],
- userRemoteConfigs: [[
- refspec: '$TFM_TESTS_REFSPEC',
- url: params.TFM_TESTS_URL
- ]]
- ]
- )
+ // Clone TF-M repositories so share folder can be reused by downstream jobs
+ sh "./clone.sh"
}
}
try {
diff --git a/jenkins/checkpatch.jpl b/jenkins/checkpatch.jpl
index 5e8637d..2f87a98 100644
--- a/jenkins/checkpatch.jpl
+++ b/jenkins/checkpatch.jpl
@@ -11,23 +11,12 @@
timestamps {
node("docker-amd64-tf-m-bionic") {
+ // Clone TF-M repositories so share folder can be reused by downstream jobs
stage("Init") {
cleanWs()
- dir("trusted-firmware-m") {
- checkout(
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: '$GERRIT_BRANCH']],
- extensions: [[$class: 'BuildChooserSetting', buildChooser: [$class: 'GerritTriggerBuildChooser']]],
- userRemoteConfigs: [[
- credentialsId: 'GIT_SSH_KEY',
- refspec: '$GERRIT_REFSPEC', url: '$CODE_REPO'
- ]]
- ])
- }
dir("tf-m-ci-scripts") {
checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]])
+ sh "./clone.sh"
}
}
stage("Check") {
diff --git a/jenkins/ci.jpl b/jenkins/ci.jpl
index 534bbe1..81c18a4 100644
--- a/jenkins/ci.jpl
+++ b/jenkins/ci.jpl
@@ -192,6 +192,15 @@
params += string(name: 'CODE_REPO', value: env.CODE_REPO)
params += string(name: 'CODE_COVERAGE_EN', value: env.CODE_COVERAGE_EN)
params += string(name: 'CI_SCRIPTS_BRANCH', value: env.CI_SCRIPTS_BRANCH)
+ params += string(name: 'MCUBOOT_REFSPEC', value: env.MCUBOOT_REFSPEC)
+ params += string(name: 'MCUBOOT_URL', value: env.MCUBOOT_URL)
+ params += string(name: 'MBEDTLS_VERSION', value: env.MBEDTLS_VERSION)
+ params += string(name: 'MCUBOOT_URL', value: env.MBEDTLS_URL)
+ params += string(name: 'TFM_TESTS_REFSPEC', value: env.TFM_TESTS_REFSPEC)
+ params += string(name: 'TFM_TESTS_URL', value: env.TFM_TESTS_URL)
+ params += string(name: 'PSA_ARCH_TESTS_VERSION', value: env.PSA_ARCH_TESTS_VERSION)
+ params += string(name: 'PSA_ARCH_TESTS_URL', value: env.PSA_ARCH_TESTS_URL)
+ params += string(name: 'SHARE_FOLDER', value: env.SHARE_FOLDER)
if (env.JOB_NAME.equals("tf-m-nightly")) { //Setting the Memory footprint gathering.
params += string(name: 'SQUAD_CONFIGURATIONS', value: env.SQUAD_CONFIGURATIONS)
}
@@ -248,6 +257,15 @@
params += string(name: 'MBEDTLS_VERSION', value: env.MBEDTLS_VERSION)
params += string(name: 'CODE_REPO', value: env.CODE_REPO)
params += string(name: 'CI_SCRIPTS_BRANCH', value: env.CI_SCRIPTS_BRANCH)
+ params += string(name: 'MCUBOOT_REFSPEC', value: env.MCUBOOT_REFSPEC)
+ params += string(name: 'MCUBOOT_URL', value: env.MCUBOOT_URL)
+ params += string(name: 'MBEDTLS_VERSION', value: env.MBEDTLS_VERSION)
+ params += string(name: 'MCUBOOT_URL', value: env.MBEDTLS_URL)
+ params += string(name: 'TFM_TESTS_REFSPEC', value: env.TFM_TESTS_REFSPEC)
+ params += string(name: 'TFM_TESTS_URL', value: env.TFM_TESTS_URL)
+ params += string(name: 'PSA_ARCH_TESTS_VERSION', value: env.PSA_ARCH_TESTS_VERSION)
+ params += string(name: 'PSA_ARCH_TESTS_URL', value: env.PSA_ARCH_TESTS_URL)
+ params += string(name: 'SHARE_FOLDER', value: env.SHARE_FOLDER)
return { -> results
def res = build(job: 'tf-m-build-docs', parameters: params, propagate:false)
print("${res.number}: Docs ${res.result} ${res.getAbsoluteUrl()}")
@@ -475,6 +493,8 @@
dir("tf-m-ci-scripts") {
checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]])
sh "git rev-parse --short HEAD"
+ // Clone TF-M repositories so share folder can be reused by downstream jobs
+ sh "./clone.sh"
}
}
stage("Configs") {
diff --git a/jenkins/cppcheck.jpl b/jenkins/cppcheck.jpl
index b88c7f2..69cc9b0 100644
--- a/jenkins/cppcheck.jpl
+++ b/jenkins/cppcheck.jpl
@@ -11,65 +11,12 @@
timestamps {
node("docker-amd64-tf-m-bionic") {
+ // Clone TF-M repositories so share folder can be reused by downstream jobs
stage("Init") {
cleanWs()
- dir("trusted-firmware-m") {
- checkout(
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: '$GERRIT_BRANCH']],
- extensions: [[$class: 'BuildChooserSetting', buildChooser: [$class: 'GerritTriggerBuildChooser']]],
- userRemoteConfigs: [[
- credentialsId: 'GIT_SSH_KEY',
- refspec: '$GERRIT_REFSPEC', url: '$CODE_REPO'
- ]]
- ])
- }
dir("tf-m-ci-scripts") {
checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]])
- }
- dir("mbedtls") {
- checkout(
- changelog: false,
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: 'FETCH_HEAD']],
- userRemoteConfigs: [[
- refspec: 'refs/tags/$MBEDTLS_VERSION',
- url: params.MBEDTLS_URL
- ]]
- ]
- )
- }
- dir("mcuboot") {
- checkout(
- changelog: false,
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: 'FETCH_HEAD']],
- userRemoteConfigs: [[
- refspec: '$MCUBOOT_REFSPEC',
- url: params.MCUBOOT_URL
- ]]
- ]
- )
- }
- dir("tf-m-tests") {
- checkout(
- changelog: false,
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: 'FETCH_HEAD']],
- userRemoteConfigs: [[
- refspec: '$TFM_TESTS_REFSPEC',
- url: params.TFM_TESTS_URL
- ]]
- ]
- )
+ sh "./clone.sh"
}
}
stage("Check") {
diff --git a/jenkins/static-checks.jpl b/jenkins/static-checks.jpl
index 4aacad1..66faecc 100644
--- a/jenkins/static-checks.jpl
+++ b/jenkins/static-checks.jpl
@@ -13,21 +13,10 @@
node("docker-amd64-tf-m-bionic") {
stage("Init") {
cleanWs()
- dir("trusted-firmware-m") {
- checkout(
- poll: false,
- scm: [
- $class: 'GitSCM',
- branches: [[name: '$GERRIT_BRANCH']],
- extensions: [[$class: 'BuildChooserSetting', buildChooser: [$class: 'GerritTriggerBuildChooser']]],
- userRemoteConfigs: [[
- credentialsId: 'GIT_SSH_KEY',
- refspec: '$GERRIT_REFSPEC', url: '$CODE_REPO'
- ]]
- ])
- }
dir("tf-m-ci-scripts") {
checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]])
+ // Clone TF-M repositories so share folder can be reused by downstream jobs
+ sh "./clone.sh"
}
}
stage("Check") {
diff --git a/jenkins/static.jpl b/jenkins/static.jpl
index a7fe36d..4341b4c 100644
--- a/jenkins/static.jpl
+++ b/jenkins/static.jpl
@@ -17,6 +17,16 @@
params += string(name: 'GERRIT_PATCHSET_REVISION', value: env.GERRIT_PATCHSET_REVISION ?: '')
params += string(name: 'MBEDTLS_VERSION', value: env.MBEDTLS_VERSION)
params += string(name: 'CODE_REPO', value: env.CODE_REPO)
+ params += string(name: 'CI_SCRIPTS_BRANCH', value: env.CI_SCRIPTS_BRANCH)
+ params += string(name: 'MCUBOOT_REFSPEC', value: env.MCUBOOT_REFSPEC)
+ params += string(name: 'MCUBOOT_URL', value: env.MCUBOOT_URL)
+ params += string(name: 'MBEDTLS_VERSION', value: env.MBEDTLS_VERSION)
+ params += string(name: 'MCUBOOT_URL', value: env.MBEDTLS_URL)
+ params += string(name: 'TFM_TESTS_REFSPEC', value: env.TFM_TESTS_REFSPEC)
+ params += string(name: 'TFM_TESTS_URL', value: env.TFM_TESTS_URL)
+ params += string(name: 'PSA_ARCH_TESTS_VERSION', value: env.PSA_ARCH_TESTS_VERSION)
+ params += string(name: 'PSA_ARCH_TESTS_URL', value: env.PSA_ARCH_TESTS_URL)
+ params += string(name: 'SHARE_FOLDER', value: env.SHARE_FOLDER)
build(job: job_name, parameters: params)
}
}
@@ -24,6 +34,16 @@
def status = 1
timestamps {
+ node("docker-amd64-tf-m-bionic") {
+ stage("Init") {
+ cleanWs()
+ dir("tf-m-ci-scripts") {
+ checkout([$class: 'GitSCM', branches: [[name: '$CI_SCRIPTS_BRANCH']], userRemoteConfigs: [[credentialsId: 'GIT_SSH_KEY', url: '$CI_SCRIPTS_REPO']]])
+ // Clone TF-M repositories so share folder can be reused by downstream jobs
+ sh "./clone.sh"
+ }
+ }
+ }
stage("Static Checks") {
def checks = [:]
checks["cppcheck"] = trigger("tf-m-cppcheck")