Various fixes
* Retrieve build commands from build_manager
* fixing psa build dir
* Use different node labels for different builds
* Add script to download jenkins artifacts
* Verify status per stage
* Moving code to library
* Ability to comment on gerrit change
Change-Id: I390674b7ed6cfd20e4746a2d32e708fd6855857b
Signed-off-by: Dean Birch <dean.birch@arm.com>
diff --git a/jenkins/build-config.jpl b/jenkins/build-config.jpl
index aea0266..b4263cd 100644
--- a/jenkins/build-config.jpl
+++ b/jenkins/build-config.jpl
@@ -6,7 +6,15 @@
//
//-------------------------------------------------------------------------------
-node("docker-amd64-xenial") {
+@Library('trustedfirmware') _
+import org.trustedfirmware.Gerrit
+
+def nodeLabel = "docker-amd64-xenial"
+if (env.COMPILER == "ARMCLANG") {
+ nodeLabel = "docker-amd64-xenial-armclang"
+}
+
+node(nodeLabel) {
stage("Init") {
cleanWs()
dir("trusted-firmware-m") {
@@ -45,12 +53,39 @@
wget -O cmsis.pack -q \${JENKINS_URL}/userContent/ARM.CMSIS.${CMSIS_VERSION}.pack
unzip -o -d CMSIS_5 cmsis.pack
"""
+ if (env.PSA_API_SUITE != "") {
+ dir("psa-arch-tests") {
+ checkout(
+ changelog: false,
+ poll: false,
+ scm: [
+ $class: 'GitSCM',
+ branches: [[name: 'FETCH_HEAD']],
+ userRemoteConfigs: [[
+ refspec: 'refs/tags/v20.03_API1.0',
+ url: 'https://github.com/ARM-software/psa-arch-tests'
+ ]]
+ ]
+ )
+ }
+ }
}
- stage("Build") {
- sh "tf-m-ci-scripts/run-build.sh 2>&1 | tee build.log"
- }
- stage("Post") {
- archiveArtifacts 'trusted-firmware-m/build/install/**,build.log'
+ try {
+ verify = 1
+ stage("Build") {
+ tee("build.log") {
+ sh "tf-m-ci-scripts/run-build.sh"
+ }
+ }
+ stage("Post") {
+ archiveArtifacts 'trusted-firmware-m/build/install/**,build.log'
+ }
+ } catch (Exception e) {
+ manager.buildFailure()
+ verify = -1
+ } finally {
+ g = new Gerrit()
+ g.verifyStatusInWorkspace(verify, env.CONFIG_NAME, 'build')
cleanWs()
}
}
diff --git a/jenkins/build-docs.jpl b/jenkins/build-docs.jpl
index ba5e1e1..71993b7 100644
--- a/jenkins/build-docs.jpl
+++ b/jenkins/build-docs.jpl
@@ -6,6 +6,9 @@
//
//-------------------------------------------------------------------------------
+@Library('trustedfirmware') _
+import org.trustedfirmware.Gerrit
+
node("docker-amd64-xenial") {
stage("Init") {
cleanWs()
@@ -46,11 +49,20 @@
unzip -o -d CMSIS_5 cmsis.pack
"""
}
- stage("Build") {
- sh "tf-m-ci-scripts/build-docs.sh"
- }
- stage("Post") {
- archiveArtifacts 'trusted-firmware-m/build/install/**'
+ try {
+ verify = 1
+ stage("Build") {
+ sh "tf-m-ci-scripts/build-docs.sh"
+ }
+ stage("Post") {
+ archiveArtifacts 'trusted-firmware-m/build/install/**'
+ }
+ } catch (Exception e) {
+ manager.buildFailure()
+ verify = -1
+ } finally {
+ g = new Gerrit()
+ g.verifyStatusInWorkspace(verify, 'tf-m-build-docs', 'build')
cleanWs()
}
}
diff --git a/jenkins/checkpatch.jpl b/jenkins/checkpatch.jpl
index 5a447b5..7855cb0 100644
--- a/jenkins/checkpatch.jpl
+++ b/jenkins/checkpatch.jpl
@@ -6,22 +6,8 @@
//
//-------------------------------------------------------------------------------
-def verifyStatus(value, stage_name) {
- withCredentials([usernamePassword(credentialsId: 'VERIFY_STATUS', passwordVariable: 'VERIFY_PASSWORD', usernameVariable: 'VERIFY_USER')]) {
- sh """
-if [ -z "\$GERRIT_HOST" ] ; then
- echo Not running for a Gerrit change, skipping vote.
- exit 0
-fi
-if [ ! -d venv ] ; then
- virtualenv -p \$(which python3) venv
-fi
-. venv/bin/activate
-pip -q install requests
-./tf-m-ci-scripts/jenkins/verify.py --value ${value} --verify-name tf-m-${stage_name} --user \$VERIFY_USER
-"""
- }
-}
+@Library('trustedfirmware') _
+import org.trustedfirmware.Gerrit
node("docker-amd64-xenial") {
stage("Init") {
@@ -75,7 +61,8 @@
manager.buildFailure()
verify = -1
} finally {
- verifyStatus(verify, 'checkpatch')
+ g = new Gerrit()
+ g.verifyStatusInWorkspace(verify, 'checkpatch', 'static')
cleanWs()
}
}
diff --git a/jenkins/ci.jpl b/jenkins/ci.jpl
index 743d6c5..e20b9a3 100644
--- a/jenkins/ci.jpl
+++ b/jenkins/ci.jpl
@@ -6,7 +6,9 @@
//
//-------------------------------------------------------------------------------
-library identifier: 'local-lib@master', retriever: legacySCM(scm)
+@Library('trustedfirmware') _
+import org.trustedfirmware.Gerrit
+import org.trustedfirmware.Summary
def listConfigs(ci_scripts_dir, config_list, filter_group) {
dir(ci_scripts_dir) {
@@ -37,6 +39,9 @@
params += string(name: key, value: value)
}
params += string(name: 'GERRIT_BRANCH', value: env.GERRIT_BRANCH)
+ params += string(name: 'GERRIT_HOST', value: env.GERRIT_HOST)
+ params += string(name: 'GERRIT_CHANGE_NUMBER', value: env.GERRIT_CHANGE_NUMBER)
+ params += string(name: 'GERRIT_PATCHSET_REVISION', value: env.GERRIT_PATCHSET_REVISION)
params += string(name: 'GERRIT_REFSPEC', value: env.GERRIT_REFSPEC)
params += string(name: 'CMSIS_VERSION', value: env.CMSIS_VERSION)
params += string(name: 'MBEDCRYPTO_VERSION', value: env.MBEDCRYPTO_VERSION)
@@ -64,6 +69,9 @@
def buildDocs() {
def params = []
params += string(name: 'GERRIT_BRANCH', value: env.GERRIT_BRANCH)
+ params += string(name: 'GERRIT_HOST', value: env.GERRIT_HOST)
+ params += string(name: 'GERRIT_CHANGE_NUMBER', value: env.GERRIT_CHANGE_NUMBER)
+ params += string(name: 'GERRIT_PATCHSET_REVISION', value: env.GERRIT_PATCHSET_REVISION)
params += string(name: 'GERRIT_REFSPEC', value: env.GERRIT_REFSPEC)
params += string(name: 'CMSIS_VERSION', value: env.CMSIS_VERSION)
params += string(name: 'MBEDCRYPTO_VERSION', value: env.MBEDCRYPTO_VERSION)
@@ -79,6 +87,7 @@
def buildCsv(results) {
+ def summary = new Summary();
def csvContent = summary.getBuildCsv(results)
node("master") {
writeCSV file: 'build_results.csv', records: csvContent, format: CSVFormat.EXCEL
@@ -87,6 +96,7 @@
}
def writeSummary(results) {
+ def summary = new Summary();
def buildLinks = summary.getLinks(results)
node("master") {
writeFile file: "build_links.html", text: buildLinks
@@ -94,29 +104,6 @@
}
}
-def verifyStatus(value, stage_name) {
- node("docker-amd64-xenial") {
- cleanWs()
- dir("tf-m-ci-scripts") {
- git url: '$CI_SCRIPTS_REPO', branch: 'master', credentialsId: 'GIT_SSH_KEY'
- }
- withCredentials([usernamePassword(credentialsId: 'VERIFY_STATUS', passwordVariable: 'VERIFY_PASSWORD', usernameVariable: 'VERIFY_USER')]) {
- sh("""
- if [ -z "\$GERRIT_HOST" ] ; then
- echo Not running for a Gerrit change, skipping vote.
- exit 0
- fi
- if [ ! -d venv ] ; then
- virtualenv -p \$(which python3) venv
- fi
- . venv/bin/activate
- pip -q install requests
- ./tf-m-ci-scripts/jenkins/verify.py --value ${value} --verify-name tf-m-${stage_name} --user \$VERIFY_USER
- """)
- }
- }
-}
-
def configs = []
def builds = [:]
@@ -148,7 +135,8 @@
verify = -1
} finally {
print("Verifying status")
- verifyStatus(verify, 'build')
+ g = new Gerrit()
+ g.verifyStatus(verify, 'tf-m-build', 'build')
print("Building CSV")
buildCsv(results['builds'])
writeSummary(results['builds'])
diff --git a/jenkins/comment.py b/jenkins/comment.py
new file mode 100755
index 0000000..edd1aeb
--- /dev/null
+++ b/jenkins/comment.py
@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+"""
+Posts a comment to Gerrit.
+"""
+
+__copyright__ = """
+/*
+ * Copyright (c) 2020, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ */
+ """
+
+import argparse
+import json
+import os
+import sys
+import requests
+
+
+def submit_comment(base_url, auth, changeset, patchset_revision, comment):
+ post_data = {"message": comment}
+ comment_url = "{}/a/changes/{}/revisions/{}/review".format(
+ base_url, changeset, patchset_revision
+ )
+ headers = {"Content-Type": "application/json; charset=UTF-8"}
+ post = None
+ try:
+ post = requests.post(
+ comment_url, data=json.dumps(post_data), auth=auth, headers=headers,
+ )
+ except requests.exceptions.RequestException as exception:
+ print("Error posting comment to Gerrit.")
+ sys.exit(0)
+ if post.status_code == 200:
+ print("Posted comment to Gerrit successfully.")
+ else:
+ print(
+ "Could not post comment to Gerrit. Error: {} {}".format(
+ post.status_code, post.text
+ )
+ )
+
+
+if __name__ == "__main__":
+ PARSER = argparse.ArgumentParser(description="Submits a comment to a Gerrit change")
+ PARSER.add_argument("--host", help="Gerrit Host", default=os.getenv("GERRIT_HOST"))
+ PARSER.add_argument(
+ "--changeset",
+ help="Changeset in Gerrit to comment on.",
+ default=os.getenv("GERRIT_CHANGE_NUMBER"),
+ )
+ PARSER.add_argument(
+ "--patchset-revision",
+ help="Commit SHA of revision in Gerrit to comment on.",
+ default=os.getenv("GERRIT_PATCHSET_REVISION"),
+ )
+ PARSER.add_argument(
+ "--user", help="Username to authenticate as.", default=os.getenv("GERRIT_USER")
+ )
+ PARSER.add_argument(
+ "--password",
+ help="Password or token to authenticate as. "
+ "Defaults to GERRIT_PASSWORD environment variable.",
+ default=os.getenv("GERRIT_PASSWORD"),
+ )
+ PARSER.add_argument("--protocol", help="Protocol to use.", default="https")
+ PARSER.add_argument("--port", help="Port to use.", default=None)
+ PARSER.add_argument("--comment", help="Comment to send.")
+ ARGS = PARSER.parse_args()
+ submit_comment(
+ "{}://{}{}".format(
+ ARGS.protocol, ARGS.host, ":{}".format(ARGS.port) if ARGS.port else ""
+ ),
+ (ARGS.user, ARGS.password),
+ ARGS.changeset,
+ ARGS.patchset_revision,
+ ARGS.comment,
+ )
diff --git a/jenkins/cppcheck.jpl b/jenkins/cppcheck.jpl
index 6ce22b4..fb40a9f 100644
--- a/jenkins/cppcheck.jpl
+++ b/jenkins/cppcheck.jpl
@@ -6,22 +6,8 @@
//
//-------------------------------------------------------------------------------
-def verifyStatus(value, stage_name) {
- withCredentials([usernamePassword(credentialsId: 'VERIFY_STATUS', passwordVariable: 'VERIFY_PASSWORD', usernameVariable: 'VERIFY_USER')]) {
- sh """
-if [ -z "\$GERRIT_HOST" ] ; then
- echo Not running for a Gerrit change, skipping vote.
- exit 0
-fi
-if [ ! -d venv ] ; then
- virtualenv -p \$(which python3) venv
-fi
-. venv/bin/activate
-pip -q install requests
-./tf-m-ci-scripts/jenkins/verify.py --value ${value} --verify-name tf-m-${stage_name} --user \$VERIFY_USER
-"""
- }
-}
+@Library('trustedfirmware') _
+import org.trustedfirmware.Gerrit
node("docker-amd64-xenial") {
stage("Init") {
@@ -78,7 +64,8 @@
manager.buildFailure()
verify = -1
} finally {
- verifyStatus(verify, 'cppcheck')
+ g = new Gerrit()
+ g.verifyStatusInWorkspace(verify, 'cppcheck', 'static')
cleanWs()
}
}
diff --git a/jenkins/download_artifacts.py b/jenkins/download_artifacts.py
new file mode 100755
index 0000000..5e9111b
--- /dev/null
+++ b/jenkins/download_artifacts.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+#
+# Downloads artifacts from a build of tf-m-build-and-test
+#
+
+__copyright__ = """
+/*
+ * Copyright (c) 2020, Arm Limited. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ */
+ """
+
+import requests
+import argparse
+import os
+from urllib.parse import urljoin
+from html.parser import HTMLParser
+
+
+class UrlExtracter(HTMLParser):
+ def __init__(self):
+ super().__init__()
+ self.last_tag = None
+ self.last_link = None
+ self.last_config = None
+ self.build_artifacts = {}
+ self.build_logs = {}
+
+ def handle_starttag(self, tag, attrs):
+ for key, value in attrs:
+ if key == "href":
+ self.last_link = value
+ self.last_tag = tag
+
+ def handle_endtag(self, tag):
+ if tag == "br":
+ self.last_tag = None
+
+ def handle_data(self, data):
+ if not self.last_tag:
+ self.last_config = data.replace(": ", "").replace("\n", "")
+ return
+
+ if self.last_tag == "a":
+ if data == "Artifacts":
+ self.build_artifacts[self.last_config] = self.last_link
+ elif data == "Logs":
+ self.build_logs[self.last_config] = self.last_link
+
+
+def download_artifacts(url, save_dir):
+ if not url.endswith("/"):
+ url += "/"
+ job_page_req = requests.get(url)
+ if job_page_req.status_code != requests.codes.ok:
+ print("Issue contacting given URL")
+ return
+ print("Found build")
+ build_links_req = requests.get(urljoin(url, "artifact/build_links.html"))
+ if build_links_req.status_code != requests.codes.ok:
+ print("Given build did not have an artifact called `build_links.html`")
+ return
+ parser = UrlExtracter()
+ print("Extracting links from build_links.html")
+ parser.feed(build_links_req.text)
+ print("Links found")
+ if not os.path.exists(save_dir):
+ print("Creating directory at {}".format(save_dir))
+ os.makedirs(save_dir)
+ else:
+ print("Reusing directory at {}.")
+ for config, log_url in parser.build_logs.items():
+ print("Downloading {}".format(log_url))
+ log_req = requests.get(log_url)
+ log_file_path = os.path.join(save_dir, "{}.log".format(config))
+ with open(log_file_path, "w") as log_file:
+ log_file.write(log_req.text)
+ print("Saved log to {}".format(log_file_path))
+ for config, artifacts_url in parser.build_artifacts.items():
+ zip_url = urljoin(artifacts_url, "*zip*/archive.zip")
+ print("Downloading {}".format(zip_url))
+ artifact_zip_req = requests.get(zip_url, stream=True)
+ zip_file = os.path.join(save_dir, "{}.zip".format(config))
+ with open(zip_file, "wb") as artifact_zip:
+ for chunk in artifact_zip_req.iter_content(chunk_size=8192):
+ artifact_zip.write(chunk)
+ print("Saved artifacts zip to {}".format(zip_file))
+ print("Finished")
+
+
+def main():
+ argparser = argparse.ArgumentParser()
+ argparser.add_argument(
+ "job_url", help="Url to a completed build of tf-m-build-and-test"
+ )
+ argparser.add_argument(
+ "-o", "--output_dir", default="artifacts", help="Location to save artifacts to."
+ )
+ args = argparser.parse_args()
+ download_artifacts(args.job_url, args.output_dir)
+
+
+if __name__ == "__main__":
+ main()