JJB: Add Jenkins job builder script
Signed-off-by: Arthur She <arthur.she@linaro.org>
Change-Id: Ic9a1d58dcd6e9cb72aa91d31e16ec2c42b6b54da
diff --git a/ci-job-configs-sanity-check.yaml b/ci-job-configs-sanity-check.yaml
new file mode 100644
index 0000000..1b61f6a
--- /dev/null
+++ b/ci-job-configs-sanity-check.yaml
@@ -0,0 +1,76 @@
+- job:
+ name: ci-job-configs-sanity-check
+ project-type: freestyle
+ defaults: global
+ properties:
+ - authorization:
+ authenticated:
+ - job-read
+ - job-extended-read
+ - job-discover
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 30
+ disabled: false
+ node: master
+ display-name: 'CI job configs sanity check'
+ scm:
+ - git:
+ url: https://review.trustedfirmware.org/${GERRIT_PROJECT}
+ refspec: ${GERRIT_REFSPEC}
+ branches:
+ - ${GERRIT_BRANCH}
+ skip-tag: true
+ clean:
+ before: true
+ choosing-strategy: gerrit
+ basedir: configs
+ triggers:
+ - gerrit:
+ server-name: 'review.trustedfirmware.org'
+ trigger-on:
+ - patchset-created-event
+ projects:
+ - project-compare-type: 'PLAIN'
+ project-pattern: 'mbedtls/tf-ci-scripts'
+ branches:
+ - branch-pattern: 'master'
+ - project-compare-type: 'PLAIN'
+ project-pattern: 'mbedtls/mbed-tls-job-configs'
+ branches:
+ - branch-pattern: 'master'
+ silent-start: true
+ wrappers:
+ - timestamps
+ builders:
+ - shell: |
+ #!/bin/bash -e
+ echo "#${BUILD_NUMBER}-${GERRIT_PATCHSET_REVISION:0:8}" > ${WORKSPACE}/version.txt
+ - build-name-setter:
+ name: 'version.txt'
+ file: true
+ - shell: |
+ #!/bin/bash
+
+ set -e
+
+ echo ""
+ echo "########################################################################"
+ echo " Gerrit Environment"
+ env |grep '^GERRIT'
+ echo "########################################################################"
+
+ cd configs/
+
+ mkdir -p ci/ && wget -q https://git.trustedfirmware.org/mbedtls/tf-ci-scripts.git/plain/ci/run-jjb.py -O ci/run-jjb.py
+ # FIXME run-jjb.py was meant to be used for deployment only
+ # use JJB 'test' command instead of 'update' command
+ sed -i "s|update|test|" ci/run-jjb.py
+
+ export GIT_PREVIOUS_COMMIT=$(git rev-parse HEAD~1)
+ export GIT_COMMIT=${GERRIT_PATCHSET_REVISION}
+ jenkins-jobs --version
+ python3 ci/run-jjb.py
+ publishers:
+ - email:
+ recipients: 'ben.copeland@linaro.org kelley.spoon@linaro.org arthur.she@linaro.org'
diff --git a/ci/run-jjb.py b/ci/run-jjb.py
new file mode 100755
index 0000000..731ab18
--- /dev/null
+++ b/ci/run-jjb.py
@@ -0,0 +1,183 @@
+#!/usr/bin/python3
+
+import os
+import shutil
+import signal
+import string
+import subprocess
+import sys
+import xml.etree.ElementTree
+from distutils.spawn import find_executable
+
+
+def findparentfiles(fname):
+ filelist = []
+ newlist = []
+ args = ['grep', '-rl', '--exclude-dir=.git', fname]
+ proc = subprocess.run(args, capture_output=True)
+ data = proc.stdout.decode()
+ if proc.returncode != 0:
+ return filelist
+ for filename in data.splitlines():
+ if filename.endswith('.yaml') and '/' not in filename:
+ filelist.append(filename)
+ else:
+ newlist = findparentfiles(filename)
+ for tempname in newlist:
+ filelist.append(tempname)
+ return filelist
+
+
+jjb_cmd = find_executable('jenkins-jobs') or sys.exit('jenkins-jobs is not found.')
+jjb_args = [jjb_cmd]
+
+jjb_user = os.environ.get('JJB_USER')
+jjb_password = os.environ.get('JJB_PASSWORD')
+if jjb_user is not None and jjb_password is not None:
+ jenkins_jobs_ini = ('[job_builder]\n'
+ 'ignore_cache=True\n'
+ 'keep_descriptions=False\n'
+ '\n'
+ '[jenkins]\n'
+ 'user=%s\n'
+ 'password=%s\n'
+ 'url=https://ci.trustedfirmware.org/\n' % (jjb_user, jjb_password))
+ with open('jenkins_jobs.ini', 'w') as f:
+ f.write(jenkins_jobs_ini)
+ jjb_args.append('--conf=jenkins_jobs.ini')
+
+jjb_test_args = list(jjb_args)
+jjb_delete_args = list(jjb_args)
+
+# !!! "update" below and through out this file is replaced by "test" (using sed)
+# !!! in the sanity-check job.
+main_action = 'update'
+jjb_args.extend([main_action, 'template.yaml'])
+jjb_test_args.extend(['test', '-o', 'out/', 'template.yaml'])
+jjb_delete_args.extend(['delete'])
+
+if main_action == 'test':
+ # Dry-run, don't delete jobs.
+ jjb_delete_args.insert(0, 'echo')
+
+try:
+ git_args = ['git', 'diff', '--raw',
+ os.environ.get('GIT_PREVIOUS_COMMIT'),
+ os.environ.get('GIT_COMMIT')]
+ proc = subprocess.run(git_args, capture_output=True)
+except (OSError, ValueError) as e:
+ raise ValueError("%s" % e)
+
+data = proc.stdout.decode()
+if proc.returncode != 0:
+ raise ValueError("command has failed with code '%s'" % proc.returncode)
+
+filelist = []
+deletelist = []
+files = []
+for line in data.splitlines():
+ # Format of the git-diff; we only need OPERATION and FILE1
+ #
+ # :<OLD MODE> <NEW MODE> <OLD REF> <NEW REF> <OPERATION> <FILE1> <FILE2>
+ elems = line.split()
+ operation = elems[4][0]
+ filename = elems[5]
+
+ if filename.endswith('.yaml') and '/' not in filename:
+ # No point trying to test deleted jobs because they don't exist any
+ # more.
+ if operation == 'D':
+ deletelist.append(filename[:-5])
+ continue
+ # operation R100 is 100% rename, which means sixth element is the renamed file
+ if operation == 'R':
+ filename = elems[6]
+ # delete old job name
+ deletelist.append(elems[5][:-5])
+ filelist.append(filename)
+ else:
+ files = findparentfiles(filename)
+ for tempname in files:
+ filelist.append(tempname)
+
+# Remove duplicate entries in the list
+filelist = list(set(filelist))
+
+for conf_filename in filelist:
+ with open(conf_filename) as f:
+ buffer = f.read()
+ template = string.Template(buffer)
+ buffer = template.safe_substitute(
+ AUTH_TOKEN=os.environ.get('AUTH_TOKEN'),
+ LT_QCOM_KEY=os.environ.get('LT_QCOM_KEY'),
+ LAVA_USER=os.environ.get('LAVA_USER'),
+ LAVA_TOKEN=os.environ.get('LAVA_TOKEN'))
+ with open('template.yaml', 'w') as f:
+ f.write(buffer)
+ try:
+ proc = subprocess.run(jjb_args, capture_output=True)
+ except (OSError, ValueError) as e:
+ raise ValueError("%s" % e)
+
+ data = proc.stdout.decode()
+ if proc.returncode != 0:
+ raise ValueError("command has failed with code '%s'" % proc.returncode)
+
+ try:
+ shutil.rmtree('out/', ignore_errors=True)
+
+ proc = subprocess.run(jjb_test_args, capture_output=True)
+ data = proc.stdout.decode()
+ if proc.returncode != 0:
+ raise ValueError("command has failed with code '%s'" % proc.returncode)
+
+ proc = subprocess.run(['ls', 'out/'], capture_output=True)
+ data = proc.stdout.decode()
+ if proc.returncode != 0:
+ raise ValueError("command has failed with code '%s'" % proc.returncode)
+
+ for filename in data.splitlines():
+ # old job conf might have been removed because the job is now generated through the template
+ # do not delete the job in this case
+ if filename in deletelist:
+ deletelist.remove(filename)
+
+ conf_name=os.path.splitext(conf_filename)[0]
+ conf_name=conf_name[:len(filename)]
+ if not filename.startswith(conf_name):
+ raise ValueError("Job name %s does not match the file it is in: %s" % (filename, conf_name))
+ try:
+ xmlroot = xml.etree.ElementTree.parse('out/' + filename).getroot()
+ disabled = next(xmlroot.iterfind('disabled')).text
+ if disabled != 'true':
+ continue
+ displayName = next(xmlroot.iterfind('displayName')).text
+ if displayName != 'DELETE ME':
+ continue
+ except:
+ continue
+
+ deletelist.append(filename)
+
+ except (OSError, ValueError) as e:
+ raise ValueError("%s" % e)
+
+ shutil.rmtree('out/', ignore_errors=True)
+ os.remove('template.yaml')
+
+
+for deletejob in deletelist:
+ delete_args = list(jjb_delete_args)
+ delete_args.extend([deletejob])
+ try:
+ proc = subprocess.run(delete_args, capture_output=True)
+ data = proc.stdout.decode()
+ if proc.returncode != 0:
+ raise ValueError("command has failed with code '%s'" % proc.returncode)
+ print(data)
+ except (OSError, ValueError) as e:
+ raise ValueError("%s" % e)
+
+if os.path.exists('jenkins_jobs.ini'):
+ os.remove('jenkins_jobs.ini')
+
diff --git a/trigger-ci-job-configs.yaml b/trigger-ci-job-configs.yaml
new file mode 100644
index 0000000..ad2e02e
--- /dev/null
+++ b/trigger-ci-job-configs.yaml
@@ -0,0 +1,86 @@
+- job:
+ name: trigger-ci-job-configs
+ project-type: freestyle
+ defaults: global
+ properties:
+ - authorization:
+ anonymous:
+ - job-discover
+ authenticated:
+ - job-discover
+ - job-read
+ - job-extended-read
+ - build-discarder:
+ days-to-keep: 30
+ num-to-keep: 200
+ disabled: false
+ node: master
+ display-name: 'CI job configs deployment'
+ scm:
+ - git:
+ url: https://review.trustedfirmware.org/${GERRIT_PROJECT}
+ refspec: ${GERRIT_REFSPEC}
+ branches:
+ - ${GERRIT_BRANCH}
+ skip-tag: true
+ clean:
+ before: true
+ choosing-strategy: gerrit
+ basedir: configs
+ triggers:
+ - gerrit:
+ server-name: 'review.trustedfirmware.org'
+ trigger-on:
+ - change-merged-event
+ projects:
+ - project-compare-type: 'PLAIN'
+ project-pattern: 'mbedtls/tf-ci-scripts'
+ branches:
+ - branch-pattern: 'master'
+ - project-compare-type: 'PLAIN'
+ project-pattern: 'mbedtls/mbed-tls-job-configs'
+ branches:
+ - branch-pattern: 'master'
+ silent-start: true
+ wrappers:
+ - timestamps
+ - credentials-binding:
+ - text:
+ credential-id: JJB_USER
+ variable: JJB_USER
+ - credentials-binding:
+ - text:
+ credential-id: JJB_PASSWORD
+ variable: JJB_PASSWORD
+ - credentials-binding:
+ - text:
+ credential-id: AUTH_TOKEN
+ variable: AUTH_TOKEN
+ builders:
+ - shell: |
+ #!/bin/bash -e
+ echo "#${BUILD_NUMBER}-${GERRIT_PATCHSET_REVISION:0:8}" > ${WORKSPACE}/version.txt
+ - build-name-setter:
+ name: 'version.txt'
+ file: true
+ - shell: |
+ #!/bin/bash
+
+ set -e
+
+ echo ""
+ echo "########################################################################"
+ echo " Gerrit Environment"
+ env |grep '^GERRIT'
+ echo "########################################################################"
+
+ cd configs/
+
+ export GIT_PREVIOUS_COMMIT=$(git rev-parse HEAD~1)
+ export GIT_COMMIT=${GERRIT_PATCHSET_REVISION}
+ jenkins-jobs --version
+ mkdir -p ci/ && wget -q https://git.trustedfirmware.org/mbedtls/tf-ci-scripts.git/plain/ci/run-jjb.py -O ci/run-jjb.py
+ python3 ci/run-jjb.py
+ publishers:
+ - email:
+ recipients: 'ben.copeland@linaro.org kelley.spoon@linaro.org arthur.she@linaro.org'