Initial commit for TF-A CI scripts
Signed-off-by: Fathi Boudra <fathi.boudra@linaro.org>
diff --git a/job/README b/job/README
new file mode 100644
index 0000000..216ac2a
--- /dev/null
+++ b/job/README
@@ -0,0 +1,8 @@
+#
+# Copyright (c) 2019, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+This directory contains snippets that are to be run from Jenkins jobs
+
diff --git a/job/clean_filer_ws.sh b/job/clean_filer_ws.sh
new file mode 100755
index 0000000..2056cb6
--- /dev/null
+++ b/job/clean_filer_ws.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# Copyright (c) 2019, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+set -e
+
+# Clean up filer work space if we had cloned the repository ourselves
+
+# On Jenkins v2, $SCRATCH_OWNER will be set to the job name and build number. If
+# that matches with that of the current job, then remove the scratch space.
+if [ "$SCRATCH_OWNER" = "${JOB_NAME:?}-${BUILD_NUMBER:?}" ]; then
+ rm -rf "${SCRATCH_OWNER_SPACE?:}"
+ exit 0
+fi
+
+# On Jenkins v1, $FILER_WS will be unset if we don't need to clean up
+if [ -z "$FILER_WS" ]; then
+ exit 0
+fi
+
+rm -rf "$FILER_WS"
diff --git a/job/tf-a-tests-master-sync/sync.sh b/job/tf-a-tests-master-sync/sync.sh
new file mode 100755
index 0000000..59ac052
--- /dev/null
+++ b/job/tf-a-tests-master-sync/sync.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# Copyright (c) 2019, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+set -e
+
+source "$CI_ROOT/utils.sh"
+
+# Clone TF-A Tests repo from tf.org.
+if [ ! -d "tf-a-tests" ]; then
+ git clone --origin tforg $tftf_src_repo_url
+ cd tf-a-tests
+ git remote add arm $tftf_arm_gerrit_repo
+else
+ cd tf-a-tests
+fi
+
+# Get the latest updates from the master branch on tf.org.
+git remote update --prune
+git checkout master
+git merge --ff-only tforg/master
+
+# Push updates to Arm internal Gerrit.
+git push arm master
diff --git a/job/tf-build-for-lava/build_test.sh b/job/tf-build-for-lava/build_test.sh
new file mode 100755
index 0000000..c16128e
--- /dev/null
+++ b/job/tf-build-for-lava/build_test.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+#
+# Copyright (c) 2019, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+set -e
+
+"$CI_ROOT/script/build_package.sh"
diff --git a/job/tf-ci-gateway/generate_report.sh b/job/tf-ci-gateway/generate_report.sh
new file mode 100755
index 0000000..c0f9a7a
--- /dev/null
+++ b/job/tf-ci-gateway/generate_report.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# Copyright (c) 2019, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+set -e
+
+# Generate test report
+if [ "$CI_ROOT" ]; then
+ # Gather Coverity scan summary if it was performed as part of this job
+ if [ "$(find -maxdepth 1 -name '*coverity*.test' -type f | wc -l)" != 0 ]; then
+ if ! "$CI_ROOT/script/coverity_summary.py" "$BUILD_URL" > coverity.data; then
+ rm -f coverity.data
+ fi
+ fi
+
+ "$CI_ROOT/script/gen_test_report.py" \
+ --job "${worker_job:-tf-worker}" \
+ --build-job "${lava_job:-tf-build-for-lava}" \
+ --meta-data clone.data \
+ --meta-data override.data \
+ --meta-data inject.data \
+ --meta-data html:coverity.data \
+ || true
+fi
diff --git a/job/tf-ci-gateway/generate_tests.sh b/job/tf-ci-gateway/generate_tests.sh
new file mode 100755
index 0000000..cc71b0b
--- /dev/null
+++ b/job/tf-ci-gateway/generate_tests.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+#
+# Copyright (c) 2019, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+set -e
+
+# Generate test descriptions
+"$CI_ROOT/script/gen_test_desc.py"
diff --git a/job/tf-github-pr/should_run_nominations.sh b/job/tf-github-pr/should_run_nominations.sh
new file mode 100755
index 0000000..c2caab1
--- /dev/null
+++ b/job/tf-github-pr/should_run_nominations.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Copyright (c) 2019, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+nom_file="$WORKSPACE/nominations"
+rules_file="$CI_ROOT/script/trusted-firmware.nomination.py"
+if [ -f "$rules_file" ]; then
+ cd "$TF_CHECKOUT_LOC"
+ "$CI_ROOT/script/gen_nomination.py" "$rules_file" > "$nom_file"
+ if [ -s "$nom_file" ]; then
+ exit 0
+ else
+ # No nominations
+ exit 1
+ fi
+fi
+
+exit 1
diff --git a/job/tf-main/should_execute_static.sh b/job/tf-main/should_execute_static.sh
new file mode 100755
index 0000000..fccde1c
--- /dev/null
+++ b/job/tf-main/should_execute_static.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+#
+# Copyright (c) 2019, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+set -e
+
+if [ "$SKIP_STATIC" = "true" ]; then
+ exit 1
+else
+ exit 0
+fi
diff --git a/job/tf-merge-watcher/check_scratch.sh b/job/tf-merge-watcher/check_scratch.sh
new file mode 100755
index 0000000..173a742
--- /dev/null
+++ b/job/tf-merge-watcher/check_scratch.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# Copyright (c) 2019, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+ci_root="$(readlink -f "$(dirname "$0")/../..")"
+source "$ci_root/utils.sh"
+
+if ! diff $project_filer/ci-scripts/ $ci_root/script/scratch_scripts/
+then
+ echo "scripts in scratch folder don't match scripts in repository" >&2
+ exit 1
+fi
diff --git a/job/tf-merge-watcher/update_reference_repo.sh b/job/tf-merge-watcher/update_reference_repo.sh
new file mode 100755
index 0000000..65fff79
--- /dev/null
+++ b/job/tf-merge-watcher/update_reference_repo.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+# Copyright (c) 2019, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+set -e
+
+ci_root="$(readlink -f "$(dirname "$0")/../..")"
+source "$ci_root/utils.sh"
+
+declare -A repo_urls=(
+[pdcs-platforms/ap/tf-topics]="name=trusted-firmware url=$tf_src_repo_url"
+[trusted-firmware/tf-a-tests]="name=trusted-firmware-tf url=$tftf_src_repo_url"
+[pdswinf/ci/pdcs-platforms/platform-ci]="name=trusted-firmware-ci url=$tf_ci_repo_url"
+)
+
+project="${GERRIT_PROJECT:-$PROJECT}"
+eval "${repo_urls[$project]?}"
+ref_dir="$project_filer/ref-repos/$name"
+
+# Create/update reference repository.
+mkdir -p "$ref_dir"
+if [ ! -d "$ref_dir" ]; then
+ # Clone afresh
+ mkdir -p "$ref_dir"
+ git clone -q "$url" "$ref_dir"
+else
+ # Update master
+ cd "$ref_dir"
+ git fetch -q origin master
+ git reset -q --hard origin/master
+fi
+
+echo "Updated $name"
diff --git a/job/tf-optee-build/build_optee.sh b/job/tf-optee-build/build_optee.sh
new file mode 100755
index 0000000..4bf24c1
--- /dev/null
+++ b/job/tf-optee-build/build_optee.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+# Copyright (c) 2019, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+ci_root="$(readlink -f "$(dirname "$0")/../..")"
+source "$ci_root/utils.sh"
+
+cd optee
+make PLATFORM=vexpress PLATFORM_FLAVOR="${PLATFORM_FLAVOR:?}" CFG_ARM64_core=y
+
+# Remove header from tee.bin
+aarch64-linux-gnu-objcopy -O binary \
+ out/arm-plat-vexpress/core/tee.elf out/arm-plat-vexpress/core/tee.bin
+
+# Gather files to export in a single directory
+mkdir -p "$workspace/artefacts"
+cp out/arm-plat-vexpress/core/tee.bin "$workspace/artefacts"
diff --git a/job/tf-topics-master-sync/sync.sh b/job/tf-topics-master-sync/sync.sh
new file mode 100755
index 0000000..d2c741b
--- /dev/null
+++ b/job/tf-topics-master-sync/sync.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+#
+# Copyright (c) 2019, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+# $1 = git remote human readable name
+# $2 = git remote URL
+sync_repo()
+{
+ local result
+ echo Pushing to $1...
+ git push $2 master
+ result=$?
+ if [ $result != 0 ]
+ then
+ echo Pushing to $1 FAILED!
+ else
+ echo Pushing to $1 SUCCEEDED!
+ fi
+ return $result
+}
+
+# exit if anything fails
+set -e
+
+source "$CI_ROOT/utils.sh"
+
+if [ ! -d "trusted-firmware-a" ]
+then
+ # Fresh clone
+ echo Cloning from trustedfirmware.org...
+ git clone $tf_src_repo_url
+else
+ echo Using existing repo...
+fi
+
+echo Pulling from trustedfirmware.org...
+cd trusted-firmware-a
+git remote update --prune
+git checkout master
+git merge --ff-only origin/master
+
+# stop exiting automatically
+set +e
+
+sync_repo GitHub https://$GH_USER:$GH_PASSWORD@github.com/ARM-software/arm-trusted-firmware.git
+github=$?
+sync_repo "internal Arm Gerrit" $tf_arm_gerrit_repo
+gerrit=$?
+
+if [ $github != 0 -o $gerrit != 0 ]
+then
+ exit 1
+fi
diff --git a/job/tf-worker/generate_yaml.sh b/job/tf-worker/generate_yaml.sh
new file mode 100755
index 0000000..37a0ae0
--- /dev/null
+++ b/job/tf-worker/generate_yaml.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+# Copyright (c) 2019, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+set -e
+
+if echo "$RUN_CONFIG" | grep -iq 'tftf'; then
+ payload_type="tftf"
+else
+ payload_type="linux"
+fi
+
+"$CI_ROOT/script/parse_lava_job.py" --payload-type "$payload_type"
diff --git a/job/tf-worker/is_juno_config.sh b/job/tf-worker/is_juno_config.sh
new file mode 100755
index 0000000..31affc9
--- /dev/null
+++ b/job/tf-worker/is_juno_config.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Copyright (c) 2019, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+set -e
+
+# If we're skipping LAVA or Juno
+if [ "$skip_juno" ] || [ "$skip_runs" ]; then
+ exit 1
+fi
+
+# For Juno runs, we need let the board download build artefacts using a URL. The
+# only way to have a board-accessible URL at the moment is to have build
+# artefacts archived. Therefore, only for Juno do we spawn the build as a
+# separate job; otherwise, we build within this job.
+if echo "$RUN_CONFIG" | grep -iq '^juno'; then
+ exit 0
+else
+ exit 1
+fi
diff --git a/job/tf-worker/manage_artefacts.sh b/job/tf-worker/manage_artefacts.sh
new file mode 100755
index 0000000..865afd9
--- /dev/null
+++ b/job/tf-worker/manage_artefacts.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+# Copyright (c) 2019, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+set -e
+
+if [ -d artefacts ]; then
+ # Remove everything except logs
+ find artefacts -type f -not \( -name "*.log" \) -exec rm -f {} +
+fi
diff --git a/job/tf-worker/parse_test.sh b/job/tf-worker/parse_test.sh
new file mode 100755
index 0000000..fead3a7
--- /dev/null
+++ b/job/tf-worker/parse_test.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+#
+# Copyright (c) 2019, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+set -e
+
+# Parse test config. This produces $workspace/env file
+$CI_ROOT/script/parse_test.sh
diff --git a/job/tf-worker/run_fvp_test.sh b/job/tf-worker/run_fvp_test.sh
new file mode 100755
index 0000000..2a62eab
--- /dev/null
+++ b/job/tf-worker/run_fvp_test.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+# Copyright (c) 2019, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+set -e
+
+# Build
+"$CI_ROOT/script/build_package.sh"
+
+if [ "$skip_runs" ]; then
+ exit 0
+fi
+
+# Execute test locally for FVP configs
+if [ "$RUN_CONFIG" != "nil" ] && echo "$RUN_CONFIG" | grep -iq '^fvp'; then
+ "$CI_ROOT/script/run_package.sh"
+fi
diff --git a/job/tf-worker/should_build_local.sh b/job/tf-worker/should_build_local.sh
new file mode 100755
index 0000000..5b47866
--- /dev/null
+++ b/job/tf-worker/should_build_local.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# Copyright (c) 2019, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+set -e
+
+# If it's a Juno build-only config, or an FVP config, we do everything locally
+if [ "$RUN_CONFIG" = "nil" ]; then
+ exit 0
+fi
+
+case "$RUN_CONFIG" in
+ fvp-*)
+ exit 0;;
+ coverity-*)
+ exit 0;;
+esac
+
+# If we're not going to run Juno, then no need to spawn tf-build-for lava;
+# build it locally.
+if [ "$skip_juno" ]; then
+ exit 0
+fi
+
+exit 1
diff --git a/job/tf-worker/submit_lava_job.sh b/job/tf-worker/submit_lava_job.sh
new file mode 100755
index 0000000..7b47e97
--- /dev/null
+++ b/job/tf-worker/submit_lava_job.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+#
+# Copyright (c) 2019, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+# Submit jobs to LAVA and wait until the job is complete. This script replace
+# the "managed script" previously used and provide the same behavior.
+#
+# Required arguments:
+# 1: yaml job file
+# 2: flag whether to save output, true/false, defaults to false
+#
+# output:
+# job_results.yaml
+# job_output.log if save output = true
+
+set -e
+
+JOB_FILE="$1"
+SAVE_OUTPUT="$2"
+
+LAVA_HOST=
+LAVA_USER=
+LAVA_TOKEN=
+LAVA_URL=
+
+if [ ! -f "${JOB_FILE}" ]; then
+ echo "error: LAVA job file does not exist: ${JOB_FILE}"
+ exit 1
+fi
+
+# Install lavacli with fixes
+virtualenv -p $(which python3) venv
+source venv/bin/activate
+pip install -q lavacli
+
+# Configure lavacli
+lavacli identities add \
+--username $LAVA_USER \
+--token $LAVA_TOKEN \
+--uri ${LAVA_URL}/RPC2 \
+default
+
+# Submit a job using lavacli
+JOB_ID=$(lavacli jobs submit ${JOB_FILE})
+if [ -z "$JOB_ID" ] ; then
+ echo "Couldn't submit. Stopping."
+ exit 1
+fi
+
+echo "Job url: https://lava.oss.arm.com/scheduler/job/$JOB_ID"
+
+# Wait for the job to finish
+lavacli jobs wait $JOB_ID
+
+if [ "${SAVE_OUTPUT}" = "true" ] ; then
+ lavacli jobs logs $JOB_ID > job_output.log
+fi
+
+# Get results
+lavacli results $JOB_ID --yaml > job_results.yaml
+
+# Exit virtualenv
+deactivate
diff --git a/job/trusted-firmware-main/should_execute_static.sh b/job/trusted-firmware-main/should_execute_static.sh
new file mode 100755
index 0000000..fccde1c
--- /dev/null
+++ b/job/trusted-firmware-main/should_execute_static.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+#
+# Copyright (c) 2019, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+set -e
+
+if [ "$SKIP_STATIC" = "true" ]; then
+ exit 1
+else
+ exit 0
+fi