blob: 53b94bb433db1b185f1d8bbc79e86a91f51565aa [file] [log] [blame]
Fathi Boudra422bf772019-12-02 11:10:16 +02001#!/bin/bash
2#
3# Copyright (c) 2019, Arm Limited. All rights reserved.
4#
5# SPDX-License-Identifier: BSD-3-Clause
6#
7#
8# Clone and sync all Trusted Firmware repositories.
9#
10# The main repository is checked out at the required refspec (GERRIT_REFSPEC).
11# The rest of repositories are attempted to sync to the topic of that refspec
12# (as pointed to by GERRIT_TOPIC). 'repo_under_test' must be set to a
13# GERRIT_PROJECT for sync to work.
14#
15# For every cloned repository, set its location to a variable so that the
16# checked out location can be passed down to sub-jobs.
17#
18# Generate an environment file that can then be sourced by the caller.
19
20set -e
21
22ci_root="$(readlink -f "$(dirname "$0")/..")"
23source "$ci_root/utils.sh"
24
25clone_log="$workspace/clone_repos.log"
26clone_data="$workspace/clone.data"
27override_data="$workspace/override.data"
28gerrit_data="$workspace/gerrit.data"
29inject_data="$workspace/inject.data"
30
31# File containing parameters for sub jobs
32param_file="$workspace/env.param"
33
34# Emit a parameter to sub jobs
35emit_param() {
36 echo "$1=$2" >> "$param_file"
37}
38
39meta_data() {
40 echo "$1" >> "$clone_data"
41}
42
43# Path into the project filer where various pieces of scripts that override
44# some CI environment variables are stored.
45ci_overrides="$project_filer/ci-overrides"
46
47display_override() {
48 echo
49 echo -n "Override: "
50 # Print the relative path of the override file.
51 echo "$1" | sed "s#$ci_overrides/\?##"
52}
53
54strip_var() {
55 local var="$1"
56 local val="$(echo "${!var}" | sed 's#^\s*\|\s*$##g')"
57 eval "$var=\"$val\""
58}
59
60prefix_tab() {
61 sed 's/^/\t/g' < "${1:?}"
62}
63
64prefix_arrow() {
65 sed 's/^/ > /g' < "${1:?}"
66}
67
68test_source() {
69 local file="${1:?}"
70 if ! bash -c "source $file" &>/dev/null; then
71 return 1
72 fi
73
74 source "$file"
75 return 0
76}
77
78post_gerrit_comment() {
79 local gerrit_url="${gerrit_url:-$GERRIT_HOST}"
80 gerrit_url="${gerrit_url:?}"
81
82 # Posting comments to gerrit.oss.arm.com does not require any special
83 # credentials, review.trustedfirmware.org does. Provide the ci-bot-user
84 # account credentials for the latter.
85 if [ "$gerrit_url" == "review.trustedfirmware.org" ]; then
86 ssh -p 29418 -i "$tforg_key" "$tforg_user@$gerrit_url" gerrit \
87 review "$GERRIT_CHANGE_NUMBER,$GERRIT_PATCHSET_NUMBER" \
88 --message "'$(cat ${msg_file:?})'"
89 else
90 ssh -p 29418 "$gerrit_url" gerrit review \
91 "$GERRIT_CHANGE_NUMBER,$GERRIT_PATCHSET_NUMBER" \
92 --message "'$(cat ${msg_file:?})'"
93 fi
94}
95
96# Whether we've synchronized branches or not
97has_synched=0
98
99# Whether we've overridden some CI environment variables.
100has_overrides=0
101
102# Whether we've injected environment via. Jenkins
103has_env_inject=0
104
105# Default Gerrit failure message file
106gerrit_fail_msg_file="$workspace/gerrit-fail"
107
108clone_and_sync() {
109 local stat
110 local topic
111 local refspec="${!ref}"
112 local s_before s_after s_diff
113 local reference_dir="$project_filer/ref-repos/${name?}"
114 local ref_repo
115 local ret
116 local gerrit_server
117 local gerrit_user
118 local gerrit_keyfile
119
120 strip_var refspec
121 strip_var url
122
123 case "$url" in
124 *${arm_gerrit_url}*)
125 gerrit_server="arm"
126 ;;
127
128 *${tforg_gerrit_url}*)
129 # SSH authentication is required on trustedfirmware.org.
130 gerrit_server="tforg"
131 gerrit_user="$tforg_user"
132 gerrit_keyfile="$tforg_key"
133 ;;
134
135 *)
136 # The project to clone might not be hosted on a Gerrit
137 # server at all (e.g. Github).
138 ;;
139 esac
140
141 # Refspec translation is supported for Gerrit patches only.
142 if [ "$gerrit_server" ]; then
143 refspec="$($ci_root/script/translate_refspec.py \
144 -p "$name" -s "$gerrit_server" -u "$gerrit_user" \
145 -k "$gerrit_keyfile" "$refspec")"
146 fi
147
148 # Clone in the filter workspace
149 mkdir -p "$ci_scratch"
150 pushd "$ci_scratch"
151
152 # Seconds before
153 s_before="$(date +%s)"
154
155 # Clone repository to the directory same as its name; HEAD stays at
156 # master.
157 if [ -d "$reference_dir" ]; then
158 ref_repo="--reference $reference_dir"
159 fi
160 git clone -q $ref_repo "$url" "$name" &>"$clone_log"
161 stat="on branch master"
162
163 pushd "$name"
164
165 if [ "$refspec" ] && [ "$refspec" != "master" ]; then
166 # If a specific revision is specified, always use that.
167 git fetch -q origin "$refspec" &>"$clone_log"
168 git checkout -q FETCH_HEAD &>"$clone_log"
169 stat="refspec $refspec"
170
171 # If it's not a commit hash, have the refspec replicated on the
172 # clone so that downstream jobs can clone from this one using
173 # the same refspec.
174 if echo "$refspec" | grep -qv '^[a-f0-9]\+$'; then
175 git branch "$refspec" FETCH_HEAD
176 fi
177 elif [ "$name" = "$repo_under_test" ]; then
178 # Main repository under test
179 if [ "$GERRIT_REFSPEC" ]; then
180 # Fetch and checkout GERRIT_REFSPEC
181 git fetch -q origin "$GERRIT_REFSPEC" \
182 &>"$clone_log"
183 git checkout -q FETCH_HEAD &>"$clone_log"
184 refspec="$GERRIT_REFSPEC"
185 stat="refspec $refspec"
186 git branch "$refspec" FETCH_HEAD
187 fi
188 elif [ "$GERRIT_TOPIC" ]; then
189 # Auxiliary repository: it's already on master when cloned above.
190 topic="$GERRIT_TOPIC"
191
192 # Check first if there's a Gerrit topic matching the topic of
193 # the main repository under test
194 ret=0
195 refspec="$("$ci_root/script/translate_refspec.py" -p "$name" \
196 -u "$gerrit_user" -k "$gerrit_keyfile" \
197 -s "$gerrit_server" "topic:$topic" 2>/dev/null)" \
198 || ret="$?"
199 if [ "$ret" = 0 ]; then
200 {
201 git fetch -q origin "$refspec"
202 git checkout -q FETCH_HEAD
203 } &>"$clone_log"
204 stat="gerrit topic $topic"
205 git branch "$refspec" FETCH_HEAD
206
207 has_synched=1
208 elif git fetch -q origin "topics/$topic" &>"$clone_log"; then
209 # If there's a remote branch matching the Gerrit topic
210 # name, checkout to that; otherwise, stay on master.
211 git checkout -q FETCH_HEAD &>"$clone_log"
212 refspec="topics/$topic"
213 stat="on branch $refspec"
214 git branch "$refspec" FETCH_HEAD
215
216 has_synched=1
217 fi
218 fi
219
220 # Generate meta data. Eliminate any quoting in commit subject as it
221 # might cause problems when reporting back to Gerrit.
222 meta_data "$name: $stat"
223 meta_data " $(git show --quiet --format=%H): $(git show --quiet --format=%s | sed "s/[\"']/ /g")"
224 meta_data " Commit date: $(git show --quiet --format=%cd)"
225 meta_data
226
227 # Calculate elapsed seconds
228 s_after="$(date +%s)"
229 let "s_diff = $s_after - $s_before" || true
230
231 echo
232 echo "Repository: $url ($stat)"
233 prefix_arrow <(git show --quiet)
234 echo "Cloned in $s_diff seconds"
235 echo
236
237 popd
238 popd
239
240 emit_env "$loc" "$ci_scratch/$name"
241 emit_env "$ref" "$refspec"
242
243 # If this repository is being tested under a Gerrit trigger, set the
244 # Gerrit test groups.
245 if [ "$name" = "$repo_under_test" ]; then
246 # For a Gerrit trigger, it's possible that users publish patch
247 # sets in quick succession. If the CI is already busy, this
248 # leads to more and more triggers queuing up. Also, it's likey
249 # that older patch sets are tested before new ones. But because
250 # there are newer patch sets already in queue, we should avoid
251 # running tests on older ones as their results will be discarded
252 # anyway.
253 pushd "$ci_scratch/$name"
254
255 change_id="$(git show -q --format=%b | awk '/Change-Id/{print $2}')"
256 commit_id="$(git show -q --format=%H)"
257 latest_commit_id="$($ci_root/script/translate_refspec.py \
258 -p "$name" -u "$gerrit_user" -k "$gerrit_keyfile" \
259 -s "$gerrit_server" "change:$change_id")"
260
261 if [ "$commit_id" != "$latest_commit_id" ]; then
262 # Overwrite Gerrit failure message
263 cat <<EOF >"$gerrit_fail_msg_file"
264Patch set $GERRIT_PATCHSET_NUMBER is not the latest; not tested.
265Please await results for the latest patch set.
266EOF
267
268 cat "$gerrit_fail_msg_file"
269 echo
270 die
271 fi
272
273 # Run nominations on this repository
274 rules_file="$ci_root/script/$name.nomination.py"
275 if [ -f "$rules_file" ]; then
276 "$ci_root/script/gen_nomination.py" "$rules_file" > "$nom_file"
277 if [ -s "$nom_file" ]; then
278 emit_env "NOMINATION_FILE" "$nom_file"
279 echo "$name has $(wc -l < $nom_file) test nominations."
280 fi
281 fi
282
283 popd
284
285 # Allow for groups to be overridden
286 GERRIT_BUILD_GROUPS="${GERRIT_BUILD_GROUPS-$gerrit_build_groups}"
287 if [ "$GERRIT_BUILD_GROUPS" ]; then
288 emit_env "GERRIT_BUILD_GROUPS" "$GERRIT_BUILD_GROUPS"
289 fi
290
291 GERRIT_TEST_GROUPS="${GERRIT_TEST_GROUPS-$gerrit_test_groups}"
292 if [ "$GERRIT_TEST_GROUPS" ]; then
293 emit_env "GERRIT_TEST_GROUPS" "$GERRIT_TEST_GROUPS"
294 fi
295 fi
296}
297
298# When triggered from Gerrit, the main repository that is under test. Can be
299# either TF, TFTF, SCP or CI.
300if [ "$GERRIT_REFSPEC" ]; then
301 repo_under_test="${repo_under_test:-$REPO_UNDER_TEST}"
302 repo_under_test="${repo_under_test:?}"
303fi
304
305# Environment file in Java property file format, that's soured in Jenkins job
306env_file="$workspace/env"
307rm -f "$env_file"
308
309# Workspace on external filer where all repositories gets cloned so that they're
310# accessible to all Jenkins slaves.
311if upon "$local_ci"; then
312 ci_scratch="$workspace/filer"
313else
314 scratch_owner="${JOB_NAME:?}-${BUILD_NUMBER:?}"
315 ci_scratch="$project_scratch/$scratch_owner"
316 tforg_key="$CI_BOT_KEY"
317 tforg_user="$CI_BOT_USERNAME"
318fi
319
320if [ -d "$ci_scratch" ]; then
321 # This could be because of jobs of same name running from
322 # production/staging/temporary VMs
323 echo "Scratch space $ci_scratch already exists; removing."
324 rm -rf "$ci_scratch"
325fi
326mkdir -p "$ci_scratch"
327
328# Nomination file
329nom_file="$ci_scratch/nominations"
330
331# Set CI_SCRATCH so that it'll be injected when sub-jobs are triggered.
332emit_param "CI_SCRATCH" "$ci_scratch"
333
334# However, on Jenkins v2, injected environment variables won't override current
335# job's parameters. This means that the current job (the scratch owner, the job
336# that's executing this script) would always observe CI_SCRATCH as empty, and
337# therefore won't be able to remove it. Therefore, use a different variable
338# other than CI_SCRATCH parameter for the current job to refer to the scratch
339# space (although they both will have the same value!)
340emit_env "SCRATCH_OWNER" "$scratch_owner"
341emit_env "SCRATCH_OWNER_SPACE" "$ci_scratch"
342
343strip_var CI_ENVIRONMENT
344if [ "$CI_ENVIRONMENT" ]; then
345 {
346 echo
347 echo "Injected environment:"
348 prefix_tab <(echo "$CI_ENVIRONMENT")
349 echo
350 } >> "$inject_data"
351
352 cat "$inject_data"
353
354 tmp_env=$(mktempfile)
355 echo "$CI_ENVIRONMENT" > "$tmp_env"
356 source "$tmp_env"
357 cat "$tmp_env" >> "$env_file"
358
359 has_env_inject=1
360fi
361
362if [ "$GERRIT_BRANCH" ]; then
363 # Overrides targeting a specific Gerrit branch.
364 target_branch_override="$ci_overrides/branch/$GERRIT_BRANCH/env"
365 if [ -f "$target_branch_override" ]; then
366 display_override "$target_branch_override"
367
368 {
369 echo
370 echo "Target branch overrides:"
371 prefix_tab "$target_branch_override"
372 echo
373 } >> "$override_data"
374
375 cat "$override_data"
376
377 source "$target_branch_override"
378 cat "$target_branch_override" >> "$env_file"
379
380 has_overrides=1
381 fi
382fi
383
384TF_REFSPEC="${tf_refspec:-$TF_REFSPEC}"
385if not_upon "$no_tf"; then
386 # Clone Trusted Firmware repository
387 url="$tf_src_repo_url" name="trusted-firmware" ref="TF_REFSPEC" \
388 loc="TF_CHECKOUT_LOC" \
389 gerrit_build_groups="tf-gerrit-build" \
390 gerrit_test_groups="tf-gerrit-tests tf-gerrit-tftf" \
391 clone_and_sync
392fi
393
394TFTF_REFSPEC="${tftf_refspec:-$TFTF_REFSPEC}"
395if not_upon "$no_tftf"; then
396 # Clone Trusted Firmware TF repository
397 url="$tftf_src_repo_url" name="trusted-firmware-tf" ref="TFTF_REFSPEC" \
398 loc="TFTF_CHECKOUT_LOC" \
399 gerrit_test_groups="tftf-master-build tftf-master-fwu tftf-l1" \
400 clone_and_sync
401fi
402
403SCP_REFSPEC="${scp_refspec:-$SCP_REFSPEC}"
404if upon "$clone_scp"; then
405 # Clone SCP Firmware repository
406 # NOTE: currently scp/firmware:master is not tracking the upstream.
407 # Therefore, if the url is gerrit.oss.arm.com/scp/firmware and there is
408 # no ref_spec, then set the ref_spec to master-upstream.
409 scp_src_repo_default="http://gerrit.oss.arm.com/scp/firmware"
410 if [ "$scp_src_repo_url" = "$scp_src_repo_default" ]; then
411 SCP_REFSPEC="${SCP_REFSPEC:-master-upstream}"
412 fi
413
414 url="$scp_src_repo_url" name="scp" ref="SCP_REFSPEC" \
415 loc="SCP_CHECKOUT_LOC" clone_and_sync
416
417 pushd "$ci_scratch/scp"
418
419 # Edit the submodule URL to point to the reference repository so that
420 # all submodule update pick from the reference repository instead of
421 # Github.
422 cmsis_ref_repo="${cmsis_root:-$project_filer/ref-repos/cmsis}"
423 if [ -d "$cmsis_ref_repo" ]; then
424 cmsis_reference="--reference $cmsis_ref_repo"
425 fi
426
427 git submodule -q update $cmsis_reference --init
428
429 popd
430fi
431
432CI_REFSPEC="${ci_refspec:-$CI_REFSPEC}"
433if not_upon "$no_ci"; then
434 # Clone Trusted Firmware CI repository
435 url="$tf_ci_repo_url" name="trusted-firmware-ci" ref="CI_REFSPEC" \
436 loc="CI_ROOT" gerrit_test_groups="ci-l1" \
437 clone_and_sync
438fi
439
440if [ "$GERRIT_BRANCH" ]; then
441 # If this CI run was in response to a Gerrit commit, post a comment back
442 # to the patch set calling out everything that we've done so far. This
443 # reassures both the developer and the reviewer about CI refspecs used
444 # for CI testing.
445 #
446 # Note the extra quoting for the message, which Gerrit requires.
447 if upon "$has_synched"; then
448 echo "Branches synchronized:" >> "$gerrit_data"
449 echo >> "$gerrit_data"
450 cat "$clone_data" >> "$gerrit_data"
451 fi
452
453 if upon "$has_overrides"; then
454 cat "$override_data" >> "$gerrit_data"
455 fi
456
457 if upon "$has_env_inject"; then
458 cat "$inject_data" >> "$gerrit_data"
459 fi
460
461 if [ -s "$gerrit_data" ]; then
462 msg_file="$gerrit_data" post_gerrit_comment
463 fi
464fi
465
466# Copy environment file to ci_scratch for sub-jobs' access
467cp "$env_file" "$ci_scratch"
468cp "$param_file" "$ci_scratch"
469
470# Copy clone data so that it's available for sub-jobs' HTML reporting
471if [ -f "$clone_data" ]; then
472 cp "$clone_data" "$ci_scratch"
473fi
474
475# vim: set tw=80 sw=8 noet: