code-coverage: add repos to increase coverage
- At individual jobs, code coverage is increased.
- At merge if repos are missing can generate error.
Change-Id: I43084247c02438d7e8a54997c3f16bc03dad2af9
Signed-off-by: Saul Romero <saul.romero@arm.com>
diff --git a/group/spm-l2-boot-tests-code-coverage/fvp-default,fvp-spm,fvp-48bit-pa:fvp-spm.48bit_pa.bmcov-debug b/group/spm-l2-boot-tests-code-coverage/fvp-default,fvp-spm,fvp-48bit-pa:fvp-spm.48bit_pa.bmcov-debug
deleted file mode 100644
index 8c2d647..0000000
--- a/group/spm-l2-boot-tests-code-coverage/fvp-default,fvp-spm,fvp-48bit-pa:fvp-spm.48bit_pa.bmcov-debug
+++ /dev/null
@@ -1,5 +0,0 @@
-#
-# Copyright (c) 2023, Arm Limited. All rights reserved.
-#
-# SPDX-License-Identifier: BSD-3-Clause
-#
diff --git a/group/spm-l2-boot-tests-code-coverage/fvp-default,fvp-spm,fvp-ivy-vhe:fvp-spm.bmcov-debug b/group/spm-l2-boot-tests-code-coverage/fvp-default,fvp-spm,fvp-ivy-vhe:fvp-spm.bmcov-debug
deleted file mode 100644
index ca6c54c..0000000
--- a/group/spm-l2-boot-tests-code-coverage/fvp-default,fvp-spm,fvp-ivy-vhe:fvp-spm.bmcov-debug
+++ /dev/null
@@ -1,10 +0,0 @@
-#
-# Copyright (c) 2023, Arm Limited. All rights reserved.
-#
-# SPDX-License-Identifier: BSD-3-Clause
-#
-
-pre_spm_build() {
- # Hafnium build repo containing Sercure hafnium binary build for vhe
- out_dir="secure_aem_v8a_fvp_vhe_clang" set_spm_out_dir
-}
diff --git a/group/spm-l2-boot-tests-code-coverage/fvp-default,fvp-spm-measured-boot,fvp-default:fvp-spm+romlib.bmcov-debug b/group/spm-l2-boot-tests-code-coverage/fvp-default,fvp-spm-measured-boot,fvp-default:fvp-spm+romlib.bmcov-debug
deleted file mode 100644
index c473896..0000000
--- a/group/spm-l2-boot-tests-code-coverage/fvp-default,fvp-spm-measured-boot,fvp-default:fvp-spm+romlib.bmcov-debug
+++ /dev/null
@@ -1,6 +0,0 @@
-#
-# Copyright (c) 2023, Arm Limited. All rights reserved.
-#
-# SPDX-License-Identifier: BSD-3-Clause
-#
-
diff --git a/group/spm-l2-boot-tests-code-coverage/fvp-default,fvp-spm-sve,fvp-default-sve:fvp-spm.sve+amu.bmcov-debug b/group/spm-l2-boot-tests-code-coverage/fvp-default,fvp-spm-sve,fvp-default-sve:fvp-spm.sve+amu.bmcov-debug
deleted file mode 100644
index 8c2d647..0000000
--- a/group/spm-l2-boot-tests-code-coverage/fvp-default,fvp-spm-sve,fvp-default-sve:fvp-spm.sve+amu.bmcov-debug
+++ /dev/null
@@ -1,5 +0,0 @@
-#
-# Copyright (c) 2023, Arm Limited. All rights reserved.
-#
-# SPDX-License-Identifier: BSD-3-Clause
-#
diff --git a/group/spm-l2-boot-tests-code-coverage/fvp-default,fvp-spm-tbb,fvp-default:fvp-spm+romlib.bmcov-debug b/group/spm-l2-boot-tests-code-coverage/fvp-default,fvp-spm-tbb,fvp-default:fvp-spm+romlib.bmcov-debug
deleted file mode 100644
index 8c2d647..0000000
--- a/group/spm-l2-boot-tests-code-coverage/fvp-default,fvp-spm-tbb,fvp-default:fvp-spm+romlib.bmcov-debug
+++ /dev/null
@@ -1,5 +0,0 @@
-#
-# Copyright (c) 2023, Arm Limited. All rights reserved.
-#
-# SPDX-License-Identifier: BSD-3-Clause
-#
diff --git a/group/spm-l2-boot-tests-code-coverage/fvp-default,fvp-spm-tbb-dualroot,fvp-default:fvp-spm+romlib.bmcov-debug b/group/spm-l2-boot-tests-code-coverage/fvp-default,fvp-spm-tbb-dualroot,fvp-default:fvp-spm+romlib.bmcov-debug
deleted file mode 100644
index 8c2d647..0000000
--- a/group/spm-l2-boot-tests-code-coverage/fvp-default,fvp-spm-tbb-dualroot,fvp-default:fvp-spm+romlib.bmcov-debug
+++ /dev/null
@@ -1,5 +0,0 @@
-#
-# Copyright (c) 2023, Arm Limited. All rights reserved.
-#
-# SPDX-License-Identifier: BSD-3-Clause
-#
diff --git a/group/tf-l3-code-coverage-minimal/fvp-default-cc,fvp-cpu-extensions:fvp-tftf-fip.tftf-aemv8a.sve.bmcov-debug b/group/tf-l3-code-coverage-minimal/fvp-default-cc,fvp-cpu-extensions:fvp-tftf-fip.tftf-aemv8a.sve.bmcov-debug
new file mode 100644
index 0000000..aab4796
--- /dev/null
+++ b/group/tf-l3-code-coverage-minimal/fvp-default-cc,fvp-cpu-extensions:fvp-tftf-fip.tftf-aemv8a.sve.bmcov-debug
@@ -0,0 +1,6 @@
+#
+# Copyright (c) 2019-2020 Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
diff --git a/group/spm-l2-boot-tests-code-coverage/fvp-default,fvp-spm-rme,fvp-default:fvp-spm.trp.tftf-tftf.rme.bmcov-debug b/group/tf-l3-code-coverage-minimal/fvp-tspd-tbb-mbedtls-cc,fvp-fwu:fvp-tftf.fwu-aemv8a.bmcov-debug
similarity index 100%
rename from group/spm-l2-boot-tests-code-coverage/fvp-default,fvp-spm-rme,fvp-default:fvp-spm.trp.tftf-tftf.rme.bmcov-debug
rename to group/tf-l3-code-coverage-minimal/fvp-tspd-tbb-mbedtls-cc,fvp-fwu:fvp-tftf.fwu-aemv8a.bmcov-debug
diff --git a/job/tf-ci-gateway/generate_report.sh b/job/tf-ci-gateway/generate_report.sh
index 4f9d8b7..00d4e86 100755
--- a/job/tf-ci-gateway/generate_report.sh
+++ b/job/tf-ci-gateway/generate_report.sh
@@ -5,7 +5,8 @@
# SPDX-License-Identifier: BSD-3-Clause
#
-set -ex
+set -e
+set -x
# Jenkins Parameterized Trigger Plugin mangles job names as passed via
# environment variables, replacing most non-alphanumeric chars with
@@ -62,6 +63,9 @@
--meta-data html:coverity.data \
|| true
- source $CI_ROOT/script/gen_merge_report.sh "${WORKSPACE}/report.json" \
- "${WORKSPACE}/report.html"
+ # Only call to merge reports if the test groups are for code coverage
+ if [[ $TEST_GROUPS == *"code-coverage"* ]]; then
+ source $CI_ROOT/script/gen_merge_report.sh "${WORKSPACE}/report.json" \
+ "${WORKSPACE}/report.html"
+ fi
fi
diff --git a/script/gen_merge_report.sh b/script/gen_merge_report.sh
index 4cd4f67..2431b22 100644
--- a/script/gen_merge_report.sh
+++ b/script/gen_merge_report.sh
@@ -33,6 +33,7 @@
python3 - << EOF
import json
import os
+import re
server = os.getenv("JENKINS_URL", "https://jenkins.oss.arm.com/")
merge_json = {} # json object
@@ -48,6 +49,8 @@
merge_number += 1
base_url = "{}job/{}/{}/{}".format(
server, data['job'], build_number, "$ARTIFACT_PATH")
+ _group_test_config = re.match(f'^[0-9%]*(?:${TEST_GROUPS}%)?(.+?)\.test', test_files[index])
+ tf_configuration = _group_test_config.groups()[0] if _group_test_config else 'N/A'
_files.append( {'id': build_number,
'config': {
'type': 'http',
@@ -58,7 +61,8 @@
'type': 'http',
'origin': "{}/{}".format(
base_url, "$INFO_PATH")
- }
+ },
+ 'tf-configuration': tf_configuration
})
merge_json = { 'files' : _files }
with open("$MERGE_CONFIGURATION", 'w') as outfile:
@@ -78,14 +82,6 @@
confs = ""
with open("$REPORT_JSON") as json_file:
data = json.load(json_file)
-test_files = data['test_files']
-test_results = data['test_results']
-for index, build_number in enumerate(test_results):
- test_file = test_files[index]
- test_configuration = test_file.rsplit('%', 1)
- if len(test_configuration) > 1:
- confs += '<a target="_blank" href="artifact/${jenkins_archive_folder}/${COVERAGE_FOLDER}/{}/index.html">{}</a>'.format(build_number,
- test_configuration[1])
with open(cov_html, "r") as f:
html_content = f.read()
@@ -168,14 +164,8 @@
</tbody>
</table>
<p>
- <button onclick="window.open('artifact/${jenkins_archive_folder}/${COVERAGE_FOLDER}/index.html','_blank');">Total Coverage Report</button>
+ <button onclick="window.open('artifact/${jenkins_archive_folder}/${COVERAGE_FOLDER}/index.html','_blank');">Total Coverage Report (${#list_of_merged_builds[@]} out of ${number_of_files_to_merge})</button>
</p>
- <div class="dropdown">
- <button class="dropbtn">Coverage Reports($number_of_files_to_merge)</button>
- <div class="dropdown-content">
- """ + confs + """
- </div>
- </div>
</div>
<script>
@@ -187,6 +177,54 @@
f.write(s)
EOF
}
+
+generate_cols() {
+ echo "List of merged build ids:${list_of_merged_builds[@]}"
+python3 - << EOF
+merged_ids=[int(i) for i in "${list_of_merged_builds[@]}".split()]
+s = """
+
+ <script>
+ window.onload = function() {
+ """ + f"const mergedIds={merged_ids}" + """
+ document.querySelector('#tf-report-main table').querySelectorAll("tr").forEach((row,i) => {
+ const cell = document.createElement(i ? "td" : "th")
+ const button = document.createElement("button")
+ button.textContent = "Report"
+ if (i) {
+ merged = false
+ if (q = row.querySelector('td.success a.buildlink')) {
+ href = q.href
+ buildId = href.split("/").at(-2)
+ if (mergedIds.include(buildId)) {
+ cell.classList.add("success")
+ const url = href.replace('console', 'artifact/trace_report/index.html')
+ button.addEventListener('click', () => {
+ window.open(url, "_blank")
+ })
+ cell.appendChild(button)
+ merged = true
+ }
+ }
+ if (!merged) {
+ cell.innerText = "N/A"
+ cell.classList.add("failure")
+ }
+ }
+ else {
+ cell.innerText = "Code Coverage"
+ }
+ row.appendChild(cell)
+ })
+ }
+ </script>
+"""
+with open("$1", "a") as f:
+ f.write(s)
+EOF
+}
+
+
OUTDIR=""
index=""
case "$TEST_GROUPS" in
@@ -212,16 +250,18 @@
mkdir -p $OUTDIR
pushd $OUTDIR
number_of_files_to_merge=$(create_merge_cfg)
- echo "Merging $number_of_files_to_merge coverage files..."
+ echo "Merging from $number_of_files_to_merge code coverage reports..."
# Only merge when more than 1 test result
if [ "$number_of_files_to_merge" -lt 2 ] ; then
echo "Only one file to merge."
exit 0
fi
- bash ${WORKSPACE}/qa-tools/coverage-tool/coverage-reporting/merge.sh \
- -j $MERGE_CONFIGURATION -l ${OUTDIR}/${COVERAGE_FOLDER} -w $WORKSPACE -c -g
+ source ${WORKSPACE}/qa-tools/coverage-tool/coverage-reporting/merge.sh \
+ -j $MERGE_CONFIGURATION -l ${OUTDIR}/${COVERAGE_FOLDER} -w $WORKSPACE -c
generate_header ${REPORT_HTML}
+ generate_cols ${REPORT_HTML}
cp ${REPORT_HTML} $OUTDIR
+
popd
diff --git a/script/qa-code-coverage.sh b/script/qa-code-coverage.sh
index 95cc32f..f8d28c1 100644
--- a/script/qa-code-coverage.sh
+++ b/script/qa-code-coverage.sh
@@ -6,6 +6,7 @@
#
# Include variables and functions to be used by these scripts
+set -x
source "$CI_ROOT/utils.sh"
################################################################################
# CI VARIABLES: