vizualisation: Add tf-main vizualization script
Add `tf-a-main-results.{bash,plot}` to fetch test results from a Jenkins
build of tf-main and vizualise them. Update README.rst with usage info
for the new script and plot description.
Change-Id: I837a2dcb4cd3247b24ba9eec6557699e9a75e9c2
Signed-off-by: Nathan Dunne <Nathan.Dunne@arm.com>
Signed-off-by: Harrison Mutai <harrison.mutai@arm.com>
diff --git a/script/graphs/README.rst b/script/graphs/README.rst
index 0860cb2..b057943 100644
--- a/script/graphs/README.rst
+++ b/script/graphs/README.rst
@@ -41,6 +41,66 @@
.. code-block::
- bash ../<this-repo>/script/graph/sloc-viz.bash > sloc.png 2> sloc.tsv
+ bash ../<this-repo>/script/graphs/sloc-viz.bash > sloc.png 2> sloc.tsv
+
+Test Results
+------------
+
+The script `tf-main-results.bash` uses curl to retrieve test results for a
+tf-a-main Jenkins job, and generates a CSV and stacked histogram PNG of the
+combined data.
+
+Usage:
+======
+
+ bash tf-main-results.bash <jenkins-url> [ci_gateway] [filter]
+
+The Jenkins URL is the URL for the target build job.
+
+ https://ci.trustedfirmware.org/job/tf-a-main/1/
+
+The sub-builds for this job will all be queried to find the ones that contain
+tests, ignoring child builds that only build and don't run any platform tests.
+
+`tf-a-ci-gateway` is the default gateway, although, different gateways may be
+specified with the optional "ci_gateway" argument. This option will be combined
+with the build numbers and the base Jenkins URL to retrieve the results of
+sub-builds.
+
+This can be filtered further using the optional "filter" argument, which will
+select only test groups that match the provided regex.
+
+Example Useful Queries
+======================
+
+Only show tests running the test framework:
+
+ bash tf-main-results.bash <jenkins-url> [ci_gateway] "tftf"
+
+Only show tests for N1SDP & Juno platforms:
+
+ bash tf-main-results.bash <jenkins-url> [ci_gateway] "n1sdp|juno"
+
+Only show boot tests on FVP platforms:
+
+ bash tf-main-results.bash <jenkins-url> [ci_gateway] 'fvp.*boot'
+
+Note: for filters that return a small number of test groups, the graph is not
+ideal as it is sized for a large number. A CSV file of the data is also produced,
+however, so that you can use it to create your own graph, if required.
+
+Additional Config
+=================
+
+The script also allows the three output files to be configured via ENV variables:
+
+ PNGFILE=out.png CSVFILE=out.csv bash tf-main-results.bash
+
+If they are not set then default values based on the build number will be generated:
+
+ PNGFILE_DEFAULT="tf-main_${build_number}.png"
+ CSVFILE_DEFAULT="tf-main_${build_number}.csv"
+
+If any of these files already exist then they will be overwritten.
*Copyright (c) 2021-2022, Arm Limited. All rights reserved.*
diff --git a/script/graphs/tf-main-results.bash b/script/graphs/tf-main-results.bash
new file mode 100644
index 0000000..34ff885
--- /dev/null
+++ b/script/graphs/tf-main-results.bash
@@ -0,0 +1,197 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2022 Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+set -euo pipefail
+
+# Build and Jenkins URL.
+sub_build_url=${1}
+job_name="${2:-"tf-a-ci-gateway"}"
+filter=${3:-".*"}
+
+jenkins="${sub_build_url%%/job*}"
+
+# Utilise default paths to output files if none provided.
+job_target="$(dirname ${sub_build_url#*/job/})"
+PNGFILE=${PNGFILE:=${job_target}-result.png}
+CSVFILE=${CSVFILE:=${job_target}-result.csv}
+
+# Remove csv output file if it exists to append to empty file
+: > "${CSVFILE}"
+
+readarray -t sub_builds < <(curl -sSL "${sub_build_url}/api/json" | jq -Rr '
+ fromjson? | [
+ .subBuilds[]? | select(.jobName == "'${job_name}'") | .url
+ ] | .[]')
+
+# Get a csv list of relative paths to report.json, or "-" if no report.json
+report_rel_paths_url="${jenkins}/{$(echo $(IFS=,; echo "${sub_builds[*]}"))}/api/json"
+
+readarray -t report_paths < <(curl -fsSL --fail-early "${report_rel_paths_url}" \
+ | sed 's/--_curl_--.*$//' \
+ | sed -e 's/^{/'$(printf "\x1e")'{/' \
+ | jq -sr --seq '
+ [ .[]
+ | [ .artifacts[]?
+ | select (.fileName == "report.json")
+ | .relativePath ]
+ | if length > 0 then .[] else "-" end ]
+ | .[]')
+
+# Combine sub build urls with relative path to "report.json"
+# the empty entries are intentionally kept as -, so the output array can
+# be mapped onto ${sub_build_list}
+report_urls="$jenkins/{"
+for i in "${!sub_builds[@]}"
+do
+ if [[ ${report_paths[i]} != "-" ]]
+ then
+ report_urls="${report_urls}${sub_builds[i]}/artifact/${report_paths[i]},"
+ fi
+done
+
+# Strip last comma and add closing brace
+report_urls="${report_urls%?}}"
+
+# Get Child build information from each report.json.
+readarray -t child_file_list_array < <(curl -sL "${report_urls}" -o -\
+ | sed 's/--_curl_--.*$//' \
+ | jq -sr --arg FILTER "${filter}" \
+ '[.[]
+ | .job as $job
+ | [ .child_build_numbers?, [(.test_files[]
+ | sub("\\.test";"")
+ | split("%") as $config
+ | { group: $config[1], suite: $config[2]})]]
+ | transpose
+ | map( {($job + "/" + .[0]) : .[1]} | to_entries )
+ | add
+ | map(select(.value.suite | test(".*"; "il")?
+ and (endswith("nil")
+ or endswith("norun-fip.dummy") | not)))
+ | if ( length > 0 )
+ then .[] else empty end
+ | .value.group, (.value.suite | gsub("\\,nil";"")), .key]
+ | .[]')
+
+# These three arrays should be the same length, and values at the same index
+# correspond to the same child build
+declare -a tftf_keys tftf_suite tftf_group
+
+for i in $(seq 0 3 $((${#child_file_list_array[@]}-1))) ; do
+ tftf_group+=("${child_file_list_array[$i]}")
+ tftf_suite+=("${child_file_list_array[$i+1]}")
+ tftf_keys+=("${child_file_list_array[$i+2]}")
+done
+
+
+child_output_results_url="${jenkins}/job/{$(echo $(IFS=,; echo "${tftf_keys[*]}"))}/api/json"
+
+# Retrieve relative path to either "uart0_full.txt" (FVP) or
+# "job_output.log" (LAVA) for each child job. Once again values where no match
+# is found are intentionally kept as "-" so the array can be correlated with
+# ${tftf_suite}.
+readarray -t child_output_results < <(curl -fsSL --fail-early "$child_output_results_url" \
+ | sed 's/}{/}\n{/g' \
+ | jq -sr '[ .[]
+ | ([ .artifacts[]?
+ | select(.fileName == "uart0_full.txt"
+ or .fileName == "job_output.log"
+ or .fileName == "lava-uart0.log") ]
+ | if length > 0
+ then .[0].relativePath else "-" end), .result ]
+ | .[]')
+
+# Combine job and child_build number with relative path to output file
+testlog_urls="${jenkins}/job/{"
+tftf_child_results=()
+
+for i in $(seq 0 2 $((${#child_output_results[@]}-1))) ; do
+ testlog_urls+="${tftf_keys[$((i/2))]}/artifact/${child_output_results[$i]},"
+ tftf_child_results+=(${child_output_results[$((i+1))]})
+done
+
+# Remove final comma and append a closing brace
+testlog_urls="${testlog_urls%?}}"
+
+# Retrieve the log for each child with --include to also retrieve the HTTP
+# header and grep for a block like:
+# Tests Skipped : 125
+# Tests Passed : 45
+# Tests Failed : 0
+# Tests Crashed : 0
+#
+# If none is found the line HTTP is used to delemit each entry
+#
+# Logs from Lava each message is wrapped with braces and has some pre-amble,
+# which is removed with sed.
+
+tftf_result_keys=(
+ "TestGroup" "TestSuite" "URL" "Result" "Passed" "Failed" "Crashed" "Skipped"
+)
+declare -A results_split="( $(for ord_ in ${tftf_result_keys[@]} ; do echo -n "[$ord_]=\"\" "; done))"
+declare output_csv_str="" csv_row=""
+
+read -ra tftf_urls <<< "$(eval "echo ${testlog_urls}")"
+
+# FIXME adjust this so we can handle both LAVA logs
+# for each test suite
+# curl the result log if its not '-'
+# remove debug information
+# if results is none:
+ # use "Result"
+# else
+ # read each key
+# write row to csv
+
+# Sort results into rows:
+for i in ${!tftf_suite[*]}; do
+ results_split["TestGroup"]="${tftf_group[$i]:-}"
+ results_split["TestSuite"]="\"${tftf_suite[$i]:-}\""
+ results_split["URL"]="${tftf_urls[$i]:-}"
+ results_split["Result"]="${tftf_child_results[$i]:-}"
+
+ # Skipped/Crashed are always zero if no test block is found
+ results_split["Skipped"]="0"
+ results_split["Crashed"]="0"
+ if [[ "${results_split["Result"]}" == "SUCCESS" ]];
+ then
+ results_split["Passed"]="1"
+ results_split["Failed"]="0"
+ else
+ results_split["Passed"]="0"
+ results_split["Failed"]="1"
+ fi
+
+ readarray -t raw_result < <(curl -sL --include "${results_split["URL"]}" \
+ | sed 's/.*msg": "//g' \
+ | grep --text -E "^Tests|HTTP\/")
+
+ for line in "${raw_result[@]}"; do
+ if [[ "${line}" == Test* ]]
+ then
+ k=$(echo "${line}" | awk -F ' ' '{print $2}')
+ count="${line//[!0-9]/}"
+ results_split[$k]=$count
+ fi
+ done
+
+ # Generate CSV row using array of ordinals to align with headers.
+ readarray -t row < <(for k in ${tftf_result_keys[@]} ; do echo "${results_split[$k]}"; done )
+ output_csv_str="${output_csv_str} $(echo $(IFS=,; echo "${row[*]}"))"
+ unset results_split[{..}] row
+done
+
+# Replace spaces in header with commas and print to the output file.
+echo $(IFS=,; echo "${tftf_result_keys[*]}") > ${CSVFILE}
+
+# Sort Filenames alphabetically and store in csv for gnuplot
+sorted=($(IFS=$' '; sort <<<$output_csv_str))
+printf "%b\n" "${sorted[@]}" >> ${CSVFILE}
+
+# Produce PNG image of graph using gnuplot and .plot description file
+gnuplot -e "jenkins_id='$sub_build_url'" -c ${0%bash}plot \
+ "$CSVFILE" > "$PNGFILE"
diff --git a/script/graphs/tf-main-results.plot b/script/graphs/tf-main-results.plot
new file mode 100644
index 0000000..3c8e345
--- /dev/null
+++ b/script/graphs/tf-main-results.plot
@@ -0,0 +1,58 @@
+#
+# Copyright (c) 2022 Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+# Determine the number of test groups, scale the terminal based on the number
+# of these.
+groups=system(sprintf("awk -F ',' 'NR!=1 {print $1}' %s | sort | uniq", ARG1))
+group_count=words(groups)
+
+# Terminal height should scale with the number of groups
+# (each group has a plot).
+set terminal pngcairo enhanced size 5400,9600 font ',14'
+
+set xrange [0:5<*] extend
+set ytic scale 0 nomirror
+set grid xtics
+set lmargin 70
+set bmargin 5
+set yrange [:] reverse
+set offsets 0,0,0.5,0.5
+set datafile separator ","
+
+set key autotitle columnhead
+
+# Create linetypes for coloured labels
+set linetype 1 linecolor "red"
+set linetype 2 linecolor "black"
+fill(n) = word("green red orange gray", n)
+
+set multiplot title "TF-A CI Test Results: " . jenkins_id \
+ font ",30" layout ceil(group_count/3.0),3
+
+ set style data histograms
+ set style fill solid 0.3 border -1
+ set key outside left vertical
+ set label "Test Suites" at screen 0.05,0.5 \
+ center front rotate font ",20"
+
+ do for [group in groups]{
+ set title group font ",18"
+ set style histogram rowstacked
+ filter = "awk -F, 'NR==1 || $1==\"".group."\"'"
+ col_count = 8
+ box_width = 0.5
+
+ plot for [col=5:col_count] '< '.filter.' '.ARG1 u col:0: \
+ (sum [i=5:col-1] column(i)): \
+ (sum [i=5:col] column(i)): \
+ ($0-box_width/2.):($0+box_width/2.):ytic(2) w boxxyerror \
+ ti columnhead(col) lc rgb fill(col-4)
+
+ unset key
+ }
+
+unset multiplot
+