Add eclair scripts
copy these scripts from ci/tf-ci-scripts
Signed-off-by: Arthur She <arthur.she@linaro.org>
Change-Id: I555cb85bcb207d247d84e06fcd47d22e55b4ac10
diff --git a/eclair/analyze_delta_index_html.sh b/eclair/analyze_delta_index_html.sh
new file mode 100755
index 0000000..155d48a
--- /dev/null
+++ b/eclair/analyze_delta_index_html.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Copyright (c) 2023 Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# Template to produce index.html for the "delta" build.
+
+cat <<EOF >index.html
+<html>
+<body>
+<h1>MISRA Delta reports for the patch</h1>
+
+<p>
+Patch: <a href="${GERRIT_CHANGE_URL}/${GERRIT_PATCHSET_NUMBER}">${GERRIT_CHANGE_URL}/${GERRIT_PATCHSET_NUMBER}</a><br />
+CI Build: <a href="${BUILD_URL}">${BUILD_URL}</a>
+</p>
+
+<li><a href="misra_delta.txt">Cumulative TXT report</a>
+<li><a href="diff_output/">Per MISRA rule TXT reports</a>
+<li><a href='new_issues_html/by_service.html#first_file/service&kind{"select":true,"selection":{"hiddenAreaKinds":[],"hiddenSubareaKinds":[],"show":false,"selector":{"enabled":true,"negated":false,"kind":2,"children":[{"enabled":true,"negated":false,"kind":0,"domain":"kind","inputs":[{"enabled":true,"text":"information"}]},{"enabled":true,"negated":false,"kind":0,"domain":"service","inputs":[{"enabled":true,"text":"MC3R1.R5.9"}]},{"enabled":true,"negated":false,"kind":0,"domain":"service","inputs":[{"enabled":true,"text":"MC3R1.R8.3"}]}]}}}'>New issues, groupped per file changed (HTML).</a>
+<li><a href='resolved_issues_html/by_service.html#first_file/service&kind{"select":true,"selection":{"hiddenAreaKinds":[],"hiddenSubareaKinds":[],"show":false,"selector":{"enabled":true,"negated":false,"kind":2,"children":[{"enabled":true,"negated":false,"kind":0,"domain":"kind","inputs":[{"enabled":true,"text":"information"}]},{"enabled":true,"negated":false,"kind":0,"domain":"service","inputs":[{"enabled":true,"text":"MC3R1.R5.9"}]},{"enabled":true,"negated":false,"kind":0,"domain":"service","inputs":[{"enabled":true,"text":"MC3R1.R8.3"}]}]}}}'>Resolved issues, groupped per file changed (HTML).</a>
+</body>
+</html>
+EOF
diff --git a/eclair/analyze_index_html.sh b/eclair/analyze_index_html.sh
new file mode 100755
index 0000000..c1a5795
--- /dev/null
+++ b/eclair/analyze_index_html.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+#
+# Copyright (c) 2022 Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# Template to produce index.html for the "full" build.
+
+if [ -n "${TF_CONFIG}" ]; then
+ build_config="TF-A Config: ${TF_CONFIG}<br />"
+fi
+
+cat <<EOF >index.html
+<html>
+<body>
+<h1>MISRA reports</h1>
+
+<p>
+${build_config}
+CI Build: <a href="${BUILD_URL}">${BUILD_URL}</a>
+</p>
+
+Reports:
+<ul>
+<li><a href='ECLAIR/full_html/by_service.html#service/first_file&kind{"select":true,"selection":{"hiddenAreaKinds":[],"hiddenSubareaKinds":[],"show":true,"selector":{"enabled":true,"negated":false,"kind":1,"children":[{"enabled":true,"negated":false,"kind":0,"domain":"strictness","inputs":[{"enabled":true,"text":"mandatory"}]},{"enabled":true,"negated":false,"kind":0,"domain":"kind","inputs":[{"enabled":true,"text":"violation"}]}]}}}'>Mandatory rules - violations</a>
+<li><a href='ECLAIR/full_html/by_service.html#service/first_file&kind{"select":true,"selection":{"hiddenAreaKinds":[],"hiddenSubareaKinds":[],"show":true,"selector":{"enabled":true,"negated":false,"kind":1,"children":[{"enabled":true,"negated":false,"kind":0,"domain":"strictness","inputs":[{"enabled":true,"text":"mandatory"}]},{"enabled":true,"negated":false,"kind":0,"domain":"kind","inputs":[{"enabled":true,"text":"violation"},{"enabled":true,"text":"caution"}]}]}}}'>Mandatory rules - violations & cautions</a>
+<li><a href='ECLAIR/full_html/by_service.html#strictness/service/first_file&strictness{"select":true,"selection":{"hiddenAreaKinds":[],"hiddenSubareaKinds":[],"show":true,"selector":{"enabled":true,"negated":false,"kind":2,"children":[{"enabled":true,"negated":false,"kind":0,"domain":"kind","inputs":[{"enabled":true,"text":"violation"}]}]}}}'>Report by issue strictness (Mandatory/Required/Advisory) (violations)</a>
+<li><a href='ECLAIR/full_html/by_service.html#strictness/service/first_file&strictness'>Report by issue strictness (Mandatory/Required/Advisory) (all)</a>
+</ul>
+
+<ul>
+<li><a href="ECLAIR/full_html/index.html">Default ECLAIR report</a>
+<li><a href="ECLAIR/full_txt/">Default ECLAIR report (plain text)</a>
+</ul>
+
+<span style="font-size: 75%">
+<p>
+ECLAIR terminology cheatsheet:
+</p>
+<ul>
+<li>"violation" is formally proven issue
+<li>"caution" is <i>not</i> formally proven issue, may be a false positive
+<li>"information" is <i>not an issue</i> (from MISRA rules PoV), just FYI aka "know your codebase better"
+</ul>
+</span>
+
+</body>
+</html>
+EOF
diff --git a/eclair/eclair_diff_report.py b/eclair/eclair_diff_report.py
new file mode 100755
index 0000000..ef3c208
--- /dev/null
+++ b/eclair/eclair_diff_report.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2022 Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+# Script to produce cumulative "diff" report from ECLAIR individual
+# per-service diff reports.
+
+import sys
+import glob
+import os
+from string import Template
+
+
+def lcut(l, prefix):
+ if l.startswith(prefix):
+ l = l[len(prefix):]
+ return l
+
+
+def process_file(fname, del_or_add):
+ with open(fname) as f:
+ for l in f:
+ l = l.rstrip()
+ if not l:
+ break
+ for l in f:
+ if l.startswith("service "):
+ l = lcut(l, "service ")
+ if del_or_add == "del":
+ l = "Resolved for " + l
+ else:
+ l = "Added for " + l
+ elif l.startswith("End of report"):
+ l = "---------------\n"
+ sys.stdout.write(l)
+
+
+path = "."
+if len(sys.argv) > 1:
+ path = sys.argv[1]
+
+files = sorted(glob.glob(path + "/*.etr"))
+#print(files)
+
+EMPTY_REPORT_HEADER = Template("""\
+No new MISRA issues detected, good work!
+${BUILD_URL}artifact/
+""").safe_substitute(os.environ)
+
+NONEMPTY_REPORT_HEADER = Template("""\
+MISRA delta report: ${BUILD_URL}artifact/
+
+= MISRA delta report for the patch (issues resolved and/or newly added) =
+""").safe_substitute(os.environ)
+
+header_done = False
+
+for f in files:
+ if "/B.EXPLAIN" in f:
+ continue
+ comp = f.rsplit(".", 2)
+# print("*", f, comp)
+ if not header_done:
+ print(NONEMPTY_REPORT_HEADER)
+ header_done = True
+ process_file(f, comp[-2])
+
+if not header_done:
+ print(EMPTY_REPORT_HEADER)
diff --git a/eclair/post_gerrit_comment.sh b/eclair/post_gerrit_comment.sh
new file mode 100755
index 0000000..9d2042f
--- /dev/null
+++ b/eclair/post_gerrit_comment.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Copyright (c) 2019-2022, Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+set -ex
+
+# Set to 0 to temporarily disable posting comments to Gerrit.
+should_post_comment=1
+
+# Directory where this script resides.
+SCRIPT_DIR="$(cd "$(dirname "$0")" ; echo "${PWD}")"
+
+# Don't post comments if run on the staging server.
+if echo "$JENKINS_PUBLIC_URL" | grep -q "ci\.staging"; then
+ should_post_comment=0
+fi
+
+# Always enable posting comments to sandbox (test) projects, even if they're
+# disabled above.
+if echo "${GERRIT_PROJECT}" | grep -q sandbox; then
+ should_post_comment=1
+fi
+
+# If run without a patch (e.g. for debugging, don't try to post comment.
+if [ -z "$GERRIT_CHANGE_NUMBER" ]; then
+ should_post_comment=0
+fi
+
+if [ $should_post_comment -eq 1 ]; then
+ mkdir -p ~/.ssh/
+ ssh-keyscan -H -p 29418 $GERRIT_HOST >> ~/.ssh/known_hosts
+
+ quoted="$(python3 $SCRIPT_DIR/prepare_gerrit_comment.py misra_delta.txt)"
+
+ ssh -o "PubkeyAcceptedKeyTypes +ssh-rsa" -p 29418 -i "$CI_BOT_KEY" "$CI_BOT_USERNAME@$GERRIT_HOST" gerrit \
+ review "$GERRIT_CHANGE_NUMBER,$GERRIT_PATCHSET_NUMBER" \
+ --message "$quoted"
+fi
diff --git a/eclair/prepare_gerrit_comment.py b/eclair/prepare_gerrit_comment.py
new file mode 100755
index 0000000..b5281b2
--- /dev/null
+++ b/eclair/prepare_gerrit_comment.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2022 Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+# Script to prepare a textual body of a comment to pass on the command line
+# to Gerrit: limit it to acceptable size and quote properly.
+
+import sys
+import shlex
+
+
+SIZE_LIMIT = 16000
+
+
+body = ""
+
+with open(sys.argv[1], "r") as f:
+ for l in f:
+ if len(body) + len(l) >= SIZE_LIMIT:
+ body += """\
+[...]
+
+WARNING: The report was trimmed due to size limit of a Gerrit comment.
+Follow the link at the beginning to see the full report.
+"""
+ break
+ body += l
+
+sys.stdout.write(shlex.quote(body))
diff --git a/eclair/relativize_urls.py b/eclair/relativize_urls.py
new file mode 100755
index 0000000..57c029f
--- /dev/null
+++ b/eclair/relativize_urls.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2022-2023 Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+# Script to replace absolute paths in ECLAIR HTML reports, pointing to
+# "dependency" files like CSS & JS, to relative paths, pointing to
+# those files copied alongside the report.
+
+import sys
+import re
+import os
+import glob
+
+
+os.chdir(sys.argv[1])
+for fn in glob.iglob("**/*.html", recursive=True):
+ depth = fn.count("/")
+ relp = "/".join([".."] * depth)
+ if relp:
+ relp += "/"
+ #print(fn, relp)
+ with open(fn) as f:
+ txt = f.read()
+ txt = re.sub(r"/opt/bugseng/eclair[^/]*/", relp, txt)
+ txt = txt.replace("/opt/bugseng/eclair-3.12.0/", relp)
+ #os.rename(fn, fn + ".bak")
+ with open(fn, "w") as f:
+ f.write(txt)
diff --git a/eclair/sel_tag_and_not_glob.ecl b/eclair/sel_tag_and_not_glob.ecl
new file mode 100644
index 0000000..8fdf8b1
--- /dev/null
+++ b/eclair/sel_tag_and_not_glob.ecl
@@ -0,0 +1,9 @@
+defun(sel_tag_and_not_glob(s,domain1,tag1,domain2,tag2),
+ create_sel(s,
+ [["clear_all",s],
+ ["add_tag_glob",s,domain1,tag1],
+ ["select_tag","",s],
+ ["reset",s],
+ ["add_tag_glob",s,domain2,tag2],
+ ["select_not_tag",s,""]]),
+ sel(s))
diff --git a/eclair/utils.sh b/eclair/utils.sh
new file mode 100644
index 0000000..84efeb1
--- /dev/null
+++ b/eclair/utils.sh
@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2022-2023 Arm Limited. All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# Common utility functions for Bugseng ECLAIR tool.
+
+# Directory where this script resides (assuming it was sourced).
+export _ECLAIR_UTILS_DIR="$(cd "$(dirname "$BASH_SOURCE")" ; echo "${PWD}")"
+
+# Absolute path of the ECLAIR bin directory.
+ECLAIR_BIN_DIR="/opt/bugseng/eclair/bin"
+
+
+# Set various path variables based on ECLAIR_ANALYSIS var (which is a name
+# of top-level dir to hold internal files and results of anaylisis). We
+# need to support this variability to e.g. support delta reports
+# between two analyses (before and after a patch).
+eclair_set_paths() {
+ if [ -z "${ECLAIR_ANALYSIS}" ]; then
+ echo "ECLAIR_ANALYSIS is not defined"
+ exit 1
+ fi
+
+ # Directory where to put all ECLAIR output and temporary files.
+ ECLAIR_OUTPUT_DIR="${WORKSPACE}/${ECLAIR_ANALYSIS}/out"
+
+ # ECLAIR binary data directory and workspace.
+ export ECLAIR_DATA_DIR="${ECLAIR_OUTPUT_DIR}/.data"
+
+ # Destination file for the ECLAIR diagnostics.
+ export ECLAIR_DIAGNOSTICS_OUTPUT="${ECLAIR_OUTPUT_DIR}/DIAGNOSTICS.txt"
+
+ PROJECT_ECD="${ECLAIR_OUTPUT_DIR}/PROJECT.ecd"
+}
+
+eclair_prepare() {
+ eclair_set_paths
+ mkdir -p "${ECLAIR_DATA_DIR}"
+}
+
+eclair_analyze() {
+ eclair_set_paths
+ (
+ # Run a build in the ECLAIR environment.
+ "${ECLAIR_BIN_DIR}/eclair_env" \
+ "-eval_file='${ECLAIR_CONFIG_DIR}/MISRA_C_2012_selection.ecl'" \
+ -- "$@"
+ )
+}
+
+# Create the project database.
+eclair_make_ecd() {
+ eclair_set_paths
+ find "${ECLAIR_DATA_DIR}" -maxdepth 1 -name "FRAME.*.ecb" \
+ | sort | xargs cat \
+ | "${ECLAIR_BIN_DIR}/eclair_report" \
+ "-create_db='${PROJECT_ECD}'" \
+ -load=/dev/stdin
+}
+
+eclair_make_report_self_contained() {
+ dir=$1
+ mkdir -p $dir/lib
+
+ cp -r /opt/bugseng/eclair/lib/html $dir/lib
+
+ ${_ECLAIR_UTILS_DIR}/relativize_urls.py $dir
+}
+
+eclair_make_reports() {
+ eclair_set_paths
+ ${ECLAIR_BIN_DIR}/eclair_report -db=${PROJECT_ECD} \
+ -summary_txt=${ECLAIR_OUTPUT_DIR}/../summary_txt \
+ -full_txt=${ECLAIR_OUTPUT_DIR}/../full_txt \
+ -reports1_html=strictness,${ECLAIR_OUTPUT_DIR}/../full_html/by_strictness/@TAG@.html \
+ -full_html=${ECLAIR_OUTPUT_DIR}/../full_html
+
+ # summary_txt contains just a single report file not present in full_txt, move it there and be done with it.
+ mv ${ECLAIR_OUTPUT_DIR}/../summary_txt/by_service.txt ${ECLAIR_OUTPUT_DIR}/../full_txt/
+ rm -rf ${ECLAIR_OUTPUT_DIR}/../summary_txt
+
+ eclair_make_report_self_contained ${ECLAIR_OUTPUT_DIR}/../full_html
+}
+
+eclair_compress_db() {
+ eclair_set_paths
+
+ # Compress database to take less disk space in Jenkins archive
+ xz ${PROJECT_ECD}
+}
+
+eclair_make_delta_report() {
+ base_dir=$1
+ target_dir=$2
+
+ diff -I '^Timestamp:' -x frames.txt -x files.txt -x explain.txt \
+ -ur ${WORKSPACE}/${base_dir}/summary_txt/ ${WORKSPACE}/${target_dir}/summary_txt/ > ${WORKSPACE}/${target_dir}/summary_txt.diff || true
+
+ ${ECLAIR_BIN_DIR}/eclair_report -diff_criteria=fingerprint -diff_full_txt=${base_dir}/out/PROJECT.ecd,${target_dir}/out/PROJECT.ecd
+ ls -l diff_output
+
+ ${ECLAIR_BIN_DIR}/eclair_report \
+ -db=${base_dir}/out/PROJECT.ecd \
+ -eval_file=${_ECLAIR_UTILS_DIR}/sel_tag_and_not_glob.ecl \
+ -sel_tag_and_not_glob=new_no_expl,diff,missing,service,B.EXPLAIN \
+ -full_html=resolved_issues_html
+ eclair_make_report_self_contained resolved_issues_html
+
+ ${ECLAIR_BIN_DIR}/eclair_report \
+ -db=${target_dir}/out/PROJECT.ecd \
+ -eval_file=${_ECLAIR_UTILS_DIR}/sel_tag_and_not_glob.ecl \
+ -sel_tag_and_not_glob=new_no_expl,diff,missing,service,B.EXPLAIN \
+ -full_html=new_issues_html
+ eclair_make_report_self_contained new_issues_html
+
+ xz ${base_dir}/out/PROJECT.ecd ${target_dir}/out/PROJECT.ecd
+}