Add ssl-opt.sh support to outcome-analysis.sh
But make it optional as it makes things much slower.
Signed-off-by: Manuel Pégourié-Gonnard <manuel.pegourie-gonnard@arm.com>
diff --git a/docs/architecture/psa-migration/outcome-analysis.sh b/docs/architecture/psa-migration/outcome-analysis.sh
index f7ca01b..6603c3a 100755
--- a/docs/architecture/psa-migration/outcome-analysis.sh
+++ b/docs/architecture/psa-migration/outcome-analysis.sh
@@ -8,6 +8,11 @@
# implementation", namely: the sets of tests skipped in the default config and
# the full config must be the same before and after the PR.
#
+# USAGE:
+# - First, commit any uncommited changes. (Also, see warning below.)
+# - including ssl-opt.sh: docs/architecture/psa-migration/outcome-analysis.sh
+# - or: SKIP_SSL_OPT=1 docs/architecture/psa-migration/outcome-analysis.sh
+#
# WARNING: this script checks out a commit other than the head of the current
# branch; it checks out the current branch again when running successfully,
# but while the script is running, or if it terminates early in error, you
@@ -18,6 +23,8 @@
set -eu
+: ${SKIP_SSL_OPT:=0}
+
cleanup() {
make clean
git checkout -- include/mbedtls/mbedtls_config.h include/psa/crypto_config.h
@@ -26,7 +33,14 @@
record() {
export MBEDTLS_TEST_OUTCOME_FILE="$PWD/outcome-$1.csv"
rm -f $MBEDTLS_TEST_OUTCOME_FILE
+
make check
+
+ if [ $SKIP_SSL_OPT -eq 0 ]; then
+ make -C programs ssl/ssl_server2 ssl/ssl_client2 \
+ test/udp_proxy test/query_compile_time_config
+ tests/ssl-opt.sh
+ fi
}
# save current HEAD
@@ -35,21 +49,27 @@
# get the numbers before this PR for default and full
cleanup
git checkout $(git merge-base HEAD development)
+
record "before-default"
cleanup
+
scripts/config.py full
record "before-full"
# get the numbers now for default and full
cleanup
git checkout $HEAD
+
record "after-default"
cleanup
+
scripts/config.py full
record "after-full"
+cleanup
+
# analysis
populate_suites () {
@@ -57,11 +77,19 @@
make generated_files >/dev/null
data_files=$(cd tests/suites && echo *.data)
for data in $data_files; do
- suite=${data#test_suite_}
- suite=${suite%.data}
+ suite=${data%.data}
SUITES="$SUITES $suite"
done
make neat
+
+ if [ $SKIP_SSL_OPT -eq 0 ]; then
+ SUITES="$SUITES ssl-opt"
+ extra_files=$(cd tests/opt-testcases && echo *.sh)
+ for extra in $extra_files; do
+ suite=${extra%.sh}
+ SUITES="$SUITES $suite"
+ done
+ fi
}
compare_suite () {
@@ -69,7 +97,7 @@
new="outcome-$2.csv"
suite="$3"
- pattern_suite=";test_suite_$suite;"
+ pattern_suite=";$suite;"
total=$(grep -c "$pattern_suite" "$ref")
sed_cmd="s/^.*$pattern_suite\(.*\);SKIP.*/\1/p"
sed -n "$sed_cmd" "$ref" > skipped-ref
@@ -77,8 +105,9 @@
nb_ref=$(wc -l <skipped-ref)
nb_new=$(wc -l <skipped-new)
- printf "%36s: total %4d; skipped %4d -> %4d\n" \
- $suite $total $nb_ref $nb_new
+ name=${suite#test_suite_}
+ printf "%40s: total %4d; skipped %4d -> %4d\n" \
+ $name $total $nb_ref $nb_new
if diff skipped-ref skipped-new | grep '^> '; then
ret=1
else