Improve driver-only outcome-analysis script
Instead of having a list of test suites of interest, have a list of
suites to ignore and look at all the others.
In order for this to only yield interesting results, we need to tune the
reference configuration a bit, in order to exclude STREAM and ECB to
more closely match the driver-based configuration.
Signed-off-by: Manuel Pégourié-Gonnard <manuel.pegourie-gonnard@arm.com>
diff --git a/docs/architecture/psa-migration/outcome-analysis.sh b/docs/architecture/psa-migration/outcome-analysis.sh
index 67510c1..f3566b2 100755
--- a/docs/architecture/psa-migration/outcome-analysis.sh
+++ b/docs/architecture/psa-migration/outcome-analysis.sh
@@ -31,13 +31,23 @@
# A similar configuration to that of the component, except without drivers,
# for comparison.
reference_config () {
- scripts/config.py set MBEDTLS_USE_PSA_CRYPTO
- scripts/config.py unset MBEDTLS_PKCS1_V21
- scripts/config.py unset MBEDTLS_X509_RSASSA_PSS_SUPPORT
+ # start with full
+ scripts/config.py full
+ # use PSA config and disable driver-less algs as in the component
+ scripts/config.py set MBEDTLS_PSA_CRYPTO_CONFIG
+ scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_STREAM_CIPHER
+ scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_ECB_NO_PADDING
+ # disable options as in the component
+ # (no need to disable whole modules, we'll just skip their test suite)
scripts/config.py unset MBEDTLS_ECDSA_DETERMINISTIC
+ scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_DETERMINISTIC_ECDSA
}
-# Space-separated list of test suites of interest.
-SUITES="rsa pkcs1_v15 pk pkparse pkwrite"
+# Space-separated list of test suites to ignore:
+# if SSS is in that list, test_suite_SSS and test_suite_SSS.* are ignored.
+IGNORE="md mdx shax" # accelerated
+IGNORE="$IGNORE entropy hmac_drbg random" # disabled (ext. RNG)
+IGNORE="$IGNORE psa_crypto_init" # needs internal RNG
+IGNORE="$IGNORE hkdf" # disabled
# ----- END edit this -----
set -eu
@@ -85,6 +95,22 @@
# analysis
+populate_suites () {
+ SUITES=''
+ make generated_files >/dev/null
+ data_files=$(cd tests/suites && echo *.data)
+ for data in $data_files; do
+ suite=${data#test_suite_}
+ suite=${suite%.data}
+ suite_base=${suite%%.*}
+ case " $IGNORE " in
+ *" $suite_base "*) :;;
+ *) SUITES="$SUITES $suite";;
+ esac
+ done
+ make neat
+}
+
compare_suite () {
ref="outcome-$1.csv"
new="outcome-$2.csv"
@@ -98,19 +124,29 @@
nb_ref=$(wc -l <skipped-ref)
nb_new=$(wc -l <skipped-new)
- printf "%12s: total %3d; skipped %3d -> %3d\n" \
+ printf "%36s: total %4d; skipped %4d -> %4d\n" \
$suite $total $nb_ref $nb_new
- diff skipped-ref skipped-new | grep '^> ' || true
+ if diff skipped-ref skipped-new | grep '^> '; then
+ ret=1
+ else
+ ret=0
+ fi
rm skipped-ref skipped-new
+ return $ret
}
compare_builds () {
printf "\n*** Comparing $1 -> $2 ***\n"
+ failed=''
for suite in $SUITES; do
- compare_suite "$1" "$2" "$suite"
+ if compare_suite "$1" "$2" "$suite"; then :; else
+ failed="$failed $suite"
+ fi
done
+ printf "suites with less coverage: %s\n" "$failed"
}
+populate_suites
compare_builds before-default after-default
compare_builds before-full after-full
compare_builds reference drivers