ssl-opt.sh, compat.sh: Error out if not executing any tests
Alert if all tests are filtered out or skipped: that probably indicates a
test script that set up an unintended configuration or an overly strict
filter. You can pass `--min 0` to bypass this check. You can pass `--min`
with a larger value to require that many test cases to run.
Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
diff --git a/tests/compat.sh b/tests/compat.sh
index 20f2dbd..073258e 100755
--- a/tests/compat.sh
+++ b/tests/compat.sh
@@ -96,6 +96,7 @@
EXCLUDE='NULL\|ARIA\|CHACHA20_POLY1305'
VERBOSE=""
MEMCHECK=0
+MIN_TESTS=1
PRESERVE_LOGS=0
PEERS="OpenSSL$PEER_GNUTLS mbedTLS"
@@ -116,6 +117,7 @@
printf " -M|--memcheck\tCheck memory leaks and errors.\n"
printf " -v|--verbose\tSet verbose output.\n"
printf " --list-test-cases\tList all potential test cases (No Execution)\n"
+ printf " --min \tMinimum number of non-skipped tests (default 1)\n"
printf " --outcome-file\tFile where test outcomes are written\n"
printf " \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n"
printf " --preserve-logs\tPreserve logs of successful tests as well\n"
@@ -190,6 +192,9 @@
list_test_cases
exit $?
;;
+ --min)
+ shift; MIN_TESTS=$1
+ ;;
--outcome-file)
shift; MBEDTLS_TEST_OUTCOME_FILE=$1
;;
@@ -1238,6 +1243,16 @@
PASSED=$(( $TESTS - $FAILED ))
echo " ($PASSED / $TESTS tests ($SKIPPED skipped$MEMREPORT))"
+if [ $((TESTS - SKIPPED)) -lt $MIN_TESTS ]; then
+ cat <<EOF
+Error: Expected to run at least $MIN_TESTS, but only ran $((TESTS - SKIPPED)).
+Maybe a bad filter ('$FILTER' excluding '$EXCLUDE') or a bad configuration?
+EOF
+ if [ $FAILED -eq 0 ]; then
+ FAILED=1
+ fi
+fi
+
FAILED=$(( $FAILED + $SRVMEM ))
if [ $FAILED -gt 255 ]; then
# Clamp at 255 as caller gets exit code & 0xFF