Test outcome file support: ssl-opt.sh
If the environment variable MBEDTLS_TEST_OUTCOME_FILE is set, then for
each test case, write a line to the file with the given name, of the
form
PLATFORM;CONFIGURATION;ssl-opt;TEST CASE DESCRIPTION;PASS/FAIL/SKIP;CAUSE
PLATFORM and CONFIGURATION come from the environment variables
MBEDTLS_TEST_PLATFORM and MBEDTLS_TEST_CONFIGURATION. If these
variables are unset, the script uses some easily-calculated values.
diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh
index 47b6b80..914f4e1 100755
--- a/tests/ssl-opt.sh
+++ b/tests/ssl-opt.sh
@@ -25,9 +25,9 @@
# where it may output seemingly unlimited length error logs.
ulimit -f 20971520
-if cd $( dirname $0 ); then :; else
- echo "cd $( dirname $0 ) failed" >&2
- exit 1
+ORIGINAL_PWD=$PWD
+if ! cd "$(dirname "$0")"; then
+ exit 125
fi
# default values, can be overridden by the environment
@@ -39,6 +39,17 @@
: ${GNUTLS_SERV:=gnutls-serv}
: ${PERL:=perl}
+guess_config_name() {
+ if git diff --quiet ../include/mbedtls/config.h 2>/dev/null; then
+ echo "default"
+ else
+ echo "unknown"
+ fi
+}
+: ${MBEDTLS_TEST_OUTCOME_FILE=}
+: ${MBEDTLS_TEST_CONFIGURATION:="$(guess_config_name)"}
+: ${MBEDTLS_TEST_PLATFORM:="$(uname -s | tr -c \\n0-9A-Za-z _)-$(uname -m | tr -c \\n0-9A-Za-z _)"}
+
O_SRV="$OPENSSL_CMD s_server -www -cert data_files/server5.crt -key data_files/server5.key"
O_CLI="echo 'GET / HTTP/1.0' | $OPENSSL_CMD s_client"
G_SRV="$GNUTLS_SERV --x509certfile data_files/server5.crt --x509keyfile data_files/server5.key"
@@ -97,9 +108,11 @@
printf " -n|--number\tExecute only numbered test (comma-separated, e.g. '245,256')\n"
printf " -s|--show-numbers\tShow test numbers in front of test names\n"
printf " -p|--preserve-logs\tPreserve logs of successful tests as well\n"
- printf " --port\tTCP/UDP port (default: randomish 1xxxx)\n"
+ printf " --outcome-file\tFile where test outcomes are written\n"
+ printf " \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n"
+ printf " --port \tTCP/UDP port (default: randomish 1xxxx)\n"
printf " --proxy-port\tTCP/UDP proxy port (default: randomish 2xxxx)\n"
- printf " --seed\tInteger seed value to use for this test run\n"
+ printf " --seed \tInteger seed value to use for this test run\n"
}
get_options() {
@@ -146,6 +159,14 @@
done
}
+# Make the outcome file path relative to the original directory, not
+# to .../tests
+case "$MBEDTLS_TEST_OUTCOME_FILE" in
+ [!/]*)
+ MBEDTLS_TEST_OUTCOME_FILE="$ORIGINAL_PWD/$MBEDTLS_TEST_OUTCOME_FILE"
+ ;;
+esac
+
# Skip next test; use this macro to skip tests which are legitimate
# in theory and expected to be re-introduced at some point, but
# aren't expected to succeed at the moment due to problems outside
@@ -359,9 +380,22 @@
}
+# record_outcome <outcome> [<failure-reason>]
+# The test name must be in $NAME.
+record_outcome() {
+ echo "$1"
+ if [ -n "$MBEDTLS_TEST_OUTCOME_FILE" ]; then
+ printf '%s;%s;%s;%s;%s;%s\n' \
+ "$MBEDTLS_TEST_PLATFORM" "$MBEDTLS_TEST_CONFIGURATION" \
+ "ssl-opt" "$NAME" \
+ "$1" "${2-}" \
+ >>"$MBEDTLS_TEST_OUTCOME_FILE"
+ fi
+}
+
# fail <message>
fail() {
- echo "FAIL"
+ record_outcome "FAIL" "$1"
echo " ! $1"
mv $SRV_OUT o-srv-${TESTS}.log
@@ -539,6 +573,7 @@
if echo "$NAME" | grep "$FILTER" | grep -v "$EXCLUDE" >/dev/null; then :
else
SKIP_NEXT="NO"
+ # There was no request to run the test, so don't record its outcome.
return
fi
@@ -586,7 +621,7 @@
# should we skip?
if [ "X$SKIP_NEXT" = "XYES" ]; then
SKIP_NEXT="NO"
- echo "SKIP"
+ record_outcome "SKIP"
SKIPS=$(( $SKIPS + 1 ))
return
fi
@@ -772,7 +807,7 @@
fi
# if we're here, everything is ok
- echo "PASS"
+ record_outcome "PASS"
if [ "$PRESERVE_LOGS" -gt 0 ]; then
mv $SRV_OUT o-srv-${TESTS}.log
mv $CLI_OUT o-cli-${TESTS}.log