Updated test script all.sh
Changes to increase the release test coverage and also allow testing with
two different versions of OpenSSL and GNUTLS for legacy features.
diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh
index 5ef74cd..a2eff08 100755
--- a/tests/scripts/all.sh
+++ b/tests/scripts/all.sh
@@ -1,19 +1,29 @@
#!/bin/sh
-# Run all available tests (mostly).
+# all.sh
#
-# Warning: includes various build modes, so it will mess with the current
-# CMake configuration. After this script is run, the CMake cache is lost and
-# CMake is not initialised any more!
+# This file is part of mbed TLS (https://tls.mbed.org)
#
-# Assumes gcc and clang (recent enough for using ASan with gcc and MemSan with
-# clang, or valgrind) are available, as well as cmake and a "good" find.
+# Copyright (c) 2014-2016, ARM Limited, All Rights Reserved
+#
+# Purpose
+#
+# To run all tests possible or available on the platform.
+#
+# Warning: the test is destructive. It includes various build modes and
+# configurations, and can and will arbitrarily change the current CMake
+# configuration. After this script has been run, the CMake cache will be lost
+# and CMake will no longer be initialised.
+#
+# The script assumes the presence of gcc and clang (recent enough for using
+# ASan with gcc and MemSan with clang, or valgrind) are available, as well as
+# cmake and a "good" find.
-# Abort on errors (and uninitiliased variables)
+# Abort on errors (and uninitialised variables)
set -eu
if [ -d library -a -d include -a -d tests ]; then :; else
- echo "Must be run from mbed TLS root" >&2
+ err_msg "Must be run from mbed TLS root"
exit 1
fi
@@ -21,20 +31,34 @@
CONFIG_BAK="$CONFIG_H.bak"
MEMORY=0
+FORCE=0
+RELEASE=0
-while [ $# -gt 0 ]; do
- case "$1" in
- -m*)
- MEMORY=${1#-m}
- ;;
- *)
- echo "Unknown argument: '$1'" >&2
- echo "Use the source, Luke!" >&2
- exit 1
- ;;
- esac
- shift
-done
+# Default commands, can be overriden by the environment
+: ${OPENSSL:="openssl"}
+: ${OPENSSL_LEGACY:="$OPENSSL"}
+: ${GNUTLS_CLI:="gnutls-cli"}
+: ${GNUTLS_SERV:="gnutls-serv"}
+: ${GNUTLS_LEGACY_CLI:="$GNUTLS_CLI"}
+: ${GNUTLS_LEGACY_SERV:="$GNUTLS_SERV"}
+: ${OUT_OF_SOURCE_DIR:=./mbedtls_out_of_source_build}
+
+usage()
+{
+ printf "Usage: $0\n"
+ printf " -h|--help\t\tPrint this help.\n"
+ printf " -m|--memory\t\tAdditional optional memory tests.\n"
+ printf " -f|--force\t\tForce the tests to overwrite any modified files.\n"
+ printf " -s|--seed\t\tInteger seed value to use for this test run.\n"
+ printf " -r|--release-test\t\tRun this script in release mode. This fixes the seed value to 1.\n"
+ printf " --out-of-source-dir=<path>\t\tDirectory used for CMake out-of-source build tests."
+ printf " --openssl=<OpenSSL_path>\t\tPath to OpenSSL executable to use for most tests.\n"
+ printf " --openssl-legacy=<OpenSSL_path>\t\tPath to OpenSSL executable to use for legacy tests e.g. SSLv3.\n"
+ printf " --gnutls-cli=<GnuTLS_cli_path>\t\tPath to GnuTLS client executable to use for most tests.\n"
+ printf " --gnutls-serv=<GnuTLS_serv_path>\t\tPath to GnuTLS server executable to use for most tests.\n"
+ printf " --gnutls-legacy-cli=<GnuTLS_cli_path>\t\tPath to GnuTLS client executable to use for legacy tests.\n"
+ printf " --gnutls-legacy-serv=<GnuTLS_serv_path>\t\tPath to GnuTLS server executable to use for legacy tests.\n"
+}
# remove built files as well as the cmake cache/config
cleanup()
@@ -62,6 +86,134 @@
echo "******************************************************************"
}
+err_msg()
+{
+ echo "$1" >&2
+}
+
+check_tools()
+{
+ for TOOL in "$@"; do
+ if ! `hash "$TOOL" >/dev/null 2>&1`; then
+ err_msg "$TOOL not found!"
+ exit 1
+ fi
+ done
+}
+
+while [ $# -gt 0 ]; do
+ case "$1" in
+ --memory|-m*)
+ MEMORY=${1#-m}
+ ;;
+ --force|-f)
+ FORCE=1
+ ;;
+ --seed|-s)
+ shift
+ SEED="$1"
+ ;;
+ --release-test|-r)
+ RELEASE=1
+ ;;
+ --out-of-source-dir)
+ shift
+ OUT_OF_SOURCE_DIR="$1"
+ ;;
+ --openssl)
+ shift
+ OPENSSL="$1"
+ ;;
+ --openssl-legacy)
+ shift
+ OPENSSL_LEGACY="$1"
+ ;;
+ --gnutls-cli)
+ shift
+ GNUTLS_CLI="$1"
+ ;;
+ --gnutls-serv)
+ shift
+ GNUTLS_SERV="$1"
+ ;;
+ --gnutls-legacy-cli)
+ shift
+ GNUTLS_LEGACY_CLI="$1"
+ ;;
+ --gnutls-legacy-serv)
+ shift
+ GNUTLS_LEGACY_SERV="$1"
+ ;;
+ --help|-h|*)
+ usage
+ exit 1
+ ;;
+ esac
+ shift
+done
+
+if [ $FORCE -eq 1 ]; then
+ rm -rf yotta/module "$OUT_OF_SOURCE_DIR"
+ git checkout-index -f -q $CONFIG_H
+ cleanup
+else
+
+ if [ -d yotta/module ]; then
+ err_msg "Warning - there is an existing yotta module in the directory 'yotta/module'"
+ echo "You can either delete your work and retry, or force the test to overwrite the"
+ echo "test by rerunning the script as: $0 --force"
+ exit 1
+ fi
+
+ if [ -d "$OUT_OF_SOURCE_DIR" ]; then
+ echo "Warning - there is an existing directory at '$OUT_OF_SOURCE_DIR'" >&2
+ echo "You can either delete this directory manually, or force the test by rerunning"
+ echo "the script as: $0 --force --out-of-source-dir $OUT_OF_SOURCE_DIR"
+ exit 1
+ fi
+
+ if ! git diff-files --quiet include/mbedtls/config.h; then
+ echo $?
+ err_msg "Warning - the configuration file 'include/mbedtls/config.h' has been edited. "
+ echo "You can either delete or preserve your work, or force the test by rerunning the"
+ echo "script as: $0 --force"
+ exit 1
+ fi
+fi
+
+if [ $RELEASE -eq 1 ]; then
+ # Fix the seed value to 1 to ensure that the tests are deterministic.
+ SEED=1
+fi
+
+msg "info: $0 configuration"
+echo "MEMORY: $MEMORY"
+echo "FORCE: $FORCE"
+echo "SEED: ${SEED-"UNSET"}"
+echo "OPENSSL: $OPENSSL"
+echo "OPENSSL_LEGACY: $OPENSSL_LEGACY"
+echo "GNUTLS_CLI: $GNUTLS_CLI"
+echo "GNUTLS_SERV: $GNUTLS_SERV"
+echo "GNUTLS_LEGACY_CLI: $GNUTLS_LEGACY_CLI"
+echo "GNUTLS_LEGACY_SERV: $GNUTLS_LEGACY_SERV"
+
+# To avoid setting OpenSSL and GnuTLS for each call to compat.sh and ssl-opt.sh
+# we just export the variables they require
+export OPENSSL_CMD="$OPENSSL"
+export GNUTLS_CLI="$GNUTLS_CLI"
+export GNUTLS_SERV="$GNUTLS_SERV"
+
+# Avoid passing --seed flag in every call to ssl-opt.sh
+[ ! -z ${SEED+set} ] && export SEED
+
+# Make sure the tools we need are available.
+check_tools "$OPENSSL" "$OPENSSL_LEGACY" "$GNUTLS_CLI" "$GNUTLS_SERV" \
+ "$GNUTLS_LEGACY_CLI" "$GNUTLS_LEGACY_SERV" "doxygen" "dot" \
+ "arm-none-eabi-gcc" "armcc"
+
+#
+# Test Suites to be executed
+#
# The test ordering tries to optimize for the following criteria:
# 1. Catch possible problems early, by running first tests that run quickly
# and/or are more likely to fail than others (eg I use Clang most of the
@@ -93,7 +245,7 @@
CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan .
make
-msg "test: main suites and selftest (ASan build)" # ~ 50s
+msg "test: main suites (inc. selftests) (ASan build)" # ~ 50s
make test
programs/test/selftest
@@ -103,8 +255,6 @@
msg "test/build: ref-configs (ASan build)" # ~ 6 min 20s
tests/scripts/test-ref-configs.pl
-# Most frequent issues are likely to be caught at this point
-
msg "build: with ASan (rebuild after ref-configs)" # ~ 1 min
make
@@ -118,12 +268,13 @@
CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan .
make
-msg "test: SSLv3 - main suites and selftest (ASan build)" # ~ 50s
+msg "test: SSLv3 - main suites (inc. selftests) (ASan build)" # ~ 50s
make test
programs/test/selftest
msg "build: SSLv3 - compat.sh (ASan build)" # ~ 6 min
-tests/compat.sh -m 'ssl3 tls1 tls1_1 tls1_2 dtls1 dtls1_2'
+tests/compat.sh -m 'tls1 tls1_1 tls1_2 dtls1 dtls1_2'
+OPENSSL_CMD="$OPENSSL_LEGACY" tests/compat.sh -m 'ssl3'
msg "build: SSLv3 - ssl-opt.sh (ASan build)" # ~ 6 min
tests/ssl-opt.sh
@@ -143,7 +294,7 @@
tests/ssl-opt.sh -f Default
msg "test: compat.sh RC4, DES & NULL (full config)" # ~ 2 min
-tests/compat.sh -e '3DES\|DES-CBC3' -f 'NULL\|DES\|RC4\|ARCFOUR'
+OPENSSL_CMD="$OPENSSL_LEGACY" GNUTLS_CLI="$GNUTLS_LEGACY_CLI" GNUTLS_SERV="$GNUTLS_LEGACY_SERV" tests/compat.sh -e '3DES\|DES-CBC3' -f 'NULL\|DES\|RC4\|ARCFOUR'
msg "test/build: curves.pl (gcc)" # ~ 4 min
cleanup
@@ -217,7 +368,6 @@
CC=gcc CFLAGS='-Werror -m32' make
fi # x86_64
-if which arm-none-eabi-gcc >/dev/null; then
msg "build: arm-none-eabi-gcc, make" # ~ 10s
cleanup
cp "$CONFIG_H" "$CONFIG_BAK"
@@ -225,6 +375,7 @@
scripts/config.pl unset MBEDTLS_NET_C
scripts/config.pl unset MBEDTLS_TIMING_C
scripts/config.pl unset MBEDTLS_FS_IO
+scripts/config.pl set MBEDTLS_NO_PLATFORM_ENTROPY
# following things are not in the default config
scripts/config.pl unset MBEDTLS_HAVEGE_C # depends on timing.c
scripts/config.pl unset MBEDTLS_THREADING_PTHREAD
@@ -232,9 +383,7 @@
scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # execinfo.h
scripts/config.pl unset MBEDTLS_MEMORY_BUFFER_ALLOC_C # calls exit
CC=arm-none-eabi-gcc AR=arm-none-eabi-ar LD=arm-none-eabi-ld CFLAGS=-Werror make lib
-fi # arm-gcc
-if which armcc >/dev/null && armcc --help >/dev/null 2>&1; then
msg "build: armcc, make"
cleanup
cp "$CONFIG_H" "$CONFIG_BAK"
@@ -244,6 +393,7 @@
scripts/config.pl unset MBEDTLS_FS_IO
scripts/config.pl unset MBEDTLS_HAVE_TIME
scripts/config.pl unset MBEDTLS_HAVE_TIME_DATE
+scripts/config.pl set MBEDTLS_NO_PLATFORM_ENTROPY
# following things are not in the default config
scripts/config.pl unset MBEDTLS_DEPRECATED_WARNING
scripts/config.pl unset MBEDTLS_HAVEGE_C # depends on timing.c
@@ -251,13 +401,7 @@
scripts/config.pl unset MBEDTLS_THREADING_C
scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # execinfo.h
scripts/config.pl unset MBEDTLS_MEMORY_BUFFER_ALLOC_C # calls exit
-CC=armcc AR=armar WARNING_CFLAGS= make lib 2> armcc.stderr
-if [ -s armcc.stderr ]; then
- cat armcc.stderr
- exit 1;
-fi
-rm armcc.stderr
-fi # armcc
+CC=armcc AR=armar WARNING_CFLAGS= make lib
if which i686-w64-mingw32-gcc >/dev/null; then
msg "build: cross-mingw64, make" # ~ 30s
@@ -317,6 +461,19 @@
fi # MemSan
+msg "build: cmake 'out-of-source' build"
+cleanup
+MBEDTLS_ROOT_DIR="$PWD"
+mkdir "$OUT_OF_SOURCE_DIR"
+cd "$OUT_OF_SOURCE_DIR"
+cmake "$MBEDTLS_ROOT_DIR"
+make
+
+msg "test: cmake 'out-of-source' build"
+make test
+cd "$MBEDTLS_ROOT_DIR"
+rm -rf "$OUT_OF_SOURCE_DIR"
+
msg "Done, cleaning up"
cleanup