Merge remote-tracking branch 'origin/pr/2293' into mbedtls-2.16
* origin/pr/2293:
Declare test_suite_aes.ofb to CMake
Add a facility to skip running some test suites
run-test-suites: update the documentation
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index eae1f57..a8e7523 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -20,6 +20,13 @@
# on non-POSIX platforms.
add_definitions("-D_POSIX_C_SOURCE=200809L")
+# Test suites caught by SKIP_TEST_SUITES are built but not executed.
+# "foo" as a skip pattern skips "test_suite_foo" and "test_suite_foo.bar"
+# but not "test_suite_foobar".
+string(REGEX REPLACE "[ ,;]" "|" SKIP_TEST_SUITES_REGEX "${SKIP_TEST_SUITES}")
+string(REPLACE "." "\\." SKIP_TEST_SUITES_REGEX "${SKIP_TEST_SUITES_REGEX}")
+set(SKIP_TEST_SUITES_REGEX "^(${SKIP_TEST_SUITES_REGEX})(\$|\\.)")
+
function(add_test_suite suite_name)
if(ARGV1)
set(data_name ${ARGV1})
@@ -36,7 +43,11 @@
include_directories(${CMAKE_CURRENT_SOURCE_DIR})
add_executable(test_suite_${data_name} test_suite_${data_name}.c)
target_link_libraries(test_suite_${data_name} ${libs})
- add_test(${data_name}-suite test_suite_${data_name} --verbose)
+ if(${data_name} MATCHES ${SKIP_TEST_SUITES_REGEX})
+ message(STATUS "The test suite ${data_name} will not be executed.")
+ else()
+ add_test(${data_name}-suite test_suite_${data_name} --verbose)
+ endif()
endfunction(add_test_suite)
if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG)
@@ -52,6 +63,7 @@
add_test_suite(aes aes.ecb)
add_test_suite(aes aes.cbc)
add_test_suite(aes aes.cfb)
+add_test_suite(aes aes.ofb)
add_test_suite(aes aes.rest)
add_test_suite(aes aes.xts)
add_test_suite(arc4)
diff --git a/tests/Makefile b/tests/Makefile
index b6e49bf..4ef7417 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -113,8 +113,9 @@
endif
endif
+# Test suites caught by SKIP_TEST_SUITES are built but not executed.
check: $(BINARIES)
- perl scripts/run-test-suites.pl
+ perl scripts/run-test-suites.pl --skip=$(SKIP_TEST_SUITES)
test: check
diff --git a/tests/scripts/run-test-suites.pl b/tests/scripts/run-test-suites.pl
index d0d4046..1c9dc1d 100755
--- a/tests/scripts/run-test-suites.pl
+++ b/tests/scripts/run-test-suites.pl
@@ -4,19 +4,24 @@
#
# This file is part of mbed TLS (https://tls.mbed.org)
#
-# Copyright (c) 2015-2016, ARM Limited, All Rights Reserved
-#
-# Purpose
-#
-# Executes all the available test suites, and provides a basic summary of the
-# results.
-#
-# Usage: run-test-suites.pl [-v]
-#
-# Options :
-# -v|--verbose - Provide a pass/fail/skip breakdown per test suite and
-# in total
-#
+# Copyright (c) 2015-2018, ARM Limited, All Rights Reserved
+
+=head1 SYNOPSIS
+
+Execute all the test suites and print a summary of the results.
+
+ run-test-suites.pl [[-v|--verbose] [VERBOSITY]] [--skip=SUITE[...]]
+
+Options:
+
+ -v|--verbose Print detailed failure information.
+ -v 2|--verbose=2 Print detailed failure information and summary messages.
+ -v 3|--verbose=3 Print detailed information about every test case.
+ --skip=SUITE[,SUITE...]
+ Skip the specified SUITE(s). This option can be used
+ multiple times.
+
+=cut
use warnings;
use strict;
@@ -24,10 +29,15 @@
use utf8;
use open qw(:std utf8);
-use Getopt::Long;
+use Getopt::Long qw(:config auto_help gnu_compat);
+use Pod::Usage;
my $verbose = 0;
-GetOptions( "verbose|v:1" => \$verbose );
+my @skip_patterns = ();
+GetOptions(
+ 'skip=s' => \@skip_patterns,
+ 'verbose|v:1' => \$verbose,
+ ) or die;
# All test suites = executable files, excluding source files, debug
# and profiling information, etc. We can't just grep {! /\./} because
@@ -36,6 +46,17 @@
@suites = grep { !/\.c$/ && !/\.data$/ && -f } @suites;
die "$0: no test suite found\n" unless @suites;
+# "foo" as a skip pattern skips "test_suite_foo" and "test_suite_foo.bar"
+# but not "test_suite_foobar".
+my $skip_re =
+ ( '\Atest_suite_(' .
+ join('|', map {
+ s/[ ,;]/|/g; # allow any of " ,;|" as separators
+ s/\./\./g; # "." in the input means ".", not "any character"
+ $_
+ } @skip_patterns) .
+ ')(\z|\.)' );
+
# in case test suites are linked dynamically
$ENV{'LD_LIBRARY_PATH'} = '../library';
$ENV{'DYLD_LIBRARY_PATH'} = '../library';
@@ -45,6 +66,7 @@
my ($failed_suites, $total_tests_run, $failed, $suite_cases_passed,
$suite_cases_failed, $suite_cases_skipped, $total_cases_passed,
$total_cases_failed, $total_cases_skipped );
+my $suites_skipped = 0;
sub pad_print_center {
my( $width, $padchar, $string ) = @_;
@@ -55,6 +77,12 @@
for my $suite (@suites)
{
print "$suite ", "." x ( 72 - length($suite) - 2 - 4 ), " ";
+ if( $suite =~ /$skip_re/o ) {
+ print "SKIP\n";
+ ++$suites_skipped;
+ next;
+ }
+
my $command = "$prefix$suite";
if( $verbose ) {
$command .= ' -v';
@@ -101,7 +129,10 @@
print "-" x 72, "\n";
print $failed_suites ? "FAILED" : "PASSED";
-printf " (%d suites, %d tests run)\n", scalar @suites, $total_tests_run;
+printf( " (%d suites, %d tests run%s)\n",
+ scalar(@suites) - $suites_skipped,
+ $total_tests_run,
+ $suites_skipped ? ", $suites_skipped suites skipped" : "" );
if( $verbose > 1 ) {
print " test cases passed :", $total_cases_passed, "\n";
@@ -111,8 +142,11 @@
"\n";
print " of available tests :",
( $total_cases_passed + $total_cases_failed + $total_cases_skipped ),
- "\n"
+ "\n";
+ if( $suites_skipped != 0 ) {
+ print "Note: $suites_skipped suites were skipped.\n";
}
+}
exit( $failed_suites ? 1 : 0 );