Merge branch 'development' into pr3431
diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh
index d3ad4d9..a20bbde 100755
--- a/tests/scripts/all.sh
+++ b/tests/scripts/all.sh
@@ -2066,6 +2066,48 @@
make test
}
+# Auxiliary function to build config for hashes with and without drivers
+config_psa_crypto_hash_use_psa () {
+ DRIVER_ONLY="$1"
+ # start with config full for maximum coverage (also enables USE_PSA)
+ scripts/config.py full
+ # enable support for drivers and configuring PSA-only algorithms
+ scripts/config.py set MBEDTLS_PSA_CRYPTO_CONFIG
+ scripts/config.py set MBEDTLS_PSA_CRYPTO_DRIVERS
+ if [ "$DRIVER_ONLY" -eq 1 ]; then
+ # disable the built-in implementation of hashes
+ scripts/config.py unset MBEDTLS_MD5_C
+ scripts/config.py unset MBEDTLS_RIPEMD160_C
+ scripts/config.py unset MBEDTLS_SHA1_C
+ scripts/config.py unset MBEDTLS_SHA224_C
+ scripts/config.py unset MBEDTLS_SHA256_C # see external RNG below
+ scripts/config.py unset MBEDTLS_SHA256_USE_A64_CRYPTO_IF_PRESENT
+ scripts/config.py unset MBEDTLS_SHA384_C
+ scripts/config.py unset MBEDTLS_SHA512_C
+ scripts/config.py unset MBEDTLS_SHA512_USE_A64_CRYPTO_IF_PRESENT
+ fi
+ # Use an external RNG as currently internal RNGs depend on entropy.c
+ # which in turn hard-depends on SHA256_C (or SHA512_C).
+ # See component_test_psa_external_rng_no_drbg_use_psa.
+ scripts/config.py set MBEDTLS_PSA_CRYPTO_EXTERNAL_RNG
+ scripts/config.py unset MBEDTLS_ENTROPY_C
+ scripts/config.py unset MBEDTLS_ENTROPY_NV_SEED # depends on ENTROPY_C
+ scripts/config.py unset MBEDTLS_PLATFORM_NV_SEED_ALT # depends on former
+ # Also unset MD_C and things that depend on it;
+ # see component_test_crypto_full_no_md.
+ if [ "$DRIVER_ONLY" -eq 1 ]; then
+ scripts/config.py unset MBEDTLS_MD_C
+ fi
+ scripts/config.py unset MBEDTLS_HKDF_C # has independent PSA implementation
+ scripts/config.py unset MBEDTLS_HMAC_DRBG_C
+ scripts/config.py unset MBEDTLS_PKCS7_C
+ scripts/config.py unset MBEDTLS_ECDSA_DETERMINISTIC
+ scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_DETERMINISTIC_ECDSA
+}
+
+# Note that component_test_psa_crypto_config_reference_hash_use_psa
+# is related to this component and both components need to be kept in sync.
+# For details please see comments for component_test_psa_crypto_config_reference_hash_use_psa.
component_test_psa_crypto_config_accel_hash_use_psa () {
msg "test: MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA"
@@ -2078,36 +2120,7 @@
loc_accel_flags=$( echo "$loc_accel_list" | sed 's/[^ ]* */-DLIBTESTDRIVER1_MBEDTLS_PSA_ACCEL_&/g' )
make -C tests libtestdriver1.a CFLAGS="$ASAN_CFLAGS $loc_accel_flags" LDFLAGS="$ASAN_CFLAGS"
- # start with config full for maximum coverage (also enables USE_PSA)
- scripts/config.py full
- # enable support for drivers and configuring PSA-only algorithms
- scripts/config.py set MBEDTLS_PSA_CRYPTO_DRIVERS
- scripts/config.py set MBEDTLS_PSA_CRYPTO_CONFIG
- # disable the built-in implementation of hashes
- scripts/config.py unset MBEDTLS_MD5_C
- scripts/config.py unset MBEDTLS_RIPEMD160_C
- scripts/config.py unset MBEDTLS_SHA1_C
- scripts/config.py unset MBEDTLS_SHA224_C
- scripts/config.py unset MBEDTLS_SHA256_C # see external RNG below
- scripts/config.py unset MBEDTLS_SHA256_USE_A64_CRYPTO_IF_PRESENT
- scripts/config.py unset MBEDTLS_SHA384_C
- scripts/config.py unset MBEDTLS_SHA512_C
- scripts/config.py unset MBEDTLS_SHA512_USE_A64_CRYPTO_IF_PRESENT
- # Use an external RNG as currently internal RNGs depend on entropy.c
- # which in turn hard-depends on SHA256_C (or SHA512_C).
- # See component_test_psa_external_rng_no_drbg_use_psa.
- scripts/config.py set MBEDTLS_PSA_CRYPTO_EXTERNAL_RNG
- scripts/config.py unset MBEDTLS_ENTROPY_C
- scripts/config.py unset MBEDTLS_ENTROPY_NV_SEED # depends on ENTROPY_C
- scripts/config.py unset MBEDTLS_PLATFORM_NV_SEED_ALT # depends on former
- # Also unset MD_C and things that depend on it;
- # see component_test_crypto_full_no_md.
- scripts/config.py unset MBEDTLS_MD_C
- scripts/config.py unset MBEDTLS_HKDF_C # has independent PSA implementation
- scripts/config.py unset MBEDTLS_HMAC_DRBG_C
- scripts/config.py unset MBEDTLS_PKCS7_C
- scripts/config.py unset MBEDTLS_ECDSA_DETERMINISTIC
- scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_DETERMINISTIC_ECDSA
+ config_psa_crypto_hash_use_psa 1
loc_accel_flags="$loc_accel_flags $( echo "$loc_accel_list" | sed 's/[^ ]* */-DMBEDTLS_PSA_ACCEL_&/g' )"
make CFLAGS="$ASAN_CFLAGS -Werror -I../tests/include -I../tests -I../../tests -DPSA_CRYPTO_DRIVER_TEST -DMBEDTLS_TEST_LIBTESTDRIVER1 $loc_accel_flags" LDFLAGS="-ltestdriver1 $ASAN_CFLAGS" all
@@ -2124,16 +2137,32 @@
msg "test: MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA"
make test
- # hidden option: when running outcome-analysis.sh, we can skip this
- if [ "${SKIP_SSL_OPT_COMPAT_SH-unset}" = "unset" ]; then
- msg "test: ssl-opt.sh, MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA"
- tests/ssl-opt.sh
+ msg "test: ssl-opt.sh, MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA"
+ tests/ssl-opt.sh
- msg "test: compat.sh, MBEDTLS_PSA_CRYPTO_CONFIG with accelerated hash and USE_PSA"
- tests/compat.sh
- else
- echo "skip ssl-opt.sh and compat.sh"
- fi
+ msg "test: compat.sh, MBEDTLS_PSA_CRYPTO_CONFIG without accelerated hash and USE_PSA"
+ tests/compat.sh
+}
+
+# This component provides reference configuration for test_psa_crypto_config_accel_hash_use_psa
+# without accelerated hash. The outcome from both components are used by the analyze_outcomes.py
+# script to find regression in test coverage when accelerated hash is used (tests and ssl-opt).
+# Both components need to be kept in sync.
+component_test_psa_crypto_config_reference_hash_use_psa() {
+ msg "test: MBEDTLS_PSA_CRYPTO_CONFIG without accelerated hash and USE_PSA"
+
+ scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_STREAM_CIPHER
+ scripts/config.py -f include/psa/crypto_config.h unset PSA_WANT_ALG_ECB_NO_PADDING
+
+ config_psa_crypto_hash_use_psa 0
+
+ make
+
+ msg "test: MBEDTLS_PSA_CRYPTO_CONFIG without accelerated hash and USE_PSA"
+ make test
+
+ msg "test: ssl-opt.sh, MBEDTLS_PSA_CRYPTO_CONFIG without accelerated hash and USE_PSA"
+ tests/ssl-opt.sh
}
component_test_psa_crypto_config_accel_cipher () {
@@ -3256,6 +3285,7 @@
msg "build: TLS 1.3 only from default, only ephemeral key exchange mode"
scripts/config.py unset MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_ENABLED
scripts/config.py unset MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_EPHEMERAL_ENABLED
+ scripts/config.py unset MBEDTLS_SSL_EARLY_DATA
make CFLAGS="'-DMBEDTLS_USER_CONFIG_FILE=\"../tests/configs/tls13-only.h\"'"
msg "test_suite_ssl: TLS 1.3 only, only ephemeral key exchange mode"
diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py
index d06a059..bb44396 100755
--- a/tests/scripts/analyze_outcomes.py
+++ b/tests/scripts/analyze_outcomes.py
@@ -9,6 +9,7 @@
import argparse
import sys
import traceback
+import re
import check_test_cases
@@ -60,6 +61,37 @@
# fixed this branch to have full coverage of test cases.
results.warning('Test case not executed: {}', key)
+def analyze_driver_vs_reference(outcomes, component_ref, component_driver, ignored_tests):
+ """Check that all tests executed in the reference component are also
+ executed in the corresponding driver component.
+ Skip test suites provided in ignored_tests list.
+ """
+ available = check_test_cases.collect_available_test_cases()
+ result = True
+
+ for key in available:
+ # Skip ignored test suites
+ test_suite = key.split(';')[0] # retrieve test suit name
+ test_suite = test_suite.split('.')[0] # retrieve main part of test suit name
+ if test_suite in ignored_tests:
+ continue
+ # Continue if test was not executed by any component
+ hits = outcomes[key].hits() if key in outcomes else 0
+ if hits == 0:
+ continue
+ # Search for tests that run in reference component and not in driver component
+ driver_test_passed = False
+ reference_test_passed = False
+ for entry in outcomes[key].successes:
+ if component_driver in entry:
+ driver_test_passed = True
+ if component_ref in entry:
+ reference_test_passed = True
+ if(driver_test_passed is False and reference_test_passed is True):
+ print('{}: driver: skipped/failed; reference: passed'.format(key))
+ result = False
+ return result
+
def analyze_outcomes(outcomes):
"""Run all analyses on the given outcome collection."""
results = Results()
@@ -87,20 +119,75 @@
outcomes[key].failures.append(setup)
return outcomes
-def analyze_outcome_file(outcome_file):
- """Analyze the given outcome file."""
+def do_analyze_coverage(outcome_file, args):
+ """Perform coverage analysis."""
+ del args # unused
outcomes = read_outcome_file(outcome_file)
- return analyze_outcomes(outcomes)
+ results = analyze_outcomes(outcomes)
+ return results.error_count == 0
+
+def do_analyze_driver_vs_reference(outcome_file, args):
+ """Perform driver vs reference analyze."""
+ ignored_tests = ['test_suite_' + x for x in args['ignored_suites']]
+
+ outcomes = read_outcome_file(outcome_file)
+ return analyze_driver_vs_reference(outcomes, args['component_ref'],
+ args['component_driver'], ignored_tests)
+
+# List of tasks with a function that can handle this task and additional arguments if required
+TASKS = {
+ 'analyze_coverage': {
+ 'test_function': do_analyze_coverage,
+ 'args': {}},
+ 'analyze_driver_vs_reference_hash': {
+ 'test_function': do_analyze_driver_vs_reference,
+ 'args': {
+ 'component_ref': 'test_psa_crypto_config_reference_hash_use_psa',
+ 'component_driver': 'test_psa_crypto_config_accel_hash_use_psa',
+ 'ignored_suites': ['shax', 'mdx', # the software implementations that are being excluded
+ 'md', # the legacy abstraction layer that's being excluded
+ ]}}
+}
def main():
try:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('outcomes', metavar='OUTCOMES.CSV',
help='Outcome file to analyze')
+ parser.add_argument('task', default='all', nargs='?',
+ help='Analysis to be done. By default, run all tasks. '
+ 'With one or more TASK, run only those. '
+ 'TASK can be the name of a single task or '
+ 'comma/space-separated list of tasks. ')
+ parser.add_argument('--list', action='store_true',
+ help='List all available tasks and exit.')
options = parser.parse_args()
- results = analyze_outcome_file(options.outcomes)
- if results.error_count > 0:
+
+ if options.list:
+ for task in TASKS:
+ print(task)
+ sys.exit(0)
+
+ result = True
+
+ if options.task == 'all':
+ tasks = TASKS.keys()
+ else:
+ tasks = re.split(r'[, ]+', options.task)
+
+ for task in tasks:
+ if task not in TASKS:
+ print('Error: invalid task: {}'.format(task))
+ sys.exit(1)
+
+ for task in TASKS:
+ if task in tasks:
+ if not TASKS[task]['test_function'](options.outcomes, TASKS[task]['args']):
+ result = False
+
+ if result is False:
sys.exit(1)
+ print("SUCCESS :-)")
except Exception: # pylint: disable=broad-except
# Print the backtrace and exit explicitly with our chosen status.
traceback.print_exc()
diff --git a/tests/scripts/generate_bignum_tests.py b/tests/scripts/generate_bignum_tests.py
index 4ac9210..eee2f65 100755
--- a/tests/scripts/generate_bignum_tests.py
+++ b/tests/scripts/generate_bignum_tests.py
@@ -66,7 +66,7 @@
# Import modules containing additional test classes
# Test function classes in these modules will be registered by
# the framework
-from mbedtls_dev import bignum_core # pylint: disable=unused-import
+from mbedtls_dev import bignum_core, bignum_mod_raw # pylint: disable=unused-import
class BignumTarget(test_data_generation.BaseTarget, metaclass=ABCMeta):
#pylint: disable=abstract-method
@@ -78,11 +78,17 @@
#pylint: disable=abstract-method
"""Common features for bignum operations in legacy tests."""
input_values = [
- "", "0", "7b", "-7b",
+ "", "0", "-", "-0",
+ "7b", "-7b",
"0000000000000000123", "-0000000000000000123",
"1230000000000000000", "-1230000000000000000"
]
+ def description_suffix(self) -> str:
+ #pylint: disable=no-self-use # derived classes need self
+ """Text to add at the end of the test case description."""
+ return ""
+
def description(self) -> str:
"""Generate a description for the test case.
@@ -96,6 +102,9 @@
self.symbol,
self.value_description(self.arg_b)
)
+ description_suffix = self.description_suffix()
+ if description_suffix:
+ self.case_description += " " + description_suffix
return super().description()
@staticmethod
@@ -107,6 +116,8 @@
"""
if val == "":
return "0 (null)"
+ if val == "-":
+ return "negative 0 (null)"
if val == "0":
return "0 (1 limb)"
@@ -171,9 +182,21 @@
]
)
- def result(self) -> List[str]:
- return [bignum_common.quote_str("{:x}").format(self.int_a + self.int_b)]
+ def __init__(self, val_a: str, val_b: str) -> None:
+ super().__init__(val_a, val_b)
+ self._result = self.int_a + self.int_b
+ def description_suffix(self) -> str:
+ if (self.int_a >= 0 and self.int_b >= 0):
+ return "" # obviously positive result or 0
+ if (self.int_a <= 0 and self.int_b <= 0):
+ return "" # obviously negative result or 0
+ # The sign of the result is not obvious, so indicate it
+ return ", result{}0".format('>' if self._result > 0 else
+ '<' if self._result < 0 else '=')
+
+ def result(self) -> List[str]:
+ return [bignum_common.quote_str("{:x}".format(self._result))]
if __name__ == '__main__':
# Use the section of the docstring relevant to the CLI as description
diff --git a/tests/scripts/generate_test_code.py b/tests/scripts/generate_test_code.py
index f5750aa..938f24c 100755
--- a/tests/scripts/generate_test_code.py
+++ b/tests/scripts/generate_test_code.py
@@ -126,33 +126,39 @@
This script replaces following fields in the template and generates
the test source file:
-$test_common_helpers <-- All common code from helpers.function
- is substituted here.
-$functions_code <-- Test functions are substituted here
- from the input test_suit_xyz.function
- file. C preprocessor checks are generated
- for the build dependencies specified
- in the input file. This script also
- generates wrappers for the test
- functions with code to expand the
- string parameters read from the data
- file.
-$expression_code <-- This script enumerates the
- expressions in the .data file and
- generates code to handle enumerated
- expression Ids and return the values.
-$dep_check_code <-- This script enumerates all
- build dependencies and generate
- code to handle enumerated build
- dependency Id and return status: if
- the dependency is defined or not.
-$dispatch_code <-- This script enumerates the functions
- specified in the input test data file
- and generates the initializer for the
- function table in the template
- file.
-$platform_code <-- Platform specific setup and test
- dispatch code.
+__MBEDTLS_TEST_TEMPLATE__TEST_COMMON_HELPERS
+ All common code from helpers.function
+ is substituted here.
+__MBEDTLS_TEST_TEMPLATE__FUNCTIONS_CODE
+ Test functions are substituted here
+ from the input test_suit_xyz.function
+ file. C preprocessor checks are generated
+ for the build dependencies specified
+ in the input file. This script also
+ generates wrappers for the test
+ functions with code to expand the
+ string parameters read from the data
+ file.
+__MBEDTLS_TEST_TEMPLATE__EXPRESSION_CODE
+ This script enumerates the
+ expressions in the .data file and
+ generates code to handle enumerated
+ expression Ids and return the values.
+__MBEDTLS_TEST_TEMPLATE__DEP_CHECK_CODE
+ This script enumerates all
+ build dependencies and generate
+ code to handle enumerated build
+ dependency Id and return status: if
+ the dependency is defined or not.
+__MBEDTLS_TEST_TEMPLATE__DISPATCH_CODE
+ This script enumerates the functions
+ specified in the input test data file
+ and generates the initializer for the
+ function table in the template
+ file.
+__MBEDTLS_TEST_TEMPLATE__PLATFORM_CODE
+ Platform specific setup and test
+ dispatch code.
"""
@@ -974,11 +980,27 @@
:param snippets: Generated and code snippets
:return:
"""
+
+ # Create a placeholder pattern with the correct named capture groups
+ # to override the default provided with Template.
+ # Match nothing (no way of escaping placeholders).
+ escaped = "(?P<escaped>(?!))"
+ # Match the "__MBEDTLS_TEST_TEMPLATE__PLACEHOLDER_NAME" pattern.
+ named = "__MBEDTLS_TEST_TEMPLATE__(?P<named>[A-Z][_A-Z0-9]*)"
+ # Match nothing (no braced placeholder syntax).
+ braced = "(?P<braced>(?!))"
+ # If not already matched, a "__MBEDTLS_TEST_TEMPLATE__" prefix is invalid.
+ invalid = "(?P<invalid>__MBEDTLS_TEST_TEMPLATE__)"
+ placeholder_pattern = re.compile("|".join([escaped, named, braced, invalid]))
+
with open(template_file, 'r') as template_f, open(c_file, 'w') as c_f:
for line_no, line in enumerate(template_f.readlines(), 1):
# Update line number. +1 as #line directive sets next line number
snippets['line_no'] = line_no + 1
- code = string.Template(line).substitute(**snippets)
+ template = string.Template(line)
+ template.pattern = placeholder_pattern
+ snippets = {k.upper():v for (k, v) in snippets.items()}
+ code = template.substitute(**snippets)
c_f.write(code)
diff --git a/tests/scripts/run-test-suites.pl b/tests/scripts/run-test-suites.pl
index 8a5bb93..cedc0bf 100755
--- a/tests/scripts/run-test-suites.pl
+++ b/tests/scripts/run-test-suites.pl
@@ -50,10 +50,10 @@
'verbose|v:1' => \$verbose,
) or die;
-# All test suites = executable files derived from a .data file.
+# All test suites = executable files with a .datax file.
my @suites = ();
-for my $data_file (glob 'suites/test_suite_*.data') {
- (my $base = $data_file) =~ s#^suites/(.*)\.data$#$1#;
+for my $data_file (glob 'test_suite_*.datax') {
+ (my $base = $data_file) =~ s/\.datax$//;
push @suites, $base if -x $base;
push @suites, "$base.exe" if -e "$base.exe";
}