Generate config test cases for single options
Generate option-on and option-off cases for test_suite_config, for all
boolean options (MBEDTLS_xxx and PSA_WANT_xxx, collected from the mbedtls
and PSA config files).
Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
diff --git a/framework b/framework
index 0484721..1893bfb 160000
--- a/framework
+++ b/framework
@@ -1 +1 @@
-Subproject commit 04847216ab964b9bdce41f1e61ccc6d8f5d2a139
+Subproject commit 1893bfb1ee4b1d971db38bf3ea1f8005fe3c8d9a
diff --git a/scripts/make_generated_files.bat b/scripts/make_generated_files.bat
index f04f6b7..b03bce2 100644
--- a/scripts/make_generated_files.bat
+++ b/scripts/make_generated_files.bat
@@ -11,6 +11,7 @@
perl scripts\generate_visualc_files.pl || exit /b 1
python scripts\generate_psa_constants.py || exit /b 1
python framework\scripts\generate_bignum_tests.py || exit /b 1
+python framework\scripts\generate_config_tests.py || exit /b 1
python framework\scripts\generate_ecp_tests.py || exit /b 1
python framework\scripts\generate_psa_tests.py || exit /b 1
python framework\scripts\generate_test_keys.py --output tests\src\test_keys.h || exit /b 1
diff --git a/tests/.gitignore b/tests/.gitignore
index 838ea69..635dd62 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -19,6 +19,8 @@
###START_GENERATED_FILES###
# Generated source files
/suites/*.generated.data
+suites/test_suite_config.mbedtls_boolean.data
+suites/test_suite_config.psa_boolean.data
/suites/test_suite_psa_crypto_storage_format.v[0-9]*.data
/suites/test_suite_psa_crypto_storage_format.current.data
/src/test_keys.h
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 62be14e..060a928 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -33,6 +33,18 @@
execute_process(
COMMAND
${MBEDTLS_PYTHON_EXECUTABLE}
+ ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_config_tests.py
+ --list-for-cmake
+ WORKING_DIRECTORY
+ ${CMAKE_CURRENT_SOURCE_DIR}/..
+ OUTPUT_VARIABLE
+ base_config_generated_data_files)
+string(REGEX REPLACE "[^;]*/" ""
+ base_config_generated_data_files "${base_config_generated_data_files}")
+
+execute_process(
+ COMMAND
+ ${MBEDTLS_PYTHON_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_ecp_tests.py
--list-for-cmake
WORKING_DIRECTORY
@@ -61,11 +73,15 @@
string(REGEX REPLACE "([^;]+)" "suites/\\1"
all_generated_data_files "${base_generated_data_files}")
set(bignum_generated_data_files "")
+set(config_generated_data_files "")
set(ecp_generated_data_files "")
set(psa_generated_data_files "")
foreach(file ${base_bignum_generated_data_files})
list(APPEND bignum_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/suites/${file})
endforeach()
+foreach(file ${base_config_generated_data_files})
+ list(APPEND config_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/suites/${file})
+endforeach()
foreach(file ${base_ecp_generated_data_files})
list(APPEND ecp_generated_data_files ${CMAKE_CURRENT_BINARY_DIR}/suites/${file})
endforeach()
@@ -94,6 +110,21 @@
)
add_custom_command(
OUTPUT
+ ${config_generated_data_files}
+ WORKING_DIRECTORY
+ ${CMAKE_CURRENT_SOURCE_DIR}/..
+ COMMAND
+ ${MBEDTLS_PYTHON_EXECUTABLE}
+ ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_config_tests.py
+ --directory ${CMAKE_CURRENT_BINARY_DIR}/suites
+ DEPENDS
+ ${CMAKE_CURRENT_SOURCE_DIR}/../framework/scripts/generate_config_tests.py
+ # Do not declare the configuration files as dependencies: they
+ # change too often in ways that don't affect the result
+ # ((un)commenting some options).
+ )
+ add_custom_command(
+ OUTPUT
${ecp_generated_data_files}
WORKING_DIRECTORY
${CMAKE_CURRENT_SOURCE_DIR}/..
diff --git a/tests/Makefile b/tests/Makefile
index dcbd066..b5637f3 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -26,6 +26,15 @@
endif
GENERATED_DATA_FILES += $(GENERATED_BIGNUM_DATA_FILES)
+GENERATED_CONFIG_DATA_FILES := $(patsubst tests/%,%,$(shell \
+ $(PYTHON) ../framework/scripts/generate_config_tests.py --list || \
+ echo FAILED \
+))
+ifeq ($(GENERATED_CONFIG_DATA_FILES),FAILED)
+$(error "$(PYTHON) ../framework/scripts/generate_config_tests.py --list" failed)
+endif
+GENERATED_DATA_FILES += $(GENERATED_CONFIG_DATA_FILES)
+
GENERATED_ECP_DATA_FILES := $(patsubst tests/%,%,$(shell \
$(PYTHON) ../framework/scripts/generate_ecp_tests.py --list || \
echo FAILED \
@@ -69,6 +78,22 @@
$(PYTHON) ../framework/scripts/generate_bignum_tests.py
.SECONDARY: generated_bignum_test_data
+# We deliberately omit the configuration files (mbedtls_config.h,
+# crypto_config.h) from the depenency list because during development
+# and on the CI, we often edit those in a way that doesn't change the
+# output, to comment out certain options, or even to remove certain
+# lines which do affect the output negatively (it will miss the
+# corresponding test cases).
+$(GENERATED_CONFIG_DATA_FILES): $(gen_file_dep) generated_config_test_data
+generated_config_test_data: ../framework/scripts/generate_config_tests.py
+generated_config_test_data: ../scripts/config.py
+generated_config_test_data: ../framework/scripts/mbedtls_framework/test_case.py
+generated_config_test_data: ../framework/scripts/mbedtls_framework/test_data_generation.py
+generated_config_test_data:
+ echo " Gen $(GENERATED_CONFIG_DATA_FILES)"
+ $(PYTHON) ../framework/scripts/generate_config_tests.py
+.SECONDARY: generated_config_test_data
+
$(GENERATED_ECP_DATA_FILES): $(gen_file_dep) generated_ecp_test_data
generated_ecp_test_data: ../framework/scripts/generate_ecp_tests.py
generated_ecp_test_data: ../framework/scripts/mbedtls_framework/bignum_common.py
diff --git a/tests/scripts/check-generated-files.sh b/tests/scripts/check-generated-files.sh
index e740f33..09c850a 100755
--- a/tests/scripts/check-generated-files.sh
+++ b/tests/scripts/check-generated-files.sh
@@ -129,6 +129,7 @@
# These checks are common to Mbed TLS and TF-PSA-Crypto
check scripts/generate_psa_constants.py programs/psa/psa_constant_names_generated.c
check framework/scripts/generate_bignum_tests.py $(framework/scripts/generate_bignum_tests.py --list)
+check framework/scripts/generate_config_tests.py $(framework/scripts/generate_config_tests.py --list)
check framework/scripts/generate_ecp_tests.py $(framework/scripts/generate_ecp_tests.py --list)
check framework/scripts/generate_psa_tests.py $(framework/scripts/generate_psa_tests.py --list)
check framework/scripts/generate_test_keys.py tests/src/test_keys.h