diff --git a/tests/Makefile b/tests/Makefile
index 5b2ee10..dbba507 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -39,7 +39,7 @@
 $(error "$(PYTHON) scripts/generate_psa_tests.py --list" failed)
 endif
 GENERATED_FILES := $(GENERATED_PSA_DATA_FILES) $(GENERATED_ECP_DATA_FILES) $(GENERATED_BIGNUM_DATA_FILES)
-generated_files: $(GENERATED_FILES)
+generated_files: $(GENERATED_FILES) src/test_keys.h src/test_certs.h
 
 # generate_bignum_tests.py and generate_psa_tests.py spend more time analyzing
 # inputs than generating outputs. Its inputs are the same no matter which files
@@ -236,7 +236,8 @@
 clean:
 ifndef WINDOWS
 	rm -rf $(BINARIES) *.c *.datax
-	rm -f src/*.o src/drivers/*.o src/test_helpers/*.o src/libmbed*
+	rm -f src/*.o src/drivers/*.o src/test_helpers/*.o src/libmbed* src/test_keys.h src/test_certs.h
+	rm -f src/test_keys.h src/test_certs.h
 	rm -f include/test/instrument_record_status.h
 	rm -f include/alt-extra/*/*_alt.h
 	rm -rf libtestdriver1
@@ -247,6 +248,8 @@
 	if exist *.datax del /Q /F *.datax
 	if exist src/*.o del /Q /F src/*.o
 	if exist src/drivers/*.o del /Q /F src/drivers/*.o
+	if exist src/test_keys.h del /Q /F src/test_keys.h
+	if exist src/test_certs.h del /Q /F src/test_cers.h
 	if exist src/test_helpers/*.o del /Q /F src/test_helpers/*.o
 	if exist src/libmbed* del /Q /F src/libmed*
 	if exist include/test/instrument_record_status.h del /Q /F include/test/instrument_record_status.h
diff --git a/tests/scripts/check-generated-files.sh b/tests/scripts/check-generated-files.sh
index 2f20026..96f1221 100755
--- a/tests/scripts/check-generated-files.sh
+++ b/tests/scripts/check-generated-files.sh
@@ -131,6 +131,7 @@
 check tests/scripts/generate_bignum_tests.py $(tests/scripts/generate_bignum_tests.py --list)
 check tests/scripts/generate_ecp_tests.py $(tests/scripts/generate_ecp_tests.py --list)
 check tests/scripts/generate_psa_tests.py $(tests/scripts/generate_psa_tests.py --list)
+check tests/scripts/generate_test_keys.py tests/src/test_keys.h
 check scripts/generate_driver_wrappers.py $library_dir/psa_crypto_driver_wrappers.h $library_dir/psa_crypto_driver_wrappers_no_static.c
 
 # Additional checks for Mbed TLS only
diff --git a/tests/scripts/generate_test_keys.py b/tests/scripts/generate_test_keys.py
index dbdb960..85ff918 100755
--- a/tests/scripts/generate_test_keys.py
+++ b/tests/scripts/generate_test_keys.py
@@ -99,9 +99,9 @@
     args = argparser.parse_args()
 
     output_file = args.output
-    # Remove output file if already existing.
+    # If the output file already exists, then we can quit (successfully)
     if os.path.exists(output_file):
-        os.remove(output_file)
+        return
 
     output_file = open(output_file, 'at')
     output_file.write(