blob: 1dc05f03bb64350ad1c010f92896a4770e672400 [file] [log] [blame]
Gilles Peskine15c2cbf2020-06-25 18:36:28 +02001#!/usr/bin/env python3
2
3"""Analyze the test outcomes from a full CI run.
4
5This script can also run on outcomes from a partial run, but the results are
6less likely to be useful.
7"""
8
Przemek Stekiel85c54ea2022-11-17 11:50:23 +01009import re
Gilles Peskined8da2fc2024-09-17 15:07:22 +020010import typing
Gilles Peskine15c2cbf2020-06-25 18:36:28 +020011
Gilles Peskine738a5972024-10-03 18:52:58 +020012import scripts_path # pylint: disable=unused-import
13from mbedtls_framework import outcome_analysis
Gilles Peskine8d3c70a2020-06-25 18:37:43 +020014
Pengyu Lvc2e8f3a2023-11-28 17:22:04 +080015
Gilles Peskine45a32b12024-10-03 18:42:37 +020016class CoverageTask(outcome_analysis.CoverageTask):
Gilles Peskine4e606db2024-10-04 16:24:26 +020017 """Justify test cases that are never executed."""
Gilles Peskine95b2b0c2024-09-16 20:23:40 +020018
Gilles Peskined8da2fc2024-09-17 15:07:22 +020019 @staticmethod
Gilles Peskine72396da2024-09-17 17:15:29 +020020 def _has_word_re(words: typing.Iterable[str],
21 exclude: typing.Optional[str] = None) -> typing.Pattern:
Gilles Peskined8da2fc2024-09-17 15:07:22 +020022 """Construct a regex that matches if any of the words appears.
23
24 The occurrence must start and end at a word boundary.
Gilles Peskine72396da2024-09-17 17:15:29 +020025
26 If exclude is specified, strings containing a match for that
27 regular expression will not match the returned pattern.
Gilles Peskined8da2fc2024-09-17 15:07:22 +020028 """
Gilles Peskine72396da2024-09-17 17:15:29 +020029 exclude_clause = r''
30 if exclude:
31 exclude_clause = r'(?!.*' + exclude + ')'
32 return re.compile(exclude_clause +
33 r'.*\b(?:' + r'|'.join(words) + r')\b.*',
Gilles Peskine1abc8002024-10-11 12:00:44 +020034 re.DOTALL)
Gilles Peskined8da2fc2024-09-17 15:07:22 +020035
Gilles Peskined8da2fc2024-09-17 15:07:22 +020036 IGNORED_TESTS = {
Gilles Peskine419a5842024-09-17 18:32:05 +020037 'ssl-opt': [
38 # We don't run ssl-opt.sh with Valgrind on the CI because
39 # it's extremely slow. We don't intend to change this.
40 'DTLS client reconnect from same port: reconnect, nbio, valgrind',
Gilles Peskine419a5842024-09-17 18:32:05 +020041 # We don't have IPv6 in our CI environment.
42 # https://github.com/Mbed-TLS/mbedtls-test/issues/176
43 'DTLS cookie: enabled, IPv6',
44 # Disabled due to OpenSSL bug.
45 # https://github.com/openssl/openssl/issues/18887
46 'DTLS fragmenting: 3d, openssl client, DTLS 1.2',
47 # We don't run ssl-opt.sh with Valgrind on the CI because
48 # it's extremely slow. We don't intend to change this.
49 'DTLS fragmenting: proxy MTU: auto-reduction (with valgrind)',
50 # It seems that we don't run `ssl-opt.sh` with
51 # `MBEDTLS_USE_PSA_CRYPTO` enabled but `MBEDTLS_SSL_ASYNC_PRIVATE`
52 # disabled.
53 # https://github.com/Mbed-TLS/mbedtls/issues/9581
54 'Opaque key for server authentication: invalid key: decrypt with ECC key, no async',
55 'Opaque key for server authentication: invalid key: ecdh with RSA key, no async',
56 ],
Gilles Peskine47243fd2024-09-17 19:46:18 +020057 'test_suite_config.mbedtls_boolean': [
58 # We never test with CBC/PKCS5/PKCS12 enabled but
59 # PKCS7 padding disabled.
60 # https://github.com/Mbed-TLS/mbedtls/issues/9580
61 'Config: !MBEDTLS_CIPHER_PADDING_PKCS7',
62 # https://github.com/Mbed-TLS/mbedtls/issues/9583
63 'Config: !MBEDTLS_ECP_NIST_OPTIM',
Gilles Peskine44fdd922024-10-10 18:19:23 +020064 # MBEDTLS_ECP_NO_FALLBACK only affects builds using a partial
65 # alternative implementation of ECP arithmetic (with
66 # MBEDTLS_ECP_INTERNAL_ALT enabled). We don't test those builds.
67 # The configuration enumeration script skips xxx_ALT options
68 # but not MBEDTLS_ECP_NO_FALLBACK, so it appears in the report,
69 # but we don't care about it.
70 'Config: MBEDTLS_ECP_NO_FALLBACK',
Gilles Peskine47243fd2024-09-17 19:46:18 +020071 # Missing coverage of test configurations.
72 # https://github.com/Mbed-TLS/mbedtls/issues/9585
73 'Config: !MBEDTLS_SSL_DTLS_ANTI_REPLAY',
74 # Missing coverage of test configurations.
75 # https://github.com/Mbed-TLS/mbedtls/issues/9585
76 'Config: !MBEDTLS_SSL_DTLS_HELLO_VERIFY',
77 # We don't run test_suite_config when we test this.
78 # https://github.com/Mbed-TLS/mbedtls/issues/9586
79 'Config: !MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_ENABLED',
80 # We only test multithreading with pthreads.
81 # https://github.com/Mbed-TLS/mbedtls/issues/9584
82 'Config: !MBEDTLS_THREADING_PTHREAD',
83 # Built but not tested.
84 # https://github.com/Mbed-TLS/mbedtls/issues/9587
85 'Config: MBEDTLS_AES_USE_HARDWARE_ONLY',
86 # Untested platform-specific optimizations.
87 # https://github.com/Mbed-TLS/mbedtls/issues/9588
88 'Config: MBEDTLS_HAVE_SSE2',
89 # Obsolete configuration option, to be replaced by
90 # PSA entropy drivers.
91 # https://github.com/Mbed-TLS/mbedtls/issues/8150
92 'Config: MBEDTLS_NO_PLATFORM_ENTROPY',
93 # Untested aspect of the platform interface.
94 # https://github.com/Mbed-TLS/mbedtls/issues/9589
95 'Config: MBEDTLS_PLATFORM_NO_STD_FUNCTIONS',
96 # In a client-server build, test_suite_config runs in the
97 # client configuration, so it will never report
98 # MBEDTLS_PSA_CRYPTO_SPM as enabled. That's ok.
99 'Config: MBEDTLS_PSA_CRYPTO_SPM',
100 # We don't test on armv8 yet.
101 'Config: MBEDTLS_SHA256_USE_A64_CRYPTO_IF_PRESENT',
102 'Config: MBEDTLS_SHA256_USE_A64_CRYPTO_ONLY',
103 'Config: MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY',
104 'Config: MBEDTLS_SHA512_USE_A64_CRYPTO_ONLY',
105 # We don't run test_suite_config when we test this.
106 # https://github.com/Mbed-TLS/mbedtls/issues/9586
107 'Config: MBEDTLS_TEST_CONSTANT_FLOW_VALGRIND',
108 ],
109 'test_suite_config.psa_boolean': [
110 # We don't test with HMAC disabled.
111 # https://github.com/Mbed-TLS/mbedtls/issues/9591
112 'Config: !PSA_WANT_ALG_HMAC',
Gilles Peskine47243fd2024-09-17 19:46:18 +0200113 # The DERIVE key type is always enabled.
114 'Config: !PSA_WANT_KEY_TYPE_DERIVE',
115 # More granularity of key pair type enablement macros
116 # than we care to test.
117 # https://github.com/Mbed-TLS/mbedtls/issues/9590
118 'Config: !PSA_WANT_KEY_TYPE_DH_KEY_PAIR_EXPORT',
119 'Config: !PSA_WANT_KEY_TYPE_DH_KEY_PAIR_GENERATE',
120 'Config: !PSA_WANT_KEY_TYPE_DH_KEY_PAIR_IMPORT',
121 # More granularity of key pair type enablement macros
122 # than we care to test.
123 # https://github.com/Mbed-TLS/mbedtls/issues/9590
124 'Config: !PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_EXPORT',
125 'Config: !PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_IMPORT',
126 # We don't test with HMAC disabled.
127 # https://github.com/Mbed-TLS/mbedtls/issues/9591
128 'Config: !PSA_WANT_KEY_TYPE_HMAC',
129 # The PASSWORD key type is always enabled.
130 'Config: !PSA_WANT_KEY_TYPE_PASSWORD',
131 # The PASSWORD_HASH key type is always enabled.
132 'Config: !PSA_WANT_KEY_TYPE_PASSWORD_HASH',
133 # The RAW_DATA key type is always enabled.
134 'Config: !PSA_WANT_KEY_TYPE_RAW_DATA',
135 # More granularity of key pair type enablement macros
136 # than we care to test.
137 # https://github.com/Mbed-TLS/mbedtls/issues/9590
138 'Config: !PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_EXPORT',
139 'Config: !PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_IMPORT',
140 # Algorithm declared but not supported.
141 'Config: PSA_WANT_ALG_CBC_MAC',
142 # Algorithm declared but not supported.
143 'Config: PSA_WANT_ALG_XTS',
144 # Family declared but not supported.
145 'Config: PSA_WANT_ECC_SECP_K1_224',
146 # More granularity of key pair type enablement macros
147 # than we care to test.
148 # https://github.com/Mbed-TLS/mbedtls/issues/9590
149 'Config: PSA_WANT_KEY_TYPE_DH_KEY_PAIR_DERIVE',
150 'Config: PSA_WANT_KEY_TYPE_ECC_KEY_PAIR',
151 'Config: PSA_WANT_KEY_TYPE_RSA_KEY_PAIR',
152 'Config: PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_DERIVE',
153 ],
154 'test_suite_config.psa_combinations': [
155 # We don't test this unusual, but sensible configuration.
156 # https://github.com/Mbed-TLS/mbedtls/issues/9592
157 'Config: PSA_WANT_ALG_DETERMINSTIC_ECDSA without PSA_WANT_ALG_ECDSA',
158 ],
Gilles Peskine1a176272024-09-17 18:33:29 +0200159 'test_suite_pkcs12': [
Gilles Peskine47243fd2024-09-17 19:46:18 +0200160 # We never test with CBC/PKCS5/PKCS12 enabled but
161 # PKCS7 padding disabled.
Gilles Peskine1a176272024-09-17 18:33:29 +0200162 # https://github.com/Mbed-TLS/mbedtls/issues/9580
163 'PBE Decrypt, (Invalid padding & PKCS7 padding disabled)',
164 'PBE Encrypt, pad = 8 (PKCS7 padding disabled)',
165 ],
166 'test_suite_pkcs5': [
Gilles Peskine47243fd2024-09-17 19:46:18 +0200167 # We never test with CBC/PKCS5/PKCS12 enabled but
168 # PKCS7 padding disabled.
Gilles Peskine1a176272024-09-17 18:33:29 +0200169 # https://github.com/Mbed-TLS/mbedtls/issues/9580
170 'PBES2 Decrypt (Invalid padding & PKCS7 padding disabled)',
171 'PBES2 Encrypt, pad=6 (PKCS7 padding disabled)',
172 'PBES2 Encrypt, pad=8 (PKCS7 padding disabled)',
173 ],
Gilles Peskine8729b102024-04-19 19:08:34 +0200174 'test_suite_psa_crypto': [
175 # We don't test this unusual, but sensible configuration.
176 # https://github.com/Mbed-TLS/mbedtls/issues/9592
177 re.compile(r'.*ECDSA.*only deterministic supported'),
178 ],
Gilles Peskine1a176272024-09-17 18:33:29 +0200179 'test_suite_psa_crypto_metadata': [
180 # Algorithms declared but not supported.
181 # https://github.com/Mbed-TLS/mbedtls/issues/9579
182 'Asymmetric signature: Ed25519ph',
183 'Asymmetric signature: Ed448ph',
184 'Asymmetric signature: pure EdDSA',
185 'Cipher: XTS',
186 'MAC: CBC_MAC-3DES',
187 'MAC: CBC_MAC-AES-128',
188 'MAC: CBC_MAC-AES-192',
189 'MAC: CBC_MAC-AES-256',
190 ],
Gilles Peskined8da2fc2024-09-17 15:07:22 +0200191 'test_suite_psa_crypto_not_supported.generated': [
Gilles Peskine1abc8002024-10-11 12:00:44 +0200192 # We never test with DH key support disabled but support
Gilles Peskine1fac3712024-09-17 17:57:11 +0200193 # for a DH group enabled. The dependencies of these test
194 # cases don't really make sense.
195 # https://github.com/Mbed-TLS/mbedtls/issues/9574
196 re.compile(r'PSA \w+ DH_.*type not supported'),
197 # We only test partial support for DH with the 2048-bit group
198 # enabled and the other groups disabled.
199 # https://github.com/Mbed-TLS/mbedtls/issues/9575
200 'PSA generate DH_KEY_PAIR(RFC7919) 2048-bit group not supported',
201 'PSA import DH_KEY_PAIR(RFC7919) 2048-bit group not supported',
202 'PSA import DH_PUBLIC_KEY(RFC7919) 2048-bit group not supported',
Gilles Peskined8da2fc2024-09-17 15:07:22 +0200203 ],
204 'test_suite_psa_crypto_op_fail.generated': [
Gilles Peskine8729b102024-04-19 19:08:34 +0200205 # We don't test this unusual, but sensible configuration.
206 # https://github.com/Mbed-TLS/mbedtls/issues/9592
207 re.compile(r'.*: !ECDSA but DETERMINISTIC_ECDSA with ECC_.*'),
Gilles Peskine1fac3712024-09-17 17:57:11 +0200208 # PBKDF2_HMAC is not in the default configuration, so we don't
209 # enable it in depends.py where we remove hashes.
210 # https://github.com/Mbed-TLS/mbedtls/issues/9576
211 re.compile(r'PSA key_derivation PBKDF2_HMAC\(\w+\): !(?!PBKDF2_HMAC\Z).*'),
Gilles Peskine1fac3712024-09-17 17:57:11 +0200212
213 # We never test with the HMAC algorithm enabled but the HMAC
214 # key type disabled. Those dependencies don't really make sense.
215 # https://github.com/Mbed-TLS/mbedtls/issues/9573
216 re.compile(r'.* !HMAC with HMAC'),
217 # There's something wrong with PSA_WANT_ALG_RSA_PSS_ANY_SALT
218 # differing from PSA_WANT_ALG_RSA_PSS.
219 # https://github.com/Mbed-TLS/mbedtls/issues/9578
220 re.compile(r'PSA sign RSA_PSS_ANY_SALT.*!(?:MD|RIPEMD|SHA).*'),
Gilles Peskined8da2fc2024-09-17 15:07:22 +0200221 ],
Gilles Peskine8729b102024-04-19 19:08:34 +0200222 'test_suite_psa_crypto_op_fail.misc': [
223 # We don't test this unusual, but sensible configuration.
224 # https://github.com/Mbed-TLS/mbedtls/issues/9592
225 'PSA sign DETERMINISTIC_ECDSA(SHA_256): !ECDSA but DETERMINISTIC_ECDSA with ECC_KEY_PAIR(SECP_R1)', #pylint: disable=line-too-long
226 ],
Gilles Peskine419a5842024-09-17 18:32:05 +0200227 'tls13-misc': [
228 # Disabled due to OpenSSL bug.
229 # https://github.com/openssl/openssl/issues/10714
230 'TLS 1.3 O->m: resumption',
231 # Disabled due to OpenSSL command line limitation.
232 # https://github.com/Mbed-TLS/mbedtls/issues/9582
233 'TLS 1.3 m->O: resumption with early data',
234 ],
Gilles Peskined8da2fc2024-09-17 15:07:22 +0200235 }
236
Gilles Peskine17e071b2024-09-16 19:57:10 +0200237
Gilles Peskine92cc8db2024-09-16 20:14:26 +0200238# The names that we give to classes derived from DriverVSReference do not
239# follow the usual naming convention, because it's more readable to use
240# underscores and parts of the configuration names. Also, these classes
241# are just there to specify some data, so they don't need repetitive
242# documentation.
243#pylint: disable=invalid-name,missing-class-docstring
244
Gilles Peskine45a32b12024-10-03 18:42:37 +0200245class DriverVSReference_hash(outcome_analysis.DriverVSReference):
Gilles Peskine92cc8db2024-09-16 20:14:26 +0200246 REFERENCE = 'test_psa_crypto_config_reference_hash_use_psa'
247 DRIVER = 'test_psa_crypto_config_accel_hash_use_psa'
248 IGNORED_SUITES = [
249 'shax', 'mdx', # the software implementations that are being excluded
250 'md.psa', # purposefully depends on whether drivers are present
251 'psa_crypto_low_hash.generated', # testing the builtins
252 ]
253 IGNORED_TESTS = {
254 'test_suite_config': [
255 re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'),
256 ],
257 'test_suite_platform': [
258 # Incompatible with sanitizers (e.g. ASan). If the driver
259 # component uses a sanitizer but the reference component
260 # doesn't, we have a PASS vs SKIP mismatch.
261 'Check mbedtls_calloc overallocation',
262 ],
263 }
264
Gilles Peskine45a32b12024-10-03 18:42:37 +0200265class DriverVSReference_hmac(outcome_analysis.DriverVSReference):
Gilles Peskine92cc8db2024-09-16 20:14:26 +0200266 REFERENCE = 'test_psa_crypto_config_reference_hmac'
267 DRIVER = 'test_psa_crypto_config_accel_hmac'
268 IGNORED_SUITES = [
269 # These suites require legacy hash support, which is disabled
270 # in the accelerated component.
271 'shax', 'mdx',
272 # This suite tests builtins directly, but these are missing
273 # in the accelerated case.
274 'psa_crypto_low_hash.generated',
275 ]
276 IGNORED_TESTS = {
277 'test_suite_config': [
278 re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'),
279 re.compile(r'.*\bMBEDTLS_MD_C\b')
280 ],
281 'test_suite_md': [
282 # Builtin HMAC is not supported in the accelerate component.
283 re.compile('.*HMAC.*'),
284 # Following tests make use of functions which are not available
285 # when MD_C is disabled, as it happens in the accelerated
286 # test component.
287 re.compile('generic .* Hash file .*'),
288 'MD list',
289 ],
290 'test_suite_md.psa': [
291 # "legacy only" tests require hash algorithms to be NOT
292 # accelerated, but this of course false for the accelerated
293 # test component.
294 re.compile('PSA dispatch .* legacy only'),
295 ],
296 'test_suite_platform': [
297 # Incompatible with sanitizers (e.g. ASan). If the driver
298 # component uses a sanitizer but the reference component
299 # doesn't, we have a PASS vs SKIP mismatch.
300 'Check mbedtls_calloc overallocation',
301 ],
302 }
303
Gilles Peskine45a32b12024-10-03 18:42:37 +0200304class DriverVSReference_cipher_aead_cmac(outcome_analysis.DriverVSReference):
Gilles Peskine92cc8db2024-09-16 20:14:26 +0200305 REFERENCE = 'test_psa_crypto_config_reference_cipher_aead_cmac'
306 DRIVER = 'test_psa_crypto_config_accel_cipher_aead_cmac'
307 # Modules replaced by drivers.
308 IGNORED_SUITES = [
309 # low-level (block/stream) cipher modules
310 'aes', 'aria', 'camellia', 'des', 'chacha20',
311 # AEAD modes and CMAC
312 'ccm', 'chachapoly', 'cmac', 'gcm',
313 # The Cipher abstraction layer
314 'cipher',
315 ]
316 IGNORED_TESTS = {
317 'test_suite_config': [
318 re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA|CHACHA20|DES)_.*'),
319 re.compile(r'.*\bMBEDTLS_(CCM|CHACHAPOLY|CMAC|GCM)_.*'),
320 re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'),
321 re.compile(r'.*\bMBEDTLS_CIPHER_.*'),
322 ],
323 # PEM decryption is not supported so far.
324 # The rest of PEM (write, unencrypted read) works though.
325 'test_suite_pem': [
326 re.compile(r'PEM read .*(AES|DES|\bencrypt).*'),
327 ],
328 'test_suite_platform': [
329 # Incompatible with sanitizers (e.g. ASan). If the driver
330 # component uses a sanitizer but the reference component
331 # doesn't, we have a PASS vs SKIP mismatch.
332 'Check mbedtls_calloc overallocation',
333 ],
334 # Following tests depend on AES_C/DES_C but are not about
335 # them really, just need to know some error code is there.
336 'test_suite_error': [
337 'Low and high error',
338 'Single low error'
339 ],
340 # Similar to test_suite_error above.
341 'test_suite_version': [
342 'Check for MBEDTLS_AES_C when already present',
343 ],
344 # The en/decryption part of PKCS#12 is not supported so far.
345 # The rest of PKCS#12 (key derivation) works though.
346 'test_suite_pkcs12': [
347 re.compile(r'PBE Encrypt, .*'),
348 re.compile(r'PBE Decrypt, .*'),
349 ],
350 # The en/decryption part of PKCS#5 is not supported so far.
351 # The rest of PKCS#5 (PBKDF2) works though.
352 'test_suite_pkcs5': [
353 re.compile(r'PBES2 Encrypt, .*'),
354 re.compile(r'PBES2 Decrypt .*'),
355 ],
356 # Encrypted keys are not supported so far.
357 # pylint: disable=line-too-long
358 'test_suite_pkparse': [
359 'Key ASN1 (Encrypted key PKCS12, trailing garbage data)',
360 'Key ASN1 (Encrypted key PKCS5, trailing garbage data)',
361 re.compile(r'Parse (RSA|EC) Key .*\(.* ([Ee]ncrypted|password).*\)'),
362 ],
363 # Encrypted keys are not supported so far.
364 'ssl-opt': [
365 'TLS: password protected server key',
366 'TLS: password protected client key',
367 'TLS: password protected server key, two certificates',
368 ],
369 }
370
Gilles Peskine45a32b12024-10-03 18:42:37 +0200371class DriverVSReference_ecp_light_only(outcome_analysis.DriverVSReference):
Gilles Peskine92cc8db2024-09-16 20:14:26 +0200372 REFERENCE = 'test_psa_crypto_config_reference_ecc_ecp_light_only'
373 DRIVER = 'test_psa_crypto_config_accel_ecc_ecp_light_only'
374 IGNORED_SUITES = [
375 # Modules replaced by drivers
376 'ecdsa', 'ecdh', 'ecjpake',
Gilles Peskine9a094432024-10-29 20:55:11 +0100377 # Unit tests for the built-in implementation
378 'psa_crypto_ecp',
Gilles Peskine92cc8db2024-09-16 20:14:26 +0200379 ]
380 IGNORED_TESTS = {
381 'test_suite_config': [
382 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
383 ],
384 'test_suite_platform': [
385 # Incompatible with sanitizers (e.g. ASan). If the driver
386 # component uses a sanitizer but the reference component
387 # doesn't, we have a PASS vs SKIP mismatch.
388 'Check mbedtls_calloc overallocation',
389 ],
390 # This test wants a legacy function that takes f_rng, p_rng
391 # arguments, and uses legacy ECDSA for that. The test is
392 # really about the wrapper around the PSA RNG, not ECDSA.
393 'test_suite_random': [
394 'PSA classic wrapper: ECDSA signature (SECP256R1)',
395 ],
396 # In the accelerated test ECP_C is not set (only ECP_LIGHT is)
397 # so we must ignore disparities in the tests for which ECP_C
398 # is required.
399 'test_suite_ecp': [
400 re.compile(r'ECP check public-private .*'),
401 re.compile(r'ECP calculate public: .*'),
402 re.compile(r'ECP gen keypair .*'),
403 re.compile(r'ECP point muladd .*'),
404 re.compile(r'ECP point multiplication .*'),
405 re.compile(r'ECP test vectors .*'),
406 ],
407 'test_suite_ssl': [
408 # This deprecated function is only present when ECP_C is On.
409 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
410 ],
411 }
412
Gilles Peskine45a32b12024-10-03 18:42:37 +0200413class DriverVSReference_no_ecp_at_all(outcome_analysis.DriverVSReference):
Gilles Peskine92cc8db2024-09-16 20:14:26 +0200414 REFERENCE = 'test_psa_crypto_config_reference_ecc_no_ecp_at_all'
415 DRIVER = 'test_psa_crypto_config_accel_ecc_no_ecp_at_all'
416 IGNORED_SUITES = [
417 # Modules replaced by drivers
418 'ecp', 'ecdsa', 'ecdh', 'ecjpake',
Gilles Peskine9a094432024-10-29 20:55:11 +0100419 # Unit tests for the built-in implementation
420 'psa_crypto_ecp',
Gilles Peskine92cc8db2024-09-16 20:14:26 +0200421 ]
422 IGNORED_TESTS = {
423 'test_suite_config': [
424 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
425 re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'),
426 ],
427 'test_suite_platform': [
428 # Incompatible with sanitizers (e.g. ASan). If the driver
429 # component uses a sanitizer but the reference component
430 # doesn't, we have a PASS vs SKIP mismatch.
431 'Check mbedtls_calloc overallocation',
432 ],
433 # See ecp_light_only
434 'test_suite_random': [
435 'PSA classic wrapper: ECDSA signature (SECP256R1)',
436 ],
437 'test_suite_pkparse': [
438 # When PK_PARSE_C and ECP_C are defined then PK_PARSE_EC_COMPRESSED
439 # is automatically enabled in build_info.h (backward compatibility)
440 # even if it is disabled in config_psa_crypto_no_ecp_at_all(). As a
441 # consequence compressed points are supported in the reference
442 # component but not in the accelerated one, so they should be skipped
443 # while checking driver's coverage.
444 re.compile(r'Parse EC Key .*compressed\)'),
445 re.compile(r'Parse Public EC Key .*compressed\)'),
446 ],
447 # See ecp_light_only
448 'test_suite_ssl': [
449 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
450 ],
451 }
452
Gilles Peskine45a32b12024-10-03 18:42:37 +0200453class DriverVSReference_ecc_no_bignum(outcome_analysis.DriverVSReference):
Gilles Peskine92cc8db2024-09-16 20:14:26 +0200454 REFERENCE = 'test_psa_crypto_config_reference_ecc_no_bignum'
455 DRIVER = 'test_psa_crypto_config_accel_ecc_no_bignum'
456 IGNORED_SUITES = [
457 # Modules replaced by drivers
458 'ecp', 'ecdsa', 'ecdh', 'ecjpake',
459 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
460 'bignum.generated', 'bignum.misc',
Gilles Peskine9a094432024-10-29 20:55:11 +0100461 # Unit tests for the built-in implementation
462 'psa_crypto_ecp',
Gilles Peskine92cc8db2024-09-16 20:14:26 +0200463 ]
464 IGNORED_TESTS = {
465 'test_suite_config': [
466 re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'),
467 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
468 re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'),
469 ],
470 'test_suite_platform': [
471 # Incompatible with sanitizers (e.g. ASan). If the driver
472 # component uses a sanitizer but the reference component
473 # doesn't, we have a PASS vs SKIP mismatch.
474 'Check mbedtls_calloc overallocation',
475 ],
476 # See ecp_light_only
477 'test_suite_random': [
478 'PSA classic wrapper: ECDSA signature (SECP256R1)',
479 ],
480 # See no_ecp_at_all
481 'test_suite_pkparse': [
482 re.compile(r'Parse EC Key .*compressed\)'),
483 re.compile(r'Parse Public EC Key .*compressed\)'),
484 ],
485 'test_suite_asn1parse': [
486 'INTEGER too large for mpi',
487 ],
488 'test_suite_asn1write': [
489 re.compile(r'ASN.1 Write mpi.*'),
490 ],
491 'test_suite_debug': [
492 re.compile(r'Debug print mbedtls_mpi.*'),
493 ],
494 # See ecp_light_only
495 'test_suite_ssl': [
496 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
497 ],
498 }
499
Gilles Peskine45a32b12024-10-03 18:42:37 +0200500class DriverVSReference_ecc_ffdh_no_bignum(outcome_analysis.DriverVSReference):
Gilles Peskine92cc8db2024-09-16 20:14:26 +0200501 REFERENCE = 'test_psa_crypto_config_reference_ecc_ffdh_no_bignum'
502 DRIVER = 'test_psa_crypto_config_accel_ecc_ffdh_no_bignum'
503 IGNORED_SUITES = [
504 # Modules replaced by drivers
505 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 'dhm',
506 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
507 'bignum.generated', 'bignum.misc',
Gilles Peskine9a094432024-10-29 20:55:11 +0100508 # Unit tests for the built-in implementation
509 'psa_crypto_ecp',
Gilles Peskine92cc8db2024-09-16 20:14:26 +0200510 ]
511 IGNORED_TESTS = {
512 'ssl-opt': [
513 # DHE support in TLS 1.2 requires built-in MBEDTLS_DHM_C
514 # (because it needs custom groups, which PSA does not
515 # provide), even with MBEDTLS_USE_PSA_CRYPTO.
516 re.compile(r'PSK callback:.*\bdhe-psk\b.*'),
517 ],
518 'test_suite_config': [
519 re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'),
520 re.compile(r'.*\bMBEDTLS_DHM_C\b.*'),
521 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'),
522 re.compile(r'.*\bMBEDTLS_KEY_EXCHANGE_DHE_PSK_ENABLED\b.*'),
523 re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'),
524 ],
525 'test_suite_platform': [
526 # Incompatible with sanitizers (e.g. ASan). If the driver
527 # component uses a sanitizer but the reference component
528 # doesn't, we have a PASS vs SKIP mismatch.
529 'Check mbedtls_calloc overallocation',
530 ],
531 # See ecp_light_only
532 'test_suite_random': [
533 'PSA classic wrapper: ECDSA signature (SECP256R1)',
534 ],
535 # See no_ecp_at_all
536 'test_suite_pkparse': [
537 re.compile(r'Parse EC Key .*compressed\)'),
538 re.compile(r'Parse Public EC Key .*compressed\)'),
539 ],
540 'test_suite_asn1parse': [
541 'INTEGER too large for mpi',
542 ],
543 'test_suite_asn1write': [
544 re.compile(r'ASN.1 Write mpi.*'),
545 ],
546 'test_suite_debug': [
547 re.compile(r'Debug print mbedtls_mpi.*'),
548 ],
549 # See ecp_light_only
550 'test_suite_ssl': [
551 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()',
552 ],
553 }
554
Gilles Peskine45a32b12024-10-03 18:42:37 +0200555class DriverVSReference_ffdh_alg(outcome_analysis.DriverVSReference):
Gilles Peskine92cc8db2024-09-16 20:14:26 +0200556 REFERENCE = 'test_psa_crypto_config_reference_ffdh'
557 DRIVER = 'test_psa_crypto_config_accel_ffdh'
558 IGNORED_SUITES = ['dhm']
559 IGNORED_TESTS = {
560 'test_suite_config': [
561 re.compile(r'.*\bMBEDTLS_DHM_C\b.*'),
562 ],
563 'test_suite_platform': [
564 # Incompatible with sanitizers (e.g. ASan). If the driver
565 # component uses a sanitizer but the reference component
566 # doesn't, we have a PASS vs SKIP mismatch.
567 'Check mbedtls_calloc overallocation',
568 ],
569 }
570
Gilles Peskine45a32b12024-10-03 18:42:37 +0200571class DriverVSReference_tfm_config(outcome_analysis.DriverVSReference):
Gilles Peskine92cc8db2024-09-16 20:14:26 +0200572 REFERENCE = 'test_tfm_config_no_p256m'
573 DRIVER = 'test_tfm_config_p256m_driver_accel_ec'
574 IGNORED_SUITES = [
575 # Modules replaced by drivers
576 'asn1parse', 'asn1write',
577 'ecp', 'ecdsa', 'ecdh', 'ecjpake',
578 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw',
579 'bignum.generated', 'bignum.misc',
Gilles Peskine9a094432024-10-29 20:55:11 +0100580 # Unit tests for the built-in implementation
581 'psa_crypto_ecp',
Gilles Peskine92cc8db2024-09-16 20:14:26 +0200582 ]
583 IGNORED_TESTS = {
584 'test_suite_config': [
585 re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'),
586 re.compile(r'.*\bMBEDTLS_(ASN1\w+)_C\b.*'),
587 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECP)_.*'),
588 re.compile(r'.*\bMBEDTLS_PSA_P256M_DRIVER_ENABLED\b.*')
589 ],
590 'test_suite_config.crypto_combinations': [
591 'Config: ECC: Weierstrass curves only',
592 ],
593 'test_suite_platform': [
594 # Incompatible with sanitizers (e.g. ASan). If the driver
595 # component uses a sanitizer but the reference component
596 # doesn't, we have a PASS vs SKIP mismatch.
597 'Check mbedtls_calloc overallocation',
598 ],
599 # See ecp_light_only
600 'test_suite_random': [
601 'PSA classic wrapper: ECDSA signature (SECP256R1)',
602 ],
603 }
604
Gilles Peskine45a32b12024-10-03 18:42:37 +0200605class DriverVSReference_rsa(outcome_analysis.DriverVSReference):
Gilles Peskine92cc8db2024-09-16 20:14:26 +0200606 REFERENCE = 'test_psa_crypto_config_reference_rsa_crypto'
607 DRIVER = 'test_psa_crypto_config_accel_rsa_crypto'
608 IGNORED_SUITES = [
609 # Modules replaced by drivers.
610 'rsa', 'pkcs1_v15', 'pkcs1_v21',
611 # We temporarily don't care about PK stuff.
612 'pk', 'pkwrite', 'pkparse'
613 ]
614 IGNORED_TESTS = {
615 'test_suite_config': [
616 re.compile(r'.*\bMBEDTLS_(PKCS1|RSA)_.*'),
617 re.compile(r'.*\bMBEDTLS_GENPRIME\b.*')
618 ],
619 'test_suite_platform': [
620 # Incompatible with sanitizers (e.g. ASan). If the driver
621 # component uses a sanitizer but the reference component
622 # doesn't, we have a PASS vs SKIP mismatch.
623 'Check mbedtls_calloc overallocation',
624 ],
625 # Following tests depend on RSA_C but are not about
626 # them really, just need to know some error code is there.
627 'test_suite_error': [
628 'Low and high error',
629 'Single high error'
630 ],
631 # Constant time operations only used for PKCS1_V15
632 'test_suite_constant_time': [
633 re.compile(r'mbedtls_ct_zeroize_if .*'),
634 re.compile(r'mbedtls_ct_memmove_left .*')
635 ],
636 'test_suite_psa_crypto': [
637 # We don't support generate_key_custom entry points
638 # in drivers yet.
639 re.compile(r'PSA generate key custom: RSA, e=.*'),
640 re.compile(r'PSA generate key ext: RSA, e=.*'),
641 ],
642 }
643
Gilles Peskine45a32b12024-10-03 18:42:37 +0200644class DriverVSReference_block_cipher_dispatch(outcome_analysis.DriverVSReference):
Gilles Peskine92cc8db2024-09-16 20:14:26 +0200645 REFERENCE = 'test_full_block_cipher_legacy_dispatch'
646 DRIVER = 'test_full_block_cipher_psa_dispatch'
647 IGNORED_SUITES = [
648 # Skipped in the accelerated component
649 'aes', 'aria', 'camellia',
650 # These require AES_C, ARIA_C or CAMELLIA_C to be enabled in
651 # order for the cipher module (actually cipher_wrapper) to work
652 # properly. However these symbols are disabled in the accelerated
653 # component so we ignore them.
654 'cipher.ccm', 'cipher.gcm', 'cipher.aes', 'cipher.aria',
655 'cipher.camellia',
656 ]
657 IGNORED_TESTS = {
658 'test_suite_config': [
659 re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA)_.*'),
660 re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'),
661 ],
662 'test_suite_cmac': [
663 # Following tests require AES_C/ARIA_C/CAMELLIA_C to be enabled,
664 # but these are not available in the accelerated component.
665 'CMAC null arguments',
666 re.compile('CMAC.* (AES|ARIA|Camellia).*'),
667 ],
668 'test_suite_cipher.padding': [
669 # Following tests require AES_C/CAMELLIA_C to be enabled,
670 # but these are not available in the accelerated component.
671 re.compile('Set( non-existent)? padding with (AES|CAMELLIA).*'),
672 ],
673 'test_suite_pkcs5': [
674 # The AES part of PKCS#5 PBES2 is not yet supported.
675 # The rest of PKCS#5 (PBKDF2) works, though.
676 re.compile(r'PBES2 .* AES-.*')
677 ],
678 'test_suite_pkparse': [
679 # PEM (called by pkparse) requires AES_C in order to decrypt
680 # the key, but this is not available in the accelerated
681 # component.
682 re.compile('Parse RSA Key.*(password|AES-).*'),
683 ],
684 'test_suite_pem': [
685 # Following tests require AES_C, but this is diabled in the
686 # accelerated component.
687 re.compile('PEM read .*AES.*'),
688 'PEM read (unknown encryption algorithm)',
689 ],
690 'test_suite_error': [
691 # Following tests depend on AES_C but are not about them
692 # really, just need to know some error code is there.
693 'Single low error',
694 'Low and high error',
695 ],
696 'test_suite_version': [
697 # Similar to test_suite_error above.
698 'Check for MBEDTLS_AES_C when already present',
699 ],
700 'test_suite_platform': [
701 # Incompatible with sanitizers (e.g. ASan). If the driver
702 # component uses a sanitizer but the reference component
703 # doesn't, we have a PASS vs SKIP mismatch.
704 'Check mbedtls_calloc overallocation',
705 ],
706 }
707
708#pylint: enable=invalid-name,missing-class-docstring
709
710
Przemek Stekiel6856f4c2022-11-09 10:50:29 +0100711# List of tasks with a function that can handle this task and additional arguments if required
Valerio Settidfd7ca62023-10-09 16:30:11 +0200712KNOWN_TASKS = {
Gilles Peskine0316f102024-09-16 19:15:29 +0200713 'analyze_coverage': CoverageTask,
Gilles Peskine92cc8db2024-09-16 20:14:26 +0200714 'analyze_driver_vs_reference_hash': DriverVSReference_hash,
715 'analyze_driver_vs_reference_hmac': DriverVSReference_hmac,
716 'analyze_driver_vs_reference_cipher_aead_cmac': DriverVSReference_cipher_aead_cmac,
717 'analyze_driver_vs_reference_ecp_light_only': DriverVSReference_ecp_light_only,
718 'analyze_driver_vs_reference_no_ecp_at_all': DriverVSReference_no_ecp_at_all,
719 'analyze_driver_vs_reference_ecc_no_bignum': DriverVSReference_ecc_no_bignum,
720 'analyze_driver_vs_reference_ecc_ffdh_no_bignum': DriverVSReference_ecc_ffdh_no_bignum,
721 'analyze_driver_vs_reference_ffdh_alg': DriverVSReference_ffdh_alg,
722 'analyze_driver_vs_reference_tfm_config': DriverVSReference_tfm_config,
723 'analyze_driver_vs_reference_rsa': DriverVSReference_rsa,
724 'analyze_block_cipher_dispatch': DriverVSReference_block_cipher_dispatch,
Przemek Stekiel4d13c832022-10-26 16:11:26 +0200725}
Przemek Stekiel4d13c832022-10-26 16:11:26 +0200726
Gilles Peskine15c2cbf2020-06-25 18:36:28 +0200727if __name__ == '__main__':
Gilles Peskine45a32b12024-10-03 18:42:37 +0200728 outcome_analysis.main(KNOWN_TASKS)