Gilles Peskine | 15c2cbf | 2020-06-25 18:36:28 +0200 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | |
| 3 | """Analyze the test outcomes from a full CI run. |
| 4 | |
| 5 | This script can also run on outcomes from a partial run, but the results are |
| 6 | less likely to be useful. |
| 7 | """ |
| 8 | |
| 9 | import argparse |
| 10 | import sys |
| 11 | import traceback |
Przemek Stekiel | 85c54ea | 2022-11-17 11:50:23 +0100 | [diff] [blame] | 12 | import re |
Gilles Peskine | 15c2cbf | 2020-06-25 18:36:28 +0200 | [diff] [blame] | 13 | |
Gilles Peskine | 8d3c70a | 2020-06-25 18:37:43 +0200 | [diff] [blame] | 14 | import check_test_cases |
| 15 | |
Gilles Peskine | 15c2cbf | 2020-06-25 18:36:28 +0200 | [diff] [blame] | 16 | class Results: |
| 17 | """Process analysis results.""" |
| 18 | |
| 19 | def __init__(self): |
| 20 | self.error_count = 0 |
| 21 | self.warning_count = 0 |
| 22 | |
| 23 | @staticmethod |
| 24 | def log(fmt, *args, **kwargs): |
| 25 | sys.stderr.write((fmt + '\n').format(*args, **kwargs)) |
| 26 | |
| 27 | def error(self, fmt, *args, **kwargs): |
| 28 | self.log('Error: ' + fmt, *args, **kwargs) |
| 29 | self.error_count += 1 |
| 30 | |
| 31 | def warning(self, fmt, *args, **kwargs): |
| 32 | self.log('Warning: ' + fmt, *args, **kwargs) |
| 33 | self.warning_count += 1 |
| 34 | |
| 35 | class TestCaseOutcomes: |
| 36 | """The outcomes of one test case across many configurations.""" |
| 37 | # pylint: disable=too-few-public-methods |
| 38 | |
| 39 | def __init__(self): |
Gilles Peskine | 3d863f2 | 2020-06-26 13:02:30 +0200 | [diff] [blame] | 40 | # Collect a list of witnesses of the test case succeeding or failing. |
| 41 | # Currently we don't do anything with witnesses except count them. |
| 42 | # The format of a witness is determined by the read_outcome_file |
| 43 | # function; it's the platform and configuration joined by ';'. |
Gilles Peskine | 15c2cbf | 2020-06-25 18:36:28 +0200 | [diff] [blame] | 44 | self.successes = [] |
| 45 | self.failures = [] |
| 46 | |
| 47 | def hits(self): |
| 48 | """Return the number of times a test case has been run. |
| 49 | |
| 50 | This includes passes and failures, but not skips. |
| 51 | """ |
| 52 | return len(self.successes) + len(self.failures) |
| 53 | |
Gilles Peskine | 8d3c70a | 2020-06-25 18:37:43 +0200 | [diff] [blame] | 54 | def analyze_coverage(results, outcomes): |
| 55 | """Check that all available test cases are executed at least once.""" |
Gilles Peskine | 686c292 | 2022-01-07 15:58:38 +0100 | [diff] [blame] | 56 | available = check_test_cases.collect_available_test_cases() |
Gilles Peskine | 8d3c70a | 2020-06-25 18:37:43 +0200 | [diff] [blame] | 57 | for key in available: |
| 58 | hits = outcomes[key].hits() if key in outcomes else 0 |
| 59 | if hits == 0: |
| 60 | # Make this a warning, not an error, as long as we haven't |
| 61 | # fixed this branch to have full coverage of test cases. |
| 62 | results.warning('Test case not executed: {}', key) |
| 63 | |
Valerio Setti | 3002c99 | 2023-01-18 17:28:36 +0100 | [diff] [blame] | 64 | def analyze_driver_vs_reference(outcomes, component_ref, component_driver, |
| 65 | ignored_suites, ignored_test=None): |
Przemek Stekiel | 4e95590 | 2022-10-21 13:42:08 +0200 | [diff] [blame] | 66 | """Check that all tests executed in the reference component are also |
| 67 | executed in the corresponding driver component. |
Valerio Setti | 3002c99 | 2023-01-18 17:28:36 +0100 | [diff] [blame] | 68 | Skip: |
| 69 | - full test suites provided in ignored_suites list |
| 70 | - only some specific test inside a test suite, for which the corresponding |
| 71 | output string is provided |
Przemek Stekiel | 4e95590 | 2022-10-21 13:42:08 +0200 | [diff] [blame] | 72 | """ |
Przemek Stekiel | 4e95590 | 2022-10-21 13:42:08 +0200 | [diff] [blame] | 73 | available = check_test_cases.collect_available_test_cases() |
| 74 | result = True |
| 75 | |
| 76 | for key in available: |
Przemek Stekiel | 4e95590 | 2022-10-21 13:42:08 +0200 | [diff] [blame] | 77 | # Continue if test was not executed by any component |
| 78 | hits = outcomes[key].hits() if key in outcomes else 0 |
Przemek Stekiel | c86dedf | 2022-10-24 09:16:04 +0200 | [diff] [blame] | 79 | if hits == 0: |
Przemek Stekiel | 4e95590 | 2022-10-21 13:42:08 +0200 | [diff] [blame] | 80 | continue |
Valerio Setti | 00c1ccb | 2023-02-02 11:33:31 +0100 | [diff] [blame] | 81 | # Skip ignored test suites |
| 82 | full_test_suite = key.split(';')[0] # retrieve full test suite name |
| 83 | test_string = key.split(';')[1] # retrieve the text string of this test |
| 84 | test_suite = full_test_suite.split('.')[0] # retrieve main part of test suite name |
Manuel Pégourié-Gonnard | 7d381f5 | 2023-03-17 15:13:08 +0100 | [diff] [blame] | 85 | if test_suite in ignored_suites or full_test_suite in ignored_suites: |
Valerio Setti | 00c1ccb | 2023-02-02 11:33:31 +0100 | [diff] [blame] | 86 | continue |
Valerio Setti | 3002c99 | 2023-01-18 17:28:36 +0100 | [diff] [blame] | 87 | if ((full_test_suite in ignored_test) and |
| 88 | (test_string in ignored_test[full_test_suite])): |
| 89 | continue |
Przemek Stekiel | 4e95590 | 2022-10-21 13:42:08 +0200 | [diff] [blame] | 90 | # Search for tests that run in reference component and not in driver component |
| 91 | driver_test_passed = False |
| 92 | reference_test_passed = False |
| 93 | for entry in outcomes[key].successes: |
Przemek Stekiel | 51f30ff | 2022-11-09 12:07:29 +0100 | [diff] [blame] | 94 | if component_driver in entry: |
Przemek Stekiel | 4e95590 | 2022-10-21 13:42:08 +0200 | [diff] [blame] | 95 | driver_test_passed = True |
Przemek Stekiel | 51f30ff | 2022-11-09 12:07:29 +0100 | [diff] [blame] | 96 | if component_ref in entry: |
Przemek Stekiel | 4e95590 | 2022-10-21 13:42:08 +0200 | [diff] [blame] | 97 | reference_test_passed = True |
Manuel Pégourié-Gonnard | c6967d2 | 2022-12-30 13:40:34 +0100 | [diff] [blame] | 98 | if(reference_test_passed and not driver_test_passed): |
Valerio Setti | 3951d1b | 2023-03-13 18:37:34 +0100 | [diff] [blame] | 99 | Results.log(key) |
Przemek Stekiel | 4e95590 | 2022-10-21 13:42:08 +0200 | [diff] [blame] | 100 | result = False |
| 101 | return result |
| 102 | |
Gilles Peskine | 15c2cbf | 2020-06-25 18:36:28 +0200 | [diff] [blame] | 103 | def analyze_outcomes(outcomes): |
| 104 | """Run all analyses on the given outcome collection.""" |
| 105 | results = Results() |
Gilles Peskine | 8d3c70a | 2020-06-25 18:37:43 +0200 | [diff] [blame] | 106 | analyze_coverage(results, outcomes) |
Gilles Peskine | 15c2cbf | 2020-06-25 18:36:28 +0200 | [diff] [blame] | 107 | return results |
| 108 | |
| 109 | def read_outcome_file(outcome_file): |
| 110 | """Parse an outcome file and return an outcome collection. |
| 111 | |
| 112 | An outcome collection is a dictionary mapping keys to TestCaseOutcomes objects. |
| 113 | The keys are the test suite name and the test case description, separated |
| 114 | by a semicolon. |
| 115 | """ |
| 116 | outcomes = {} |
| 117 | with open(outcome_file, 'r', encoding='utf-8') as input_file: |
| 118 | for line in input_file: |
| 119 | (platform, config, suite, case, result, _cause) = line.split(';') |
| 120 | key = ';'.join([suite, case]) |
| 121 | setup = ';'.join([platform, config]) |
| 122 | if key not in outcomes: |
| 123 | outcomes[key] = TestCaseOutcomes() |
| 124 | if result == 'PASS': |
| 125 | outcomes[key].successes.append(setup) |
| 126 | elif result == 'FAIL': |
| 127 | outcomes[key].failures.append(setup) |
| 128 | return outcomes |
| 129 | |
Przemek Stekiel | 4d13c83 | 2022-10-26 16:11:26 +0200 | [diff] [blame] | 130 | def do_analyze_coverage(outcome_file, args): |
Przemek Stekiel | 6856f4c | 2022-11-09 10:50:29 +0100 | [diff] [blame] | 131 | """Perform coverage analysis.""" |
Przemek Stekiel | 4d13c83 | 2022-10-26 16:11:26 +0200 | [diff] [blame] | 132 | del args # unused |
Gilles Peskine | 15c2cbf | 2020-06-25 18:36:28 +0200 | [diff] [blame] | 133 | outcomes = read_outcome_file(outcome_file) |
Valerio Setti | 3951d1b | 2023-03-13 18:37:34 +0100 | [diff] [blame] | 134 | Results.log("\n*** Analyze coverage ***\n") |
Przemek Stekiel | 4e95590 | 2022-10-21 13:42:08 +0200 | [diff] [blame] | 135 | results = analyze_outcomes(outcomes) |
Przemek Stekiel | c86dedf | 2022-10-24 09:16:04 +0200 | [diff] [blame] | 136 | return results.error_count == 0 |
Przemek Stekiel | 4e95590 | 2022-10-21 13:42:08 +0200 | [diff] [blame] | 137 | |
Przemek Stekiel | 4d13c83 | 2022-10-26 16:11:26 +0200 | [diff] [blame] | 138 | def do_analyze_driver_vs_reference(outcome_file, args): |
Przemek Stekiel | 4e95590 | 2022-10-21 13:42:08 +0200 | [diff] [blame] | 139 | """Perform driver vs reference analyze.""" |
Valerio Setti | 3002c99 | 2023-01-18 17:28:36 +0100 | [diff] [blame] | 140 | ignored_suites = ['test_suite_' + x for x in args['ignored_suites']] |
Przemek Stekiel | 51f30ff | 2022-11-09 12:07:29 +0100 | [diff] [blame] | 141 | |
Przemek Stekiel | 4e95590 | 2022-10-21 13:42:08 +0200 | [diff] [blame] | 142 | outcomes = read_outcome_file(outcome_file) |
Valerio Setti | 3951d1b | 2023-03-13 18:37:34 +0100 | [diff] [blame] | 143 | Results.log("\n*** Analyze driver {} vs reference {} ***\n".format( |
Manuel Pégourié-Gonnard | c6967d2 | 2022-12-30 13:40:34 +0100 | [diff] [blame] | 144 | args['component_driver'], args['component_ref'])) |
Przemek Stekiel | 51f30ff | 2022-11-09 12:07:29 +0100 | [diff] [blame] | 145 | return analyze_driver_vs_reference(outcomes, args['component_ref'], |
Valerio Setti | 3002c99 | 2023-01-18 17:28:36 +0100 | [diff] [blame] | 146 | args['component_driver'], ignored_suites, |
| 147 | args['ignored_tests']) |
Gilles Peskine | 15c2cbf | 2020-06-25 18:36:28 +0200 | [diff] [blame] | 148 | |
Przemek Stekiel | 6856f4c | 2022-11-09 10:50:29 +0100 | [diff] [blame] | 149 | # List of tasks with a function that can handle this task and additional arguments if required |
Przemek Stekiel | 4d13c83 | 2022-10-26 16:11:26 +0200 | [diff] [blame] | 150 | TASKS = { |
| 151 | 'analyze_coverage': { |
| 152 | 'test_function': do_analyze_coverage, |
Manuel Pégourié-Gonnard | 10e3963 | 2022-12-29 12:29:09 +0100 | [diff] [blame] | 153 | 'args': {} |
| 154 | }, |
| 155 | # How to use analyze_driver_vs_reference_xxx locally: |
| 156 | # 1. tests/scripts/all.sh --outcome-file "$PWD/out.csv" <component_ref> <component_driver> |
| 157 | # 2. tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx |
Przemek Stekiel | 4d13c83 | 2022-10-26 16:11:26 +0200 | [diff] [blame] | 158 | 'analyze_driver_vs_reference_hash': { |
| 159 | 'test_function': do_analyze_driver_vs_reference, |
| 160 | 'args': { |
Przemek Stekiel | 51f30ff | 2022-11-09 12:07:29 +0100 | [diff] [blame] | 161 | 'component_ref': 'test_psa_crypto_config_reference_hash_use_psa', |
| 162 | 'component_driver': 'test_psa_crypto_config_accel_hash_use_psa', |
Manuel Pégourié-Gonnard | 10e3963 | 2022-12-29 12:29:09 +0100 | [diff] [blame] | 163 | 'ignored_suites': [ |
| 164 | 'shax', 'mdx', # the software implementations that are being excluded |
Manuel Pégourié-Gonnard | 7d381f5 | 2023-03-17 15:13:08 +0100 | [diff] [blame] | 165 | 'md.psa', # purposefully depends on whether drivers are present |
Valerio Setti | 3002c99 | 2023-01-18 17:28:36 +0100 | [diff] [blame] | 166 | ], |
| 167 | 'ignored_tests': { |
| 168 | } |
| 169 | } |
| 170 | }, |
Manuel Pégourié-Gonnard | 10e3963 | 2022-12-29 12:29:09 +0100 | [diff] [blame] | 171 | 'analyze_driver_vs_reference_ecdsa': { |
| 172 | 'test_function': do_analyze_driver_vs_reference, |
| 173 | 'args': { |
| 174 | 'component_ref': 'test_psa_crypto_config_reference_ecdsa_use_psa', |
| 175 | 'component_driver': 'test_psa_crypto_config_accel_ecdsa_use_psa', |
| 176 | 'ignored_suites': [ |
| 177 | 'ecdsa', # the software implementation that's excluded |
Valerio Setti | 3002c99 | 2023-01-18 17:28:36 +0100 | [diff] [blame] | 178 | ], |
| 179 | 'ignored_tests': { |
Valerio Setti | 9cb0f7a | 2023-01-18 17:29:29 +0100 | [diff] [blame] | 180 | 'test_suite_random': [ |
| 181 | 'PSA classic wrapper: ECDSA signature (SECP256R1)', |
| 182 | ], |
Valerio Setti | 3002c99 | 2023-01-18 17:28:36 +0100 | [diff] [blame] | 183 | } |
| 184 | } |
| 185 | }, |
Manuel Pégourié-Gonnard | e91bcf3 | 2023-02-21 13:07:19 +0100 | [diff] [blame] | 186 | 'analyze_driver_vs_reference_ecdh': { |
| 187 | 'test_function': do_analyze_driver_vs_reference, |
| 188 | 'args': { |
| 189 | 'component_ref': 'test_psa_crypto_config_reference_ecdh_use_psa', |
| 190 | 'component_driver': 'test_psa_crypto_config_accel_ecdh_use_psa', |
| 191 | 'ignored_suites': [ |
| 192 | 'ecdh', # the software implementation that's excluded |
| 193 | ], |
| 194 | 'ignored_tests': { |
Manuel Pégourié-Gonnard | e91bcf3 | 2023-02-21 13:07:19 +0100 | [diff] [blame] | 195 | } |
| 196 | } |
| 197 | }, |
Valerio Setti | d0fffc5 | 2023-03-13 16:08:03 +0100 | [diff] [blame] | 198 | 'analyze_driver_vs_reference_ecjpake': { |
| 199 | 'test_function': do_analyze_driver_vs_reference, |
| 200 | 'args': { |
| 201 | 'component_ref': 'test_psa_crypto_config_reference_ecjpake_use_psa', |
| 202 | 'component_driver': 'test_psa_crypto_config_accel_ecjpake_use_psa', |
| 203 | 'ignored_suites': [ |
| 204 | 'ecjpake', # the software implementation that's excluded |
| 205 | ], |
| 206 | 'ignored_tests': { |
| 207 | } |
| 208 | } |
| 209 | }, |
Valerio Setti | 42d5f19 | 2023-03-20 13:54:41 +0100 | [diff] [blame^] | 210 | 'analyze_driver_vs_reference_all_ec_algs': { |
| 211 | 'test_function': do_analyze_driver_vs_reference, |
| 212 | 'args': { |
| 213 | 'component_ref': 'test_psa_crypto_config_reference_all_ec_algs_use_psa', |
| 214 | 'component_driver': 'test_psa_crypto_config_accel_all_ec_algs_use_psa', |
| 215 | # ignore the suites of the accelerated components |
| 216 | 'ignored_suites': [ |
| 217 | 'ecdsa', |
| 218 | 'ecdh', |
| 219 | 'ecjpake', |
| 220 | ], |
| 221 | 'ignored_tests': { |
| 222 | 'test_suite_random': [ |
| 223 | 'PSA classic wrapper: ECDSA signature (SECP256R1)', |
| 224 | ], |
| 225 | } |
| 226 | } |
| 227 | }, |
Przemek Stekiel | 4d13c83 | 2022-10-26 16:11:26 +0200 | [diff] [blame] | 228 | } |
Przemek Stekiel | 4d13c83 | 2022-10-26 16:11:26 +0200 | [diff] [blame] | 229 | |
Gilles Peskine | 15c2cbf | 2020-06-25 18:36:28 +0200 | [diff] [blame] | 230 | def main(): |
| 231 | try: |
| 232 | parser = argparse.ArgumentParser(description=__doc__) |
Przemek Stekiel | 58bbc23 | 2022-10-24 08:10:10 +0200 | [diff] [blame] | 233 | parser.add_argument('outcomes', metavar='OUTCOMES.CSV', |
Gilles Peskine | 15c2cbf | 2020-06-25 18:36:28 +0200 | [diff] [blame] | 234 | help='Outcome file to analyze') |
Przemek Stekiel | 542d932 | 2022-11-17 09:43:34 +0100 | [diff] [blame] | 235 | parser.add_argument('task', default='all', nargs='?', |
Przemek Stekiel | 992de3c | 2022-11-09 13:54:49 +0100 | [diff] [blame] | 236 | help='Analysis to be done. By default, run all tasks. ' |
| 237 | 'With one or more TASK, run only those. ' |
| 238 | 'TASK can be the name of a single task or ' |
Przemek Stekiel | 85c54ea | 2022-11-17 11:50:23 +0100 | [diff] [blame] | 239 | 'comma/space-separated list of tasks. ') |
Przemek Stekiel | 992de3c | 2022-11-09 13:54:49 +0100 | [diff] [blame] | 240 | parser.add_argument('--list', action='store_true', |
| 241 | help='List all available tasks and exit.') |
Gilles Peskine | 15c2cbf | 2020-06-25 18:36:28 +0200 | [diff] [blame] | 242 | options = parser.parse_args() |
Przemek Stekiel | 4e95590 | 2022-10-21 13:42:08 +0200 | [diff] [blame] | 243 | |
Przemek Stekiel | 992de3c | 2022-11-09 13:54:49 +0100 | [diff] [blame] | 244 | if options.list: |
| 245 | for task in TASKS: |
Valerio Setti | 3951d1b | 2023-03-13 18:37:34 +0100 | [diff] [blame] | 246 | Results.log(task) |
Przemek Stekiel | 992de3c | 2022-11-09 13:54:49 +0100 | [diff] [blame] | 247 | sys.exit(0) |
| 248 | |
Przemek Stekiel | 4d13c83 | 2022-10-26 16:11:26 +0200 | [diff] [blame] | 249 | result = True |
Przemek Stekiel | 4e95590 | 2022-10-21 13:42:08 +0200 | [diff] [blame] | 250 | |
Przemek Stekiel | 4d13c83 | 2022-10-26 16:11:26 +0200 | [diff] [blame] | 251 | if options.task == 'all': |
Przemek Stekiel | d3068af | 2022-11-14 16:15:19 +0100 | [diff] [blame] | 252 | tasks = TASKS.keys() |
Przemek Stekiel | 992de3c | 2022-11-09 13:54:49 +0100 | [diff] [blame] | 253 | else: |
Przemek Stekiel | 85c54ea | 2022-11-17 11:50:23 +0100 | [diff] [blame] | 254 | tasks = re.split(r'[, ]+', options.task) |
Przemek Stekiel | 992de3c | 2022-11-09 13:54:49 +0100 | [diff] [blame] | 255 | |
Przemek Stekiel | d3068af | 2022-11-14 16:15:19 +0100 | [diff] [blame] | 256 | for task in tasks: |
| 257 | if task not in TASKS: |
Valerio Setti | 3951d1b | 2023-03-13 18:37:34 +0100 | [diff] [blame] | 258 | Results.log('Error: invalid task: {}'.format(task)) |
Przemek Stekiel | d3068af | 2022-11-14 16:15:19 +0100 | [diff] [blame] | 259 | sys.exit(1) |
Przemek Stekiel | 992de3c | 2022-11-09 13:54:49 +0100 | [diff] [blame] | 260 | |
| 261 | for task in TASKS: |
| 262 | if task in tasks: |
Przemek Stekiel | 4d13c83 | 2022-10-26 16:11:26 +0200 | [diff] [blame] | 263 | if not TASKS[task]['test_function'](options.outcomes, TASKS[task]['args']): |
| 264 | result = False |
Przemek Stekiel | 4e95590 | 2022-10-21 13:42:08 +0200 | [diff] [blame] | 265 | |
Przemek Stekiel | c86dedf | 2022-10-24 09:16:04 +0200 | [diff] [blame] | 266 | if result is False: |
Gilles Peskine | 15c2cbf | 2020-06-25 18:36:28 +0200 | [diff] [blame] | 267 | sys.exit(1) |
Valerio Setti | 3951d1b | 2023-03-13 18:37:34 +0100 | [diff] [blame] | 268 | Results.log("SUCCESS :-)") |
Gilles Peskine | 15c2cbf | 2020-06-25 18:36:28 +0200 | [diff] [blame] | 269 | except Exception: # pylint: disable=broad-except |
| 270 | # Print the backtrace and exit explicitly with our chosen status. |
| 271 | traceback.print_exc() |
| 272 | sys.exit(120) |
| 273 | |
| 274 | if __name__ == '__main__': |
| 275 | main() |