Replace stringly typed data by class: coverage
Work on replacing the stringly typed KNOWN_TASKS by classes for each category
of tasks, with a structure that matches the behavior.
This commit migrates test coverage analysis.
No intended behavior change.
Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py
index 5835c80..98c4afd 100755
--- a/tests/scripts/analyze_outcomes.py
+++ b/tests/scripts/analyze_outcomes.py
@@ -170,11 +170,6 @@
if ignored and suite_case in driver_outcomes.successes:
results.error("uselessly ignored: {}", suite_case)
-def analyze_outcomes(results: Results, outcomes: Outcomes, args) -> None:
- """Run all analyses on the given outcome collection."""
- analyze_coverage(results, outcomes, args['allow_list'],
- args['full_coverage'])
-
def read_outcome_file(outcome_file: str) -> Outcomes:
"""Parse an outcome file and return an outcome collection.
"""
@@ -195,11 +190,6 @@
return outcomes
-def do_analyze_coverage(results: Results, outcomes: Outcomes, args) -> None:
- """Perform coverage analysis."""
- results.new_section("Analyze coverage")
- analyze_outcomes(results, outcomes, args)
-
def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -> None:
"""Perform driver vs reference analyze."""
results.new_section("Analyze driver {} vs reference {}",
@@ -222,6 +212,9 @@
"""
pass
+ def section_name(self) -> str:
+ """The section name to use in results."""
+
def run(self, results: Results, outcomes: Outcomes):
"""Run the analysis on the specified outcomes.
@@ -230,20 +223,34 @@
raise NotImplementedError
+class CoverageTask(Task):
+ """Analyze test coverage."""
+
+ ALLOW_LIST = [
+ # Algorithm not supported yet
+ 'test_suite_psa_crypto_metadata;Asymmetric signature: pure EdDSA',
+ # Algorithm not supported yet
+ 'test_suite_psa_crypto_metadata;Cipher: XTS',
+ ]
+
+ def __init__(self, options) -> None:
+ super().__init__(options)
+ self.full_coverage = options.full_coverage #type: bool
+
+ @staticmethod
+ def section_name() -> str:
+ return "Analyze coverage"
+
+ def run(self, results: Results, outcomes: Outcomes):
+ """Check that all test cases are executed at least once."""
+ analyze_coverage(results, outcomes,
+ self.ALLOW_LIST, self.full_coverage)
+
+
# List of tasks with a function that can handle this task and additional arguments if required
KNOWN_TASKS = {
- 'analyze_coverage': {
- 'test_function': do_analyze_coverage,
- 'args': {
- 'allow_list': [
- # Algorithm not supported yet
- 'test_suite_psa_crypto_metadata;Asymmetric signature: pure EdDSA',
- # Algorithm not supported yet
- 'test_suite_psa_crypto_metadata;Cipher: XTS',
- ],
- 'full_coverage': False,
- }
- },
+ 'analyze_coverage': CoverageTask,
+
# There are 2 options to use analyze_driver_vs_reference_xxx locally:
# 1. Run tests and then analysis:
# - tests/scripts/all.sh --outcome-file "$PWD/out.csv" <component_ref> <component_driver>
@@ -773,8 +780,6 @@
sys.stderr.write('invalid task: {}\n'.format(task))
sys.exit(2)
- KNOWN_TASKS['analyze_coverage']['args']['full_coverage'] = options.full_coverage
-
# If the outcome file exists, parse it once and share the result
# among tasks to improve performance.
# Otherwise, it will be generated by execute_reference_driver_tests.
@@ -805,6 +810,7 @@
test_function(main_results, outcomes, test_args)
else:
task = task_constructor(options)
+ main_results.new_section(task.section_name())
task.run(main_results, outcomes)
main_results.info("Overall results: {} warnings and {} errors",