Test program for psa_constant_names
Test psa_constant_names on many inputs. For each input, find out the
numerical value by compiling and running a C program, pass the
numerical value to psa_constant_names and compare the output with the
original input.
Gather inputs by parsing psa/crypto.h and
test_suite_psa_crypto_metadata.data. For macros that take an argument,
list some possible arguments using the parsed data.
diff --git a/tests/scripts/test_psa_constant_names.py b/tests/scripts/test_psa_constant_names.py
new file mode 100755
index 0000000..5929242
--- /dev/null
+++ b/tests/scripts/test_psa_constant_names.py
@@ -0,0 +1,241 @@
+#!/usr/bin/env python3
+'''Test the program psa_constant_names.
+Gather constant names from header files and test cases. Compile a C program
+to print out their numerical values, feed these numerical values to
+psa_constant_names, and check that the output is the original name.
+Return 0 if all test cases pass, 1 if the output was not always as expected,
+or 1 (with a Python backtrace) if there was an operational error.'''
+
+import argparse
+import itertools
+import os
+import platform
+import re
+import subprocess
+import sys
+import tempfile
+
+class Inputs:
+ '''Accumulate information about macros to test.
+This includes macro names as well as information about their arguments
+when applicable.'''
+ def __init__(self):
+ # Sets of names per type
+ self.statuses = set(['PSA_SUCCESS'])
+ self.algorithms = set(['0xffffffff'])
+ self.ecc_curves = set(['0xffff'])
+ self.key_types = set(['0xffffffff'])
+ self.key_usage_flags = set(['0x80000000'])
+ # Hard-coded value for an unknown hash algorithm
+ self.hash_algorithms = set(['0x010000ff'])
+ # Identifier prefixes
+ self.table_by_prefix = {
+ 'ERROR': self.statuses,
+ 'ALG': self.algorithms,
+ 'CURVE': self.ecc_curves,
+ 'KEY_TYPE': self.key_types,
+ 'KEY_USAGE': self.key_usage_flags,
+ }
+ # macro name -> list of argument names
+ self.argspecs = {}
+ # argument name -> list of values
+ self.arguments_for = {}
+
+ def gather_arguments(self):
+ '''Populate the list of values for macro arguments.
+Call this after parsing all the inputs.'''
+ self.arguments_for['hash_alg'] = sorted(self.hash_algorithms)
+ self.arguments_for['curve'] = sorted(self.ecc_curves)
+
+ def format_arguments(self, name, arguments):
+ '''Format a macro call with arguments..'''
+ return name + '(' + ', '.join(arguments) + ')'
+
+ def distribute_arguments(self, name):
+ '''Generate macro calls with each tested argument set.
+If name is a macro without arguments, just yield "name".
+If name is a macro with arguments, yield a series of "name(arg1,...,argN)"
+where each argument takes each possible value at least once.'''
+ if name not in self.argspecs:
+ yield name
+ return
+ argspec = self.argspecs[name]
+ if argspec == []:
+ yield name + '()'
+ return
+ argument_lists = [self.arguments_for[arg] for arg in argspec]
+ arguments = [values[0] for values in argument_lists]
+ yield self.format_arguments(name, arguments)
+ for i in range(len(arguments)):
+ for value in argument_lists[i][1:]:
+ arguments[i] = value
+ yield self.format_arguments(name, arguments)
+ arguments[i] = argument_lists[0]
+
+ # Regex for interesting header lines.
+ # Groups: 1=macro name, 2=type, 3=argument list (optional).
+ header_line_re = \
+ re.compile(r'#define +' +
+ r'(PSA_((?:KEY_)?[A-Z]+)_\w+)' +
+ r'(?:\(([^\n()]*)\))?')
+ # Regex of macro names to exclude.
+ excluded_name_re = re.compile('_(?:GET|IS|OF)_|_(?:BASE|FLAG|MASK)\Z')
+ argument_split_re = re.compile(r' *, *')
+ def parse_header_line(self, line):
+ '''Parse a C header line, looking for "#define PSA_xxx".'''
+ m = re.match(self.header_line_re, line)
+ if not m:
+ return
+ name = m.group(1)
+ if re.search(self.excluded_name_re, name):
+ return
+ dest = self.table_by_prefix.get(m.group(2))
+ if dest is None:
+ return
+ dest.add(name)
+ if m.group(3):
+ self.argspecs[name] = re.split(self.argument_split_re, m.group(3))
+
+ def parse_header(self, filename):
+ '''Parse a C header file, looking for "#define PSA_xxx".'''
+ with open(filename, 'r') as input:
+ for line in input:
+ self.parse_header_line(line)
+
+ def add_test_case_line(self, function, argument):
+ '''Parse a test case data line, looking for algorithm metadata tests.'''
+ if function.endswith('_algorithm'):
+ self.algorithms.add(argument)
+ if function == 'hash_algorithm':
+ self.hash_algorithms.add(argument)
+ elif function == 'key_type':
+ self.key_types.add(argument)
+ elif function == 'ecc_key_types':
+ self.ecc_curves.add(argument)
+
+ # Regex matching a *.data line containing a test function call and
+ # its arguments. The actual definition is partly positional, but this
+ # regex is good enough in practice.
+ test_case_line_re = re.compile('(?!depends_on:)(\w+):([^\n :][^:\n]*)')
+ def parse_test_cases(self, filename):
+ '''Parse a test case file (*.data), looking for algorithm metadata tests.'''
+ with open(filename, 'r') as input:
+ for line in input:
+ m = re.match(self.test_case_line_re, line)
+ if m:
+ self.add_test_case_line(m.group(1), m.group(2))
+
+def gather_inputs(headers, test_suites):
+ '''Read the list of inputs to test psa_constant_names with.'''
+ inputs = Inputs()
+ for header in headers:
+ inputs.parse_header(header)
+ for test_cases in test_suites:
+ inputs.parse_test_cases(test_cases)
+ inputs.gather_arguments()
+ return inputs
+
+def remove_file_if_exists(filename):
+ '''Remove the specified file, ignoring errors.'''
+ if not filename:
+ return
+ try:
+ os.remove(filename)
+ except:
+ pass
+
+def run_c(options, names):
+ '''Generate and run a program to print out numerical values for names.'''
+ c_name = None
+ exe_name = None
+ try:
+ c_fd, c_name = tempfile.mkstemp(suffix='.c',
+ dir='programs/psa')
+ exe_suffix = '.exe' if platform.system() == 'Windows' else ''
+ exe_name = c_name[:-2] + exe_suffix
+ remove_file_if_exists(exe_name)
+ c_file = os.fdopen(c_fd, 'w', encoding='ascii')
+ c_file.write('''/* Generated by test_psa_constant_names.py */
+#include <stdio.h>
+#include <psa/crypto.h>
+int main(void)
+{
+''')
+ for name in names:
+ c_file.write(' printf("0x%08x\\n", {});\n'.format(name))
+ c_file.write(''' return 0;
+}
+''')
+ c_file.close()
+ cc = os.getenv('CC', 'cc')
+ subprocess.check_call([cc] +
+ ['-I' + dir for dir in options.include] +
+ ['-o', exe_name, c_name])
+ os.remove(c_name)
+ output = subprocess.check_output([exe_name])
+ return output.decode('ascii').strip().split('\n')
+ finally:
+ remove_file_if_exists(exe_name)
+
+normalize_strip_re = re.compile(r'\s+')
+def normalize(expr):
+ '''Normalize the C expression so as not to care about trivial differences.
+Currently "trivial differences" means whitespace.'''
+ expr = re.sub(normalize_strip_re, '', expr, len(expr))
+ return expr.strip().split('\n')
+
+def do_test(options, inputs, type, names):
+ '''Test psa_constant_names for the specified type.
+Run program on names.
+Use inputs to figure out what arguments to pass to macros that take arguments.'''
+ names = sorted(itertools.chain(*map(inputs.distribute_arguments, names)))
+ values = run_c(options, names)
+ output = subprocess.check_output([options.program, type] + values)
+ outputs = output.decode('ascii').strip().split('\n')
+ errors = [(type, name, value, output)
+ for (name, value, output) in zip(names, values, outputs)
+ if normalize(name) != normalize(output)]
+ return len(names), errors
+
+def report_errors(errors):
+ '''Describe each case where the output is not as expected.'''
+ for type, name, value, output in errors:
+ print('For {} "{}", got "{}" (value: {})'
+ .format(type, name, output, value))
+
+def run_tests(options, inputs):
+ '''Run psa_constant_names on all the gathered inputs.
+Return a tuple (count, errors) where count is the total number of inputs
+that were tested and errors is the list of cases where the output was
+not as expected.'''
+ count = 0
+ errors = []
+ for type, names in [('status', inputs.statuses),
+ ('algorithm', inputs.algorithms),
+ ('ecc_curve', inputs.ecc_curves),
+ ('key_type', inputs.key_types),
+ ('key_usage', inputs.key_usage_flags)]:
+ c, e = do_test(options, inputs, type, names)
+ count += c
+ errors += e
+ return count, errors
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(description=globals()['__doc__'])
+ parser.add_argument('--include', '-I',
+ action='append', default=['include'],
+ help='Directory for header files')
+ parser.add_argument('--program',
+ default='programs/psa/psa_constant_names',
+ help='Program to test')
+ options = parser.parse_args()
+ headers = [os.path.join(options.include[0], 'psa/crypto.h')]
+ test_suites = ['tests/suites/test_suite_psa_crypto_metadata.data']
+ inputs = gather_inputs(headers, test_suites)
+ count, errors = run_tests(options, inputs)
+ report_errors(errors)
+ if errors == []:
+ print('{} test cases PASS'.format(count))
+ else:
+ print('{} test cases, {} FAIL'.format(count, len(errors)))
+ exit(1)