#!/usr/bin/env python3 #pylint: disable=missing-module-docstring import os import re import shutil import subprocess import sys EXPECTED_FAILURES = { 216, 221, 224, 225, 248, 249, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263 } PSA_ARCH_TESTS_REPO = 'https://github.com/ronald-cron-arm/psa-arch-tests.git' PSA_ARCH_TESTS_REF = 'crypto1.0-3.0' #pylint: disable=too-many-statements def main(): mbedtls_dir = os.getcwd() if not os.path.exists('library/libmbedcrypto.a'): subprocess.check_call(['make', '-C', 'library', 'libmbedcrypto.a']) psa_arch_tests_dir = 'psa-arch-tests' try: os.mkdir(psa_arch_tests_dir) except FileExistsError: pass os.chdir(psa_arch_tests_dir) subprocess.check_call(['git', 'init']) subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF]) subprocess.check_call(['git', 'checkout', 'FETCH_HEAD']) build_dir = 'api-tests/build' try: shutil.rmtree(build_dir) except FileNotFoundError: pass os.mkdir(build_dir) os.chdir(build_dir) #pylint: disable=bad-continuation subprocess.check_call([ 'cmake', '..', '-GUnix Makefiles', '-DTARGET=tgt_dev_apis_stdc', '-DTOOLCHAIN=HOST_GCC', '-DSUITE=CRYPTO', '-DPSA_CRYPTO_LIB_FILENAME={}/library/libmbedcrypto.a'.format(mbedtls_dir), '-DPSA_INCLUDE_PATHS={}/include'.format(mbedtls_dir) ]) subprocess.check_call(['cmake', '--build', '.']) proc = subprocess.Popen(['./psa-arch-tests-crypto'], bufsize=1, stdout=subprocess.PIPE, universal_newlines=True) test_re = re.compile('^TEST(?:: ([0-9]*)| RESULT: (FAILED|PASSED))') test = -1 unexpected_successes = set(EXPECTED_FAILURES) expected_failures = [] unexpected_failures = [] for line in proc.stdout: print(line, end='') match = test_re.match(line) if match is not None: if match.group(1) is not None: test = int(match.group(1)) elif match.group(2) == 'FAILED': try: unexpected_successes.remove(test) expected_failures.append(test) print('Expected failure, ignoring') except KeyError: unexpected_failures.append(test) print('ERROR: Unexpected failure') elif test in unexpected_successes: print('ERROR: Unexpected success') proc.wait() print() print('***** test_psa_compliance.py report ******') print() print('Expected failures:', ', '.join(str(i) for i in expected_failures)) print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures)) print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes))) print() if unexpected_successes or unexpected_failures: if unexpected_successes: print('Unexpected successes encountered.') #pylint: disable=line-too-long print('Please remove the corresponding tests from EXPECTED_FAILURES in tests/scripts/compliance_test.py') print() print('FAILED') sys.exit(1) else: os.chdir(mbedtls_dir) shutil.rmtree(psa_arch_tests_dir) print('SUCCESS') if __name__ == '__main__': main()