From 9f84911d55c63b7213b3fc54c987ef1fba52d410 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Tue, 19 Oct 2021 15:05:36 +0200 Subject: [PATCH 01/20] Run the PSA Compliance test suite in all.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit adds a component to all.sh which clones, builds and runs the compliance test suite. Signed-off-by: Bence Szépkúti --- tests/scripts/all.sh | 13 ++++ tests/scripts/test_psa_compliance.py | 96 ++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100755 tests/scripts/test_psa_compliance.py diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index aefe0295f..4fd66af75 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -2842,6 +2842,19 @@ component_test_zeroize () { unset gdb_disable_aslr } +component_test_psa_compliance () { + msg "build: make, default config (out-of-box), libmbedcrypto.a only" + make library/libmbedcrypto.a + + msg "unit test: test_psa_compliance.py" + ./tests/scripts/test_psa_compliance.py +} + +support_test_psa_compliance () { + local ver=($(cmake --version | sed 's/cmake version //; y/./ /; q')) + [ "${ver[0]}" -eq 3 ] && [ "${ver[1]}" -ge 10 ] +} + component_check_python_files () { msg "Lint: Python scripts" tests/scripts/check-python-files.sh diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py new file mode 100755 index 000000000..07fa76e60 --- /dev/null +++ b/tests/scripts/test_psa_compliance.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +#pylint: disable=missing-module-docstring +import os +import re +import shutil +import subprocess +import sys + +EXPECTED_FAILURES = { + 216, 221, 224, 225, 248, 249, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263 +} +PSA_ARCH_TESTS_REPO = 'https://github.com/ronald-cron-arm/psa-arch-tests.git' +PSA_ARCH_TESTS_REF = 'crypto1.0-3.0' + +#pylint: disable=too-many-statements +def main(): + mbedtls_dir = os.getcwd() + + mbedcrypto_lib = 'library/libmbedcrypto.a' + if not os.path.exists(mbedcrypto_lib): + subprocess.check_call(['make', mbedcrypto_lib]) + + psa_arch_tests_dir = 'psa-arch-tests' + try: + os.mkdir(psa_arch_tests_dir) + except FileExistsError: + pass + os.chdir(psa_arch_tests_dir) + + subprocess.check_call(['git', 'init']) + subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF]) + subprocess.check_call(['git', 'checkout', 'FETCH_HEAD']) + + build_dir = 'api-tests/build' + try: + shutil.rmtree(build_dir) + except FileNotFoundError: + pass + os.mkdir(build_dir) + os.chdir(build_dir) + + #pylint: disable=bad-continuation + subprocess.check_call([ + 'cmake', '..', '-GUnix Makefiles', + '-DTARGET=tgt_dev_apis_stdc', + '-DTOOLCHAIN=HOST_GCC', + '-DSUITE=CRYPTO', + '-DPSA_CRYPTO_LIB_FILENAME={}/library/libmbedcrypto.a'.format(mbedtls_dir), + '-DPSA_INCLUDE_PATHS={}/include'.format(mbedtls_dir) + ]) + subprocess.check_call(['cmake', '--build', '.']) + + proc = subprocess.Popen(['./psa-arch-tests-crypto'], + bufsize=1, stdout=subprocess.PIPE, universal_newlines=True) + + test_re = re.compile('^TEST(?:: ([0-9]*)| RESULT: FAILED)') + test = -1 + unexpected_successes = set(EXPECTED_FAILURES) + expected_failures = [] + unexpected_failures = [] + for line in proc.stdout: + print(line[:-1]) + match = test_re.match(line) + if match is not None: + if match.group(1) is not None: + test = int(match.group(1)) + else: + try: + unexpected_successes.remove(test) + expected_failures.append(test) + except KeyError: + unexpected_failures.append(test) + proc.wait() + + print() + print('***** test_psa_compliance.py report ******') + print() + print('Expected failures:', ', '.join(str(i) for i in expected_failures)) + print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures)) + print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes))) + print() + if unexpected_successes or unexpected_failures: + if unexpected_successes: + print('Unexpected successes encountered.') + #pylint: disable=line-too-long + print('Please remove the corresponding tests from EXPECTED_FAILURES in tests/scripts/compliance_test.py') + print() + print('FAILED') + sys.exit(1) + else: + os.chdir(mbedtls_dir) + shutil.rmtree(psa_arch_tests_dir) + print('SUCCESS') + +if __name__ == '__main__': + main() From ab796e656bf64a607097a3c8de31cdfce0b899b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Mon, 25 Oct 2021 19:29:07 +0200 Subject: [PATCH 02/20] Make the changes easier to backport MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The code replaced in this patch was not compatible with the development_2.x branch. Signed-off-by: Bence Szépkúti --- tests/scripts/all.sh | 12 +++++++++--- tests/scripts/test_psa_compliance.py | 5 ++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 4fd66af75..3ca29e855 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -2844,15 +2844,21 @@ component_test_zeroize () { component_test_psa_compliance () { msg "build: make, default config (out-of-box), libmbedcrypto.a only" - make library/libmbedcrypto.a + make -C library libmbedcrypto.a msg "unit test: test_psa_compliance.py" ./tests/scripts/test_psa_compliance.py } support_test_psa_compliance () { - local ver=($(cmake --version | sed 's/cmake version //; y/./ /; q')) - [ "${ver[0]}" -eq 3 ] && [ "${ver[1]}" -ge 10 ] + ver="$(cmake --version)" + ver="${ver#cmake version }" + ver_major="${ver%%.*}" + + ver="${ver#*.}" + ver_minor="${ver%%.*}" + + [ "$ver_major" -eq 3 ] && [ "$ver_minor" -ge 10 ] } component_check_python_files () { diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index 07fa76e60..d6fe8c440 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -16,9 +16,8 @@ PSA_ARCH_TESTS_REF = 'crypto1.0-3.0' def main(): mbedtls_dir = os.getcwd() - mbedcrypto_lib = 'library/libmbedcrypto.a' - if not os.path.exists(mbedcrypto_lib): - subprocess.check_call(['make', mbedcrypto_lib]) + if not os.path.exists('library/libmbedcrypto.a'): + subprocess.check_call(['make', '-C', 'library', 'libmbedcrypto.a']) psa_arch_tests_dir = 'psa-arch-tests' try: From f54a9d2adf811a6930f936c727ea19d8861f4fbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Mon, 25 Oct 2021 20:58:14 +0200 Subject: [PATCH 03/20] Indicate errors interleaved with test suite output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Indicate whether a success or failure is unexpected, or expected and ignored as they happen. Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index d6fe8c440..aa0a480e5 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -52,7 +52,7 @@ def main(): proc = subprocess.Popen(['./psa-arch-tests-crypto'], bufsize=1, stdout=subprocess.PIPE, universal_newlines=True) - test_re = re.compile('^TEST(?:: ([0-9]*)| RESULT: FAILED)') + test_re = re.compile('^TEST(?:: ([0-9]*)| RESULT: (FAILED|PASSED))') test = -1 unexpected_successes = set(EXPECTED_FAILURES) expected_failures = [] @@ -63,12 +63,16 @@ def main(): if match is not None: if match.group(1) is not None: test = int(match.group(1)) - else: + elif match.group(2) == 'FAILED': try: unexpected_successes.remove(test) expected_failures.append(test) + print('Expected failure, ignoring') except KeyError: unexpected_failures.append(test) + print('ERROR: Unexpected failure') + elif test in unexpected_successes: + print('ERROR: Unexpected success') proc.wait() print() From faf7f1b55479df35708ee17b4572126364f8cf68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Mon, 25 Oct 2021 20:58:14 +0200 Subject: [PATCH 04/20] Use print(end='') to silence double newline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index aa0a480e5..ca9387954 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -58,7 +58,7 @@ def main(): expected_failures = [] unexpected_failures = [] for line in proc.stdout: - print(line[:-1]) + print(line, end='') match = test_re.match(line) if match is not None: if match.group(1) is not None: From 60256555980fbf1596dcb333adec6b02855cac83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Fri, 29 Oct 2021 12:06:19 +0200 Subject: [PATCH 05/20] Simplify regex and use named capture groups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index ca9387954..dfd23938a 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -52,7 +52,10 @@ def main(): proc = subprocess.Popen(['./psa-arch-tests-crypto'], bufsize=1, stdout=subprocess.PIPE, universal_newlines=True) - test_re = re.compile('^TEST(?:: ([0-9]*)| RESULT: (FAILED|PASSED))') + test_re = re.compile( + '^TEST: (?P[0-9]*)|' + '^TEST RESULT: (?PFAILED|PASSED)' + ) test = -1 unexpected_successes = set(EXPECTED_FAILURES) expected_failures = [] @@ -61,9 +64,11 @@ def main(): print(line, end='') match = test_re.match(line) if match is not None: - if match.group(1) is not None: - test = int(match.group(1)) - elif match.group(2) == 'FAILED': + groupdict = match.groupdict() + test_num = groupdict['test_num'] + if test_num is not None: + test = int(test_num) + elif groupdict['test_result'] == 'FAILED': try: unexpected_successes.remove(test) expected_failures.append(test) From 19a124d677d1b8cac812c19f16d1b150a7680d2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Tue, 2 Nov 2021 13:41:14 +0100 Subject: [PATCH 06/20] Fix pylint errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index dfd23938a..41003d80d 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -1,5 +1,11 @@ #!/usr/bin/env python3 -#pylint: disable=missing-module-docstring +"""Run the PSA Cryto API compliance test suite. +Clone the repo and check out the commit specified by PSA_ARCH_TEST_REPO and PSA_ARCH_TEST_REF, +then complie and run the test suite. +Known defects in either the test suite or mbedtls - identified by their test number - are ignored, +while unexpected failures AND successes are reported as errors, +to help keep the list of known defects as up to date as possible. +""" import os import re import shutil @@ -90,8 +96,8 @@ def main(): if unexpected_successes or unexpected_failures: if unexpected_successes: print('Unexpected successes encountered.') - #pylint: disable=line-too-long - print('Please remove the corresponding tests from EXPECTED_FAILURES in tests/scripts/compliance_test.py') + print('Please remove the corresponding tests from ' + 'EXPECTED_FAILURES in tests/scripts/compliance_test.py') print() print('FAILED') sys.exit(1) From 559f1ce0a324d8d77086f3a035356e748116a1de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Tue, 2 Nov 2021 13:48:39 +0100 Subject: [PATCH 07/20] Make main() suitable to being called from python MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don't call sys.exit(), and don't clobber the working directory. Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 146 ++++++++++++++------------- 1 file changed, 75 insertions(+), 71 deletions(-) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index 41003d80d..7d7192f06 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -18,7 +18,7 @@ EXPECTED_FAILURES = { PSA_ARCH_TESTS_REPO = 'https://github.com/ronald-cron-arm/psa-arch-tests.git' PSA_ARCH_TESTS_REF = 'crypto1.0-3.0' -#pylint: disable=too-many-statements +#pylint: disable=too-many-branches,too-many-statements def main(): mbedtls_dir = os.getcwd() @@ -30,81 +30,85 @@ def main(): os.mkdir(psa_arch_tests_dir) except FileExistsError: pass - os.chdir(psa_arch_tests_dir) - - subprocess.check_call(['git', 'init']) - subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF]) - subprocess.check_call(['git', 'checkout', 'FETCH_HEAD']) - - build_dir = 'api-tests/build' try: - shutil.rmtree(build_dir) - except FileNotFoundError: - pass - os.mkdir(build_dir) - os.chdir(build_dir) + os.chdir(psa_arch_tests_dir) - #pylint: disable=bad-continuation - subprocess.check_call([ - 'cmake', '..', '-GUnix Makefiles', - '-DTARGET=tgt_dev_apis_stdc', - '-DTOOLCHAIN=HOST_GCC', - '-DSUITE=CRYPTO', - '-DPSA_CRYPTO_LIB_FILENAME={}/library/libmbedcrypto.a'.format(mbedtls_dir), - '-DPSA_INCLUDE_PATHS={}/include'.format(mbedtls_dir) - ]) - subprocess.check_call(['cmake', '--build', '.']) + subprocess.check_call(['git', 'init']) + subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF]) + subprocess.check_call(['git', 'checkout', 'FETCH_HEAD']) - proc = subprocess.Popen(['./psa-arch-tests-crypto'], - bufsize=1, stdout=subprocess.PIPE, universal_newlines=True) + build_dir = 'api-tests/build' + try: + shutil.rmtree(build_dir) + except FileNotFoundError: + pass + os.mkdir(build_dir) + os.chdir(build_dir) - test_re = re.compile( - '^TEST: (?P[0-9]*)|' - '^TEST RESULT: (?PFAILED|PASSED)' - ) - test = -1 - unexpected_successes = set(EXPECTED_FAILURES) - expected_failures = [] - unexpected_failures = [] - for line in proc.stdout: - print(line, end='') - match = test_re.match(line) - if match is not None: - groupdict = match.groupdict() - test_num = groupdict['test_num'] - if test_num is not None: - test = int(test_num) - elif groupdict['test_result'] == 'FAILED': - try: - unexpected_successes.remove(test) - expected_failures.append(test) - print('Expected failure, ignoring') - except KeyError: - unexpected_failures.append(test) - print('ERROR: Unexpected failure') - elif test in unexpected_successes: - print('ERROR: Unexpected success') - proc.wait() + #pylint: disable=bad-continuation + subprocess.check_call([ + 'cmake', '..', + '-GUnix Makefiles', + '-DTARGET=tgt_dev_apis_stdc', + '-DTOOLCHAIN=HOST_GCC', + '-DSUITE=CRYPTO', + '-DPSA_CRYPTO_LIB_FILENAME={}/library/libmbedcrypto.a'.format(mbedtls_dir), + '-DPSA_INCLUDE_PATHS={}/include'.format(mbedtls_dir) + ]) + subprocess.check_call(['cmake', '--build', '.']) - print() - print('***** test_psa_compliance.py report ******') - print() - print('Expected failures:', ', '.join(str(i) for i in expected_failures)) - print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures)) - print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes))) - print() - if unexpected_successes or unexpected_failures: - if unexpected_successes: - print('Unexpected successes encountered.') - print('Please remove the corresponding tests from ' - 'EXPECTED_FAILURES in tests/scripts/compliance_test.py') - print() - print('FAILED') - sys.exit(1) - else: + proc = subprocess.Popen(['./psa-arch-tests-crypto'], + bufsize=1, stdout=subprocess.PIPE, universal_newlines=True) + + test_re = re.compile( + '^TEST: (?P[0-9]*)|' + '^TEST RESULT: (?PFAILED|PASSED)' + ) + test = -1 + unexpected_successes = set(EXPECTED_FAILURES) + expected_failures = [] + unexpected_failures = [] + for line in proc.stdout: + print(line, end='') + match = test_re.match(line) + if match is not None: + groupdict = match.groupdict() + test_num = groupdict['test_num'] + if test_num is not None: + test = int(test_num) + elif groupdict['test_result'] == 'FAILED': + try: + unexpected_successes.remove(test) + expected_failures.append(test) + print('Expected failure, ignoring') + except KeyError: + unexpected_failures.append(test) + print('ERROR: Unexpected failure') + elif test in unexpected_successes: + print('ERROR: Unexpected success') + proc.wait() + + print() + print('***** test_psa_compliance.py report ******') + print() + print('Expected failures:', ', '.join(str(i) for i in expected_failures)) + print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures)) + print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes))) + print() + if unexpected_successes or unexpected_failures: + if unexpected_successes: + print('Unexpected successes encountered.') + print('Please remove the corresponding tests from ' + 'EXPECTED_FAILURES in tests/scripts/compliance_test.py') + print() + print('FAILED') + return 1 + else: + shutil.rmtree(psa_arch_tests_dir) + print('SUCCESS') + return 0 + finally: os.chdir(mbedtls_dir) - shutil.rmtree(psa_arch_tests_dir) - print('SUCCESS') if __name__ == '__main__': - main() + sys.exit(main()) From c2ca135f82d9d7211f10ec9444f5ef441539151e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Tue, 2 Nov 2021 14:01:08 +0100 Subject: [PATCH 08/20] Add licence header to script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index 7d7192f06..d94f6c242 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -6,6 +6,22 @@ Known defects in either the test suite or mbedtls - identified by their test num while unexpected failures AND successes are reported as errors, to help keep the list of known defects as up to date as possible. """ + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import re import shutil From eda2fb958334b705ba16d65d6e667834f6f39375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Tue, 2 Nov 2021 14:06:40 +0100 Subject: [PATCH 09/20] Make directory creation code more compact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index d94f6c242..33207c014 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -42,10 +42,7 @@ def main(): subprocess.check_call(['make', '-C', 'library', 'libmbedcrypto.a']) psa_arch_tests_dir = 'psa-arch-tests' - try: - os.mkdir(psa_arch_tests_dir) - except FileExistsError: - pass + os.makedirs(psa_arch_tests_dir, exist_ok=True) try: os.chdir(psa_arch_tests_dir) From bd66d184ff9acb4f62ffeebeca2bb938823fc495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Wed, 3 Nov 2021 11:32:51 +0100 Subject: [PATCH 10/20] Keep local clone around even if the test succeeds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- .gitignore | 3 +++ tests/scripts/test_psa_compliance.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 5a5860938..358262f49 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,9 @@ massif-* # Generated documentation: /apidoc +# PSA Crypto compliance test repo, cloned by test_psa_complaince.py +/psa-arch-tests + # Editor navigation files: /GPATH /GRTAGS diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index 33207c014..2f67f08c8 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Run the PSA Cryto API compliance test suite. Clone the repo and check out the commit specified by PSA_ARCH_TEST_REPO and PSA_ARCH_TEST_REF, -then complie and run the test suite. +then complie and run the test suite. The clone is stored at /psa-arch-tests. Known defects in either the test suite or mbedtls - identified by their test number - are ignored, while unexpected failures AND successes are reported as errors, to help keep the list of known defects as up to date as possible. @@ -46,6 +46,7 @@ def main(): try: os.chdir(psa_arch_tests_dir) + # Reuse existing local clone subprocess.check_call(['git', 'init']) subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF]) subprocess.check_call(['git', 'checkout', 'FETCH_HEAD']) @@ -117,7 +118,6 @@ def main(): print('FAILED') return 1 else: - shutil.rmtree(psa_arch_tests_dir) print('SUCCESS') return 0 finally: From d6cf089b37eb3915bcb4d3e2d141d2689c45dd8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Wed, 3 Nov 2021 11:36:09 +0100 Subject: [PATCH 11/20] Explain why support_test_psa_compliance is needed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- tests/scripts/all.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 3ca29e855..c9fa684a0 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -2851,6 +2851,7 @@ component_test_psa_compliance () { } support_test_psa_compliance () { + # psa-compliance-tests only supports CMake >= 3.10.0 ver="$(cmake --version)" ver="${ver#cmake version }" ver_major="${ver%%.*}" From b38686500e49687e7f7dd9a5602be35d6308e930 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Thu, 4 Nov 2021 16:39:48 +0100 Subject: [PATCH 12/20] Fix typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 358262f49..9b185c6ec 100644 --- a/.gitignore +++ b/.gitignore @@ -49,7 +49,7 @@ massif-* # Generated documentation: /apidoc -# PSA Crypto compliance test repo, cloned by test_psa_complaince.py +# PSA Crypto compliance test repo, cloned by test_psa_compliance.py /psa-arch-tests # Editor navigation files: From 355f8050cc1c1ca05ef30c3e79a45e5623ab0e1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Tue, 9 Nov 2021 17:33:57 +0100 Subject: [PATCH 13/20] Move to an updated fork of psa-arch-tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new fork was rebased on top of the upstream master, removing the need for most of the downstream patches we carried. On the other hand, the new fork includes a couple of fixes to problems that were not addressed by the original fork, or were introduced with the new version of psa-arch-tests. Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index 2f67f08c8..58cb8f1a9 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -29,10 +29,18 @@ import subprocess import sys EXPECTED_FAILURES = { - 216, 221, 224, 225, 248, 249, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263 + 221, 224, 225, 252, 253, 254, 255, 256, 257, 258, 259, 261, 262, 263 } -PSA_ARCH_TESTS_REPO = 'https://github.com/ronald-cron-arm/psa-arch-tests.git' -PSA_ARCH_TESTS_REF = 'crypto1.0-3.0' + +# We currently use a fork of ARM-software/psa-arch-tests, with a couple of downstream patches +# that allow it to build with MbedTLS 3, and fixes a couple of issues in the compliance test suite. +# These fixes allow the tests numbered 216, 248 and 249 to complete successfully. +# +# Once all the fixes are upstreamed, this fork should be replaced with an upstream commit/tag. +# +# Web URL: https://github.com/bensze01/psa-arch-tests/tree/fixes-for-mbedtls-3 +PSA_ARCH_TESTS_REPO = 'https://github.com/bensze01/psa-arch-tests.git' +PSA_ARCH_TESTS_REF = 'fixes-for-mbedtls-3' #pylint: disable=too-many-branches,too-many-statements def main(): From 7ccbea6e47d5e430d230f16ec87a40adea652b94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Tue, 9 Nov 2021 21:30:43 +0100 Subject: [PATCH 14/20] Document the values in EXPECTED_FAILURES MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Including the issues where the corresponding defects are tracked. Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index 58cb8f1a9..31e3fce77 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -28,8 +28,28 @@ import shutil import subprocess import sys +# PSA Compliance tests we expect to fail due to known defects in Mbed TLS (or the test suite) +# The test numbers correspond to the numbers used by the console output of the test suite. +# Test number 2xx corresponds to the files in the folder +# psa-arch-tests/api-tests/dev_apis/crypto/test_c0xx EXPECTED_FAILURES = { - 221, 224, 225, 252, 253, 254, 255, 256, 257, 258, 259, 261, 262, 263 + # psa_key_derivation_output_key() returns PSA_ERROR_NOT_PERMITTED instead of + # PSA_ERROR_BAD_STATE when called after the operation was aborted. + # - Tracked in issue #5143 + 221, + + # psa_aead_[encrypt/decrypt]() returns PSA_ERROR_NOT_SUPPORTED instead of + # PSA_ERROR_INVALID_ARGUMENT when called with an invalid nonce. + # - Tracked in issue #5144 + 224, 225, + + # Multipart CCM is not supported. + # - Tracked in issue #3721 + 252, 253, 254, 255, 256, 257, 258, 259, 261, + + # psa_hash_suspend() and psa_hash_resume() are not supported. + # - Tracked in issue #3274 + 262, 263 } # We currently use a fork of ARM-software/psa-arch-tests, with a couple of downstream patches From 340352ad743ee1a6285ad0b78eec15925083040d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Tue, 9 Nov 2021 22:13:46 +0100 Subject: [PATCH 15/20] Track upstreaming task in an issue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index 31e3fce77..2f6358132 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -57,6 +57,7 @@ EXPECTED_FAILURES = { # These fixes allow the tests numbered 216, 248 and 249 to complete successfully. # # Once all the fixes are upstreamed, this fork should be replaced with an upstream commit/tag. +# - Tracked in issue #5145 # # Web URL: https://github.com/bensze01/psa-arch-tests/tree/fixes-for-mbedtls-3 PSA_ARCH_TESTS_REPO = 'https://github.com/bensze01/psa-arch-tests.git' From d1c6420aba78c431cced755843730de42d0714e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Wed, 10 Nov 2021 17:43:20 +0100 Subject: [PATCH 16/20] Move to a fork with Mbed TLS 2.x specific fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index 2f6358132..7bc9727f2 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -53,15 +53,15 @@ EXPECTED_FAILURES = { } # We currently use a fork of ARM-software/psa-arch-tests, with a couple of downstream patches -# that allow it to build with MbedTLS 3, and fixes a couple of issues in the compliance test suite. +# that allow it to build with Mbed TLS 2, and fixes a couple of issues in the compliance test suite. # These fixes allow the tests numbered 216, 248 and 249 to complete successfully. # # Once all the fixes are upstreamed, this fork should be replaced with an upstream commit/tag. # - Tracked in issue #5145 # -# Web URL: https://github.com/bensze01/psa-arch-tests/tree/fixes-for-mbedtls-3 +# Web URL: https://github.com/bensze01/psa-arch-tests/tree/fixes-for-mbedtls-2 PSA_ARCH_TESTS_REPO = 'https://github.com/bensze01/psa-arch-tests.git' -PSA_ARCH_TESTS_REF = 'fixes-for-mbedtls-3' +PSA_ARCH_TESTS_REF = 'fixes-for-mbedtls-2' #pylint: disable=too-many-branches,too-many-statements def main(): From 1dbaaba067a6914809edea71cbcbfbd74164550f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Wed, 10 Nov 2021 17:44:37 +0100 Subject: [PATCH 17/20] Build psa-arch-tests with MISSING_CRYPTO_1_0=1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This disables references to the missing multipart AEAD functions. Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index 7bc9727f2..7c159192c 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -95,6 +95,7 @@ def main(): '-DTARGET=tgt_dev_apis_stdc', '-DTOOLCHAIN=HOST_GCC', '-DSUITE=CRYPTO', + '-DMISSING_CRYPTO_1_0=1', '-DPSA_CRYPTO_LIB_FILENAME={}/library/libmbedcrypto.a'.format(mbedtls_dir), '-DPSA_INCLUDE_PATHS={}/include'.format(mbedtls_dir) ]) From e30fcb6ed5942fc5bbd90fff802932a322834bbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Thu, 11 Nov 2021 16:24:19 +0100 Subject: [PATCH 18/20] Remove superfluous expected failures from list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #5144 doesn't affect development_2.x Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index 7c159192c..f31640332 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -38,11 +38,6 @@ EXPECTED_FAILURES = { # - Tracked in issue #5143 221, - # psa_aead_[encrypt/decrypt]() returns PSA_ERROR_NOT_SUPPORTED instead of - # PSA_ERROR_INVALID_ARGUMENT when called with an invalid nonce. - # - Tracked in issue #5144 - 224, 225, - # Multipart CCM is not supported. # - Tracked in issue #3721 252, 253, 254, 255, 256, 257, 258, 259, 261, From 24ec529f8250c6a71b917e304e9466e5270ab5ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Thu, 11 Nov 2021 16:33:48 +0100 Subject: [PATCH 19/20] Multipart AEAD is not supported in Mbed TLS 2.x MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- tests/scripts/test_psa_compliance.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/scripts/test_psa_compliance.py b/tests/scripts/test_psa_compliance.py index f31640332..c4dbc946b 100755 --- a/tests/scripts/test_psa_compliance.py +++ b/tests/scripts/test_psa_compliance.py @@ -38,9 +38,8 @@ EXPECTED_FAILURES = { # - Tracked in issue #5143 221, - # Multipart CCM is not supported. - # - Tracked in issue #3721 - 252, 253, 254, 255, 256, 257, 258, 259, 261, + # Multipart AEAD is not supported in Mbed TLS 2.x. + 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, # psa_hash_suspend() and psa_hash_resume() are not supported. # - Tracked in issue #3274 From c1e79fd2e35fa3cc742c167ee0a6c3c47cceaf6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20Sz=C3=A9pk=C3=BAti?= Date: Thu, 11 Nov 2021 20:13:14 +0100 Subject: [PATCH 20/20] Enable CMAC for PSA crypto compliance tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Szépkúti --- tests/scripts/all.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index c9fa684a0..bbc2d9c4f 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -2843,7 +2843,8 @@ component_test_zeroize () { } component_test_psa_compliance () { - msg "build: make, default config (out-of-box), libmbedcrypto.a only" + msg "build: make, default config + CMAC, libmbedcrypto.a only" + scripts/config.py set MBEDTLS_CMAC_C make -C library libmbedcrypto.a msg "unit test: test_psa_compliance.py"