diff --git a/.travis.yml b/.travis.yml index c8ca79dc0..6a9b6f611 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,12 +5,6 @@ compiler: sudo: false cache: ccache -# blocklist -branches: - except: - - development-psa - - coverity_scan - script: - tests/scripts/recursion.pl library/*.c - tests/scripts/check-generated-files.sh @@ -27,7 +21,7 @@ script: env: global: - SEED=1 - - secure: "barHldniAfXyoWOD/vcO+E6/Xm4fmcaUoC9BeKW+LwsHqlDMLvugaJnmLXkSpkbYhVL61Hzf3bo0KPJn88AFc5Rkf8oYHPjH4adMnVXkf3B9ghHCgznqHsAH3choo6tnPxaFgOwOYmLGb382nQxfE5lUdvnM/W/psQjWt66A1+k=" + - secure: "FrI5d2s+ckckC17T66c8jm2jV6i2DkBPU5nyWzwbedjmEBeocREfQLd/x8yKpPzLDz7ghOvr+/GQvsPPn0dVkGlNzm3Q+hGHc/ujnASuUtGrcuMM+0ALnJ3k4rFr9xEvjJeWb4SmhJO5UCAZYvTItW4k7+bj9L+R6lt3TzQbXzg=" addons: apt: @@ -37,7 +31,7 @@ addons: coverity_scan: project: name: "ARMmbed/mbedtls" - notification_email: simon.butcher@arm.com + notification_email: support-mbedtls@arm.com build_command_prepend: build_command: make branch_pattern: coverity_scan diff --git a/docs/architecture/Makefile b/docs/architecture/Makefile index 258abcdb0..ab22fb16d 100644 --- a/docs/architecture/Makefile +++ b/docs/architecture/Makefile @@ -5,6 +5,7 @@ default: all all_markdown = \ mbed-crypto-storage-specification.md \ testing/driver-interface-test-strategy.md \ + testing/test-framework.md \ # This line is intentionally left blank html: $(all_markdown:.md=.html) diff --git a/docs/architecture/testing/test-framework.md b/docs/architecture/testing/test-framework.md new file mode 100644 index 000000000..e0e960f87 --- /dev/null +++ b/docs/architecture/testing/test-framework.md @@ -0,0 +1,58 @@ +# Mbed TLS test framework + +This document is an overview of the Mbed TLS test framework and test tools. + +This document is incomplete. You can help by expanding it. + +## Unit tests + +See + +### Unit test descriptions + +Each test case has a description which succinctly describes for a human audience what the test does. The first non-comment line of each paragraph in a `.data` file is the test description. The following rules and guidelines apply: + +* Test descriptions may not contain semicolons, line breaks and other control characters, or non-ASCII characters.
+ Rationale: keep the tools that process test descriptions (`generate_test_code.py`, [outcome file](#outcome-file) tools) simple. +* Test descriptions must be unique within a `.data` file. If you can't think of a better description, the convention is to append `#1`, `#2`, etc.
+ Rationale: make it easy to relate a failure log to the test data. Avoid confusion between cases in the [outcome file](#outcome-file). +* Test descriptions should be a maximum of **66 characters**.
+ Rationale: 66 characters is what our various tools assume (leaving room for 14 more characters on an 80-column line). Longer descriptions may be truncated or may break a visual alignment.
+ We have a lot of test cases with longer descriptions, but they should be avoided. At least please make sure that the first 66 characters describe the test uniquely. +* Make the description descriptive. “foo: x=2, y=4” is more descriptive than “foo #2”. “foo: 0 #include - int main( int argc, char *argv[] ) { FILE *f; diff --git a/scripts/config.py b/scripts/config.py index 8fe98a889..db2661c92 100755 --- a/scripts/config.py +++ b/scripts/config.py @@ -169,6 +169,9 @@ def include_in_full(name): 'MBEDTLS_ECDH_VARIANT_EVEREST_ENABLED', 'MBEDTLS_ECP_RESTARTABLE', 'MBEDTLS_HAVE_SSE2', + 'MBEDTLS_MEMORY_BACKTRACE', + 'MBEDTLS_MEMORY_BUFFER_ALLOC_C', + 'MBEDTLS_MEMORY_DEBUG', 'MBEDTLS_NO_64BIT_MULTIPLICATION', 'MBEDTLS_NO_DEFAULT_ENTROPY_SOURCES', 'MBEDTLS_NO_PLATFORM_ENTROPY', @@ -201,8 +204,6 @@ def keep_in_baremetal(name): 'MBEDTLS_HAVEGE_C', 'MBEDTLS_HAVE_TIME', 'MBEDTLS_HAVE_TIME_DATE', - 'MBEDTLS_MEMORY_BACKTRACE', - 'MBEDTLS_MEMORY_BUFFER_ALLOC_C', 'MBEDTLS_PLATFORM_FPRINTF_ALT', 'MBEDTLS_PLATFORM_TIME_ALT', 'MBEDTLS_PSA_CRYPTO_SE_C', diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 99abda3bf..2567cc0dd 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -112,9 +112,15 @@ pre_initialize_variables () { CONFIG_H='include/mbedtls/config.h' CONFIG_BAK="$CONFIG_H.bak" + append_outcome=0 FORCE=0 KEEP_GOING=0 + : ${MBEDTLS_TEST_OUTCOME_FILE=} + : ${MBEDTLS_TEST_PLATFORM="$(uname -s | tr -c \\n0-9A-Za-z _)-$(uname -m | tr -c \\n0-9A-Za-z _)"} + export MBEDTLS_TEST_OUTCOME_FILE + export MBEDTLS_TEST_PLATFORM + # Default commands, can be overridden by the environment : ${OUT_OF_SOURCE_DIR:=./mbedtls_out_of_source_build} : ${ARMC5_BIN_DIR:=/usr/bin} @@ -183,14 +189,18 @@ General options: -f|--force Force the tests to overwrite any modified files. -k|--keep-going Run all tests and report errors at the end. -m|--memory Additional optional memory tests. + --append-outcome Append to the outcome file (if used). --armcc Run ARM Compiler builds (on by default). --except Exclude the COMPONENTs listed on the command line, instead of running only those. + --no-append-outcome Write a new outcome file and analyze it (default). --no-armcc Skip ARM Compiler builds. --no-force Refuse to overwrite modified files (default). --no-keep-going Stop at the first error (default). --no-memory No additional memory tests (default). --out-of-source-dir= Directory used for CMake out-of-source build tests. + --outcome-file= File where test outcomes are written (not done if + empty; default: \$MBEDTLS_TEST_OUTCOME_FILE). --random-seed Use a random seed value for randomized tests (default). -r|--release-test Run this script in release mode. This fixes the seed value to 1. -s|--seed Integer seed value to use for this test run. @@ -309,6 +319,7 @@ pre_parse_command_line () { # all.sh will still run and work properly. while [ $# -gt 0 ]; do case "$1" in + --append-outcome) append_outcome=1;; --armcc) no_armcc=;; --armc5-bin-dir) shift; ARMC5_BIN_DIR="$1";; --armc6-bin-dir) shift; ARMC6_BIN_DIR="$1";; @@ -323,6 +334,7 @@ pre_parse_command_line () { --list-all-components) printf '%s\n' $ALL_COMPONENTS; exit;; --list-components) printf '%s\n' $SUPPORTED_COMPONENTS; exit;; --memory|-m) ;; + --no-append-outcome) append_outcome=0;; --no-armcc) no_armcc=1;; --no-force) FORCE=0;; --no-keep-going) KEEP_GOING=0;; @@ -330,6 +342,7 @@ pre_parse_command_line () { --openssl) shift;; --openssl-legacy) shift;; --openssl-next) shift;; + --outcome-file) shift; MBEDTLS_TEST_OUTCOME_FILE="$1";; --out-of-source-dir) shift; OUT_OF_SOURCE_DIR="$1";; --random-seed) ;; --release-test|-r) ;; @@ -464,9 +477,19 @@ not() { ! "$@" } +pre_prepare_outcome_file () { + case "$MBEDTLS_TEST_OUTCOME_FILE" in + [!/]*) MBEDTLS_TEST_OUTCOME_FILE="$PWD/$MBEDTLS_TEST_OUTCOME_FILE";; + esac + if [ -n "$MBEDTLS_TEST_OUTCOME_FILE" ] && [ "$append_outcome" -eq 0 ]; then + rm -f "$MBEDTLS_TEST_OUTCOME_FILE" + fi +} + pre_print_configuration () { msg "info: $0 configuration" echo "FORCE: $FORCE" + echo "MBEDTLS_TEST_OUTCOME_FILE: ${MBEDTLS_TEST_OUTCOME_FILE:-(none)}" echo "ARMC5_BIN_DIR: $ARMC5_BIN_DIR" echo "ARMC6_BIN_DIR: $ARMC6_BIN_DIR" } @@ -528,32 +551,37 @@ pre_check_tools () { # Indicative running times are given for reference. component_check_recursion () { - msg "test: recursion.pl" # < 1s + msg "Check: recursion.pl" # < 1s record_status tests/scripts/recursion.pl library/*.c } component_check_generated_files () { - msg "test: freshness of generated source files" # < 1s + msg "Check: freshness of generated source files" # < 1s record_status tests/scripts/check-generated-files.sh } component_check_doxy_blocks () { - msg "test: doxygen markup outside doxygen blocks" # < 1s + msg "Check: doxygen markup outside doxygen blocks" # < 1s record_status tests/scripts/check-doxy-blocks.pl } component_check_files () { - msg "test: check-files.py" # < 1s + msg "Check: file sanity checks (permissions, encodings)" # < 1s record_status tests/scripts/check-files.py } component_check_names () { - msg "test/build: declared and exported names" # < 3s + msg "Check: declared and exported names (builds the library)" # < 3s record_status tests/scripts/check-names.sh -v } +component_check_test_cases () { + msg "Check: test case descriptions" # < 1s + record_status tests/scripts/check-test-cases.py +} + component_check_doxygen_warnings () { - msg "test: doxygen warnings" # ~ 3s + msg "Check: doxygen warnings (builds the documentation)" # ~ 3s record_status tests/scripts/doxygen.sh } @@ -565,12 +593,18 @@ component_check_doxygen_warnings () { component_test_default_out_of_box () { msg "build: make, default config (out-of-box)" # ~1min make + # Disable fancy stuff + SAVE_MBEDTLS_TEST_OUTCOME_FILE="$MBEDTLS_TEST_OUTCOME_FILE" + unset MBEDTLS_TEST_OUTCOME_FILE msg "test: main suites make, default config (out-of-box)" # ~10s make test msg "selftest: make, default config (out-of-box)" # ~10s programs/test/selftest + + export MBEDTLS_TEST_OUTCOME_FILE="$SAVE_MBEDTLS_TEST_OUTCOME_FILE" + unset SAVE_MBEDTLS_TEST_OUTCOME_FILE } component_test_default_cmake_gcc_asan () { @@ -582,6 +616,16 @@ component_test_default_cmake_gcc_asan () { make test } +component_test_full_cmake_gcc_asan () { + msg "build: full config, cmake, gcc, ASan" + scripts/config.py full + CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . + make + + msg "test: main suites (inc. selftests) (full config, ASan build)" + make test +} + component_test_ref_configs () { msg "test/build: ref-configs (ASan build)" # ~ 6 min 20s CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan . @@ -635,7 +679,6 @@ component_test_everest () { component_test_psa_collect_statuses () { msg "build+test: psa_collect_statuses" # ~30s scripts/config.py full - scripts/config.py unset MBEDTLS_MEMORY_BUFFER_ALLOC_C # slow and irrelevant record_status tests/scripts/psa_collect_statuses.py # Check that psa_crypto_init() succeeded at least once record_status grep -q '^0:psa_crypto_init:' tests/statuses.log @@ -645,7 +688,6 @@ component_test_psa_collect_statuses () { component_test_full_cmake_clang () { msg "build: cmake, full config, clang" # ~ 50s scripts/config.py full - scripts/config.py unset MBEDTLS_MEMORY_BACKTRACE # too slow for tests CC=clang cmake -D CMAKE_BUILD_TYPE:String=Check -D ENABLE_TESTING=On . make @@ -659,7 +701,6 @@ component_test_full_cmake_clang () { component_test_full_make_gcc_o0 () { msg "build: make, full config, gcc -O0" # ~ 50s scripts/config.py full - scripts/config.py unset MBEDTLS_MEMORY_BACKTRACE # too slow for tests make CC=gcc CFLAGS='-O0' msg "test: main suites (full config, gcc -O0)" # ~ 5s @@ -712,9 +753,8 @@ component_build_default_make_gcc_and_cxx () { component_test_no_use_psa_crypto_full_cmake_asan() { # full minus MBEDTLS_USE_PSA_CRYPTO: run the same set of tests as basic-build-test.sh - msg "build: cmake, full config + MBEDTLS_USE_PSA_CRYPTO, ASan" + msg "build: cmake, full config minus MBEDTLS_USE_PSA_CRYPTO, ASan" scripts/config.py full - scripts/config.py unset MBEDTLS_MEMORY_BUFFER_ALLOC_C scripts/config.py set MBEDTLS_ECP_RESTARTABLE # not using PSA, so enable restartable ECC scripts/config.py set MBEDTLS_PSA_CRYPTO_C scripts/config.py unset MBEDTLS_USE_PSA_CRYPTO @@ -732,7 +772,6 @@ component_test_check_params_functionality () { scripts/config.py full # includes CHECK_PARAMS # Make MBEDTLS_PARAM_FAILED call mbedtls_param_failed(). scripts/config.py unset MBEDTLS_CHECK_PARAMS_ASSERT - scripts/config.py unset MBEDTLS_MEMORY_BUFFER_ALLOC_C # Only build and run tests. Do not build sample programs, because # they don't have a mbedtls_param_failed() function. make CC=gcc CFLAGS='-Werror -O1' lib test @@ -742,8 +781,6 @@ component_test_check_params_without_platform () { msg "build+test: MBEDTLS_CHECK_PARAMS without MBEDTLS_PLATFORM_C" scripts/config.py full # includes CHECK_PARAMS # Keep MBEDTLS_PARAM_FAILED as assert. - scripts/config.py unset MBEDTLS_MEMORY_BACKTRACE # too slow for tests - scripts/config.py unset MBEDTLS_MEMORY_BUFFER_ALLOC_C scripts/config.py unset MBEDTLS_PLATFORM_EXIT_ALT scripts/config.py unset MBEDTLS_PLATFORM_TIME_ALT scripts/config.py unset MBEDTLS_PLATFORM_FPRINTF_ALT @@ -758,7 +795,6 @@ component_test_check_params_without_platform () { component_test_check_params_silent () { msg "build+test: MBEDTLS_CHECK_PARAMS with alternative MBEDTLS_PARAM_FAILED()" scripts/config.py full # includes CHECK_PARAMS - scripts/config.py unset MBEDTLS_MEMORY_BACKTRACE # too slow for tests # Set MBEDTLS_PARAM_FAILED to nothing. sed -i 's/.*\(#define MBEDTLS_PARAM_FAILED( cond )\).*/\1/' "$CONFIG_H" make CC=gcc CFLAGS='-Werror -O1' all test @@ -778,7 +814,6 @@ component_test_no_platform () { scripts/config.py unset MBEDTLS_PLATFORM_TIME_ALT scripts/config.py unset MBEDTLS_PLATFORM_EXIT_ALT scripts/config.py unset MBEDTLS_ENTROPY_NV_SEED - scripts/config.py unset MBEDTLS_MEMORY_BUFFER_ALLOC_C scripts/config.py unset MBEDTLS_FS_IO scripts/config.py unset MBEDTLS_PSA_CRYPTO_SE_C scripts/config.py unset MBEDTLS_PSA_CRYPTO_STORAGE_C @@ -828,7 +863,6 @@ component_test_platform_calloc_macro () { component_test_malloc_0_null () { msg "build: malloc(0) returns NULL (ASan+UBSan build)" scripts/config.py full - scripts/config.py unset MBEDTLS_MEMORY_BUFFER_ALLOC_C make CC=gcc CFLAGS="'-DMBEDTLS_CONFIG_FILE=\"$PWD/tests/configs/config-wrapper-malloc-0-null.h\"' $ASAN_CFLAGS -O" LDFLAGS="$ASAN_CFLAGS" msg "test: malloc(0) returns NULL (ASan+UBSan build)" @@ -840,6 +874,30 @@ component_test_malloc_0_null () { if_build_succeeded programs/test/selftest calloc } +component_test_memory_buffer_allocator_backtrace () { + msg "build: default config with memory buffer allocator and backtrace enabled" + scripts/config.py set MBEDTLS_MEMORY_BUFFER_ALLOC_C + scripts/config.py set MBEDTLS_PLATFORM_MEMORY + scripts/config.py set MBEDTLS_MEMORY_BACKTRACE + scripts/config.py set MBEDTLS_MEMORY_DEBUG + CC=gcc cmake . + make + + msg "test: MBEDTLS_MEMORY_BUFFER_ALLOC_C and MBEDTLS_MEMORY_BACKTRACE" + make test +} + +component_test_memory_buffer_allocator () { + msg "build: default config with memory buffer allocator" + scripts/config.py set MBEDTLS_MEMORY_BUFFER_ALLOC_C + scripts/config.py set MBEDTLS_PLATFORM_MEMORY + CC=gcc cmake . + make + + msg "test: MBEDTLS_MEMORY_BUFFER_ALLOC_C" + make test +} + component_test_aes_fewer_tables () { msg "build: default config with AES_FEWER_TABLES enabled" scripts/config.py set MBEDTLS_AES_FEWER_TABLES @@ -880,7 +938,6 @@ component_test_se_default () { component_test_se_full () { msg "build: full config + MBEDTLS_PSA_CRYPTO_SE_C" scripts/config.py full - scripts/config.py unset MBEDTLS_MEMORY_BUFFER_ALLOC_C scripts/config.py set MBEDTLS_PSA_CRYPTO_SE_C make CC=gcc CFLAGS="$ASAN_CFLAGS -O2" LDFLAGS="$ASAN_CFLAGS" @@ -890,7 +947,7 @@ component_test_se_full () { component_test_make_shared () { msg "build/test: make shared" # ~ 40s - make SHARED=1 all check -j1 + make SHARED=1 all check ldd programs/util/strerror | grep libmbedcrypto } @@ -933,9 +990,6 @@ component_test_m32_o1 () { # Build again with -O1, to compile in the i386 specific inline assembly msg "build: i386, make, gcc -O1 (ASan build)" # ~ 30s scripts/config.py full - scripts/config.py unset MBEDTLS_MEMORY_BACKTRACE - scripts/config.py unset MBEDTLS_MEMORY_BUFFER_ALLOC_C - scripts/config.py unset MBEDTLS_MEMORY_DEBUG make CC=gcc CFLAGS="$ASAN_CFLAGS -m32 -O1" LDFLAGS="-m32 $ASAN_CFLAGS" msg "test: i386, make, gcc -O1 (ASan build)" @@ -1008,7 +1062,6 @@ component_test_have_int64 () { component_test_no_udbl_division () { msg "build: MBEDTLS_NO_UDBL_DIVISION native" # ~ 10s scripts/config.py full - scripts/config.py unset MBEDTLS_MEMORY_BACKTRACE # too slow for tests scripts/config.py set MBEDTLS_NO_UDBL_DIVISION make CFLAGS='-Werror -O1' @@ -1019,7 +1072,6 @@ component_test_no_udbl_division () { component_test_no_64bit_multiplication () { msg "build: MBEDTLS_NO_64BIT_MULTIPLICATION native" # ~ 10s scripts/config.py full - scripts/config.py unset MBEDTLS_MEMORY_BACKTRACE # too slow for tests scripts/config.py set MBEDTLS_NO_64BIT_MULTIPLICATION make CFLAGS='-Werror -O1' @@ -1087,15 +1139,15 @@ component_build_armcc () { component_build_mingw () { msg "build: Windows cross build - mingw64, make (Link Library)" # ~ 30s - make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 lib programs -j1 + make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 lib programs # note Make tests only builds the tests, but doesn't run them - make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror' WINDOWS_BUILD=1 tests -j1 + make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror' WINDOWS_BUILD=1 tests make WINDOWS_BUILD=1 clean msg "build: Windows cross build - mingw64, make (DLL)" # ~ 30s - make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 SHARED=1 lib programs -j1 - make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 SHARED=1 tests -j1 + make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 SHARED=1 lib programs + make CC=i686-w64-mingw32-gcc AR=i686-w64-mingw32-ar LD=i686-w64-minggw32-ld CFLAGS='-Werror -Wall -Wextra' WINDOWS_BUILD=1 SHARED=1 tests make WINDOWS_BUILD=1 clean } support_build_mingw() { @@ -1219,6 +1271,7 @@ run_component () { # The cleanup function will restore it. cp -p "$CONFIG_H" "$CONFIG_BAK" current_component="$1" + export MBEDTLS_TEST_CONFIGURATION="$current_component" "$@" cleanup } @@ -1239,6 +1292,7 @@ else "$@" } fi +pre_prepare_outcome_file pre_print_configuration pre_check_tools cleanup diff --git a/tests/scripts/check-test-cases.py b/tests/scripts/check-test-cases.py new file mode 100755 index 000000000..87a35e47e --- /dev/null +++ b/tests/scripts/check-test-cases.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 + +"""Sanity checks for test data. +""" + +# Copyright (C) 2019, Arm Limited, All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file is part of Mbed TLS (https://tls.mbed.org) + +import glob +import os +import re +import sys + +class Results: + def __init__(self): + self.errors = 0 + self.warnings = 0 + + def error(self, file_name, line_number, fmt, *args): + sys.stderr.write(('{}:{}:ERROR:' + fmt + '\n'). + format(file_name, line_number, *args)) + self.errors += 1 + + def warning(self, file_name, line_number, fmt, *args): + sys.stderr.write(('{}:{}:Warning:' + fmt + '\n') + .format(file_name, line_number, *args)) + self.warnings += 1 + +def collect_test_directories(): + if os.path.isdir('tests'): + tests_dir = 'tests' + elif os.path.isdir('suites'): + tests_dir = '.' + elif os.path.isdir('../suites'): + tests_dir = '..' + directories = [tests_dir] + crypto_tests_dir = os.path.normpath(os.path.join(tests_dir, + '../crypto/tests')) + if os.path.isdir(crypto_tests_dir): + directories.append(crypto_tests_dir) + return directories + +def check_description(results, seen, file_name, line_number, description): + if description in seen: + results.error(file_name, line_number, + 'Duplicate description (also line {})', + seen[description]) + return + if re.search(br'[\t;]', description): + results.error(file_name, line_number, + 'Forbidden character \'{}\' in description', + re.search(br'[\t;]', description).group(0).decode('ascii')) + if re.search(br'[^ -~]', description): + results.error(file_name, line_number, + 'Non-ASCII character in description') + if len(description) > 66: + results.warning(file_name, line_number, + 'Test description too long ({} > 66)', + len(description)) + seen[description] = line_number + +def check_test_suite(results, data_file_name): + in_paragraph = False + descriptions = {} + with open(data_file_name, 'rb') as data_file: + for line_number, line in enumerate(data_file, 1): + line = line.rstrip(b'\r\n') + if not line: + in_paragraph = False + continue + if line.startswith(b'#'): + continue + if not in_paragraph: + # This is a test case description line. + check_description(results, descriptions, + data_file_name, line_number, line) + in_paragraph = True + +def check_ssl_opt_sh(results, file_name): + descriptions = {} + with open(file_name, 'rb') as file_contents: + for line_number, line in enumerate(file_contents, 1): + # Assume that all run_test calls have the same simple form + # with the test description entirely on the same line as the + # function name. + m = re.match(br'\s*run_test\s+"((?:[^\\"]|\\.)*)"', line) + if not m: + continue + description = m.group(1) + check_description(results, descriptions, + file_name, line_number, description) + +def main(): + test_directories = collect_test_directories() + results = Results() + for directory in test_directories: + for data_file_name in glob.glob(os.path.join(directory, 'suites', + '*.data')): + check_test_suite(results, data_file_name) + ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh') + if os.path.exists(ssl_opt_sh): + check_ssl_opt_sh(results, ssl_opt_sh) + if results.warnings or results.errors: + sys.stderr.write('{}: {} errors, {} warnings\n' + .format(sys.argv[0], results.errors, results.warnings)) + sys.exit(1 if results.errors else 0) + +if __name__ == '__main__': + main() diff --git a/tests/scripts/curves.pl b/tests/scripts/curves.pl index 3e2255277..8119a46e6 100755 --- a/tests/scripts/curves.pl +++ b/tests/scripts/curves.pl @@ -51,6 +51,7 @@ for my $curve (@curves) { print "\n******************************************\n"; print "* Testing without curve: $curve\n"; print "******************************************\n"; + $ENV{MBEDTLS_TEST_CONFIGURATION} = "-$curve"; system( "scripts/config.py unset $curve" ) and abort "Failed to disable $curve\n"; diff --git a/tests/scripts/depends-hashes.pl b/tests/scripts/depends-hashes.pl index 92bcceb82..7cb41b55c 100755 --- a/tests/scripts/depends-hashes.pl +++ b/tests/scripts/depends-hashes.pl @@ -57,6 +57,7 @@ for my $hash (@hashes) { print "\n******************************************\n"; print "* Testing without hash: $hash\n"; print "******************************************\n"; + $ENV{MBEDTLS_TEST_CONFIGURATION} = "-$hash"; system( "scripts/config.py unset $hash" ) and abort "Failed to disable $hash\n"; diff --git a/tests/scripts/depends-pkalgs.pl b/tests/scripts/depends-pkalgs.pl index 70e77b046..7fbd6d71e 100755 --- a/tests/scripts/depends-pkalgs.pl +++ b/tests/scripts/depends-pkalgs.pl @@ -59,6 +59,7 @@ while( my ($alg, $extras) = each %algs ) { print "\n******************************************\n"; print "* Testing without alg: $alg\n"; print "******************************************\n"; + $ENV{MBEDTLS_TEST_CONFIGURATION} = "-$alg"; system( "scripts/config.py unset $alg" ) and abort "Failed to disable $alg\n"; diff --git a/tests/scripts/test-ref-configs.pl b/tests/scripts/test-ref-configs.pl index 1e6596928..b29d0dd1b 100755 --- a/tests/scripts/test-ref-configs.pl +++ b/tests/scripts/test-ref-configs.pl @@ -66,6 +66,7 @@ while( my ($conf, $data) = each %configs ) { print "\n******************************************\n"; print "* Testing configuration: $conf\n"; print "******************************************\n"; + $ENV{MBEDTLS_TEST_CONFIGURATION} = $conf; system( "cp configs/$conf $config_h" ) and abort "Failed to activate $conf\n"; diff --git a/tests/suites/helpers.function b/tests/suites/helpers.function index 00320bca3..0cce463c0 100644 --- a/tests/suites/helpers.function +++ b/tests/suites/helpers.function @@ -406,7 +406,7 @@ typedef enum TEST_RESULT_SKIPPED } test_result_t; -static struct +typedef struct { paramfail_test_state_t paramfail_test_state; test_result_t result; @@ -415,7 +415,8 @@ static struct int line_no; unsigned long step; } -test_info; +test_info_t; +static test_info_t test_info; #if defined(MBEDTLS_PLATFORM_C) mbedtls_platform_context platform_ctx; diff --git a/tests/suites/host_test.function b/tests/suites/host_test.function index 24d9b9747..a87dc139c 100644 --- a/tests/suites/host_test.function +++ b/tests/suites/host_test.function @@ -368,6 +368,118 @@ static int run_test_snprintf( void ) test_snprintf( 5, "123", 3 ) != 0 ); } +/** \brief Write the description of the test case to the outcome CSV file. + * + * \param outcome_file The file to write to. + * If this is \c NULL, this function does nothing. + * \param argv0 The test suite name. + * \param test_case The test case description. + */ +static void write_outcome_entry( FILE *outcome_file, + const char *argv0, + const char *test_case ) +{ + /* The non-varying fields are initialized on first use. */ + static const char *platform = NULL; + static const char *configuration = NULL; + static const char *test_suite = NULL; + + if( outcome_file == NULL ) + return; + + if( platform == NULL ) + { + platform = getenv( "MBEDTLS_TEST_PLATFORM" ); + if( platform == NULL ) + platform = "unknown"; + } + if( configuration == NULL ) + { + configuration = getenv( "MBEDTLS_TEST_CONFIGURATION" ); + if( configuration == NULL ) + configuration = "unknown"; + } + if( test_suite == NULL ) + { + test_suite = strrchr( argv0, '/' ); + if( test_suite != NULL ) + test_suite += 1; // skip the '/' + else + test_suite = argv0; + } + + /* Write the beginning of the outcome line. + * Ignore errors: writing the outcome file is on a best-effort basis. */ + mbedtls_fprintf( outcome_file, "%s;%s;%s;%s;", + platform, configuration, test_suite, test_case ); +} + +/** \brief Write the result of the test case to the outcome CSV file. + * + * \param outcome_file The file to write to. + * If this is \c NULL, this function does nothing. + * \param unmet_dep_count The number of unmet dependencies. + * \param unmet_dependencies The array of unmet dependencies. + * \param ret The test dispatch status (DISPATCH_xxx). + * \param test_info A pointer to the test info structure. + */ +static void write_outcome_result( FILE *outcome_file, + size_t unmet_dep_count, + char *unmet_dependencies[], + int ret, + const test_info_t *info ) +{ + if( outcome_file == NULL ) + return; + + /* Write the end of the outcome line. + * Ignore errors: writing the outcome file is on a best-effort basis. */ + switch( ret ) + { + case DISPATCH_TEST_SUCCESS: + if( unmet_dep_count > 0 ) + { + size_t i; + mbedtls_fprintf( outcome_file, "SKIP" ); + for( i = 0; i < unmet_dep_count; i++ ) + { + mbedtls_fprintf( outcome_file, "%c%s", + i == 0 ? ';' : ':', + unmet_dependencies[i] ); + } + break; + } + switch( info->result ) + { + case TEST_RESULT_SUCCESS: + mbedtls_fprintf( outcome_file, "PASS;" ); + break; + case TEST_RESULT_SKIPPED: + mbedtls_fprintf( outcome_file, "SKIP;Runtime skip" ); + break; + default: + mbedtls_fprintf( outcome_file, "FAIL;%s:%d:%s", + info->filename, info->line_no, + info->test ); + break; + } + break; + case DISPATCH_TEST_FN_NOT_FOUND: + mbedtls_fprintf( outcome_file, "FAIL;Test function not found" ); + break; + case DISPATCH_INVALID_TEST_DATA: + mbedtls_fprintf( outcome_file, "FAIL;Invalid test data" ); + break; + case DISPATCH_UNSUPPORTED_SUITE: + mbedtls_fprintf( outcome_file, "SKIP;Unsupported suite" ); + break; + default: + mbedtls_fprintf( outcome_file, "FAIL;Unknown cause" ); + break; + } + mbedtls_fprintf( outcome_file, "\n" ); + fflush( outcome_file ); +} /** * \brief Desktop implementation of execute_tests(). @@ -385,15 +497,16 @@ int execute_tests( int argc , const char ** argv ) const char *default_filename = "DATA_FILE"; const char *test_filename = NULL; const char **test_files = NULL; - int testfile_count = 0; + size_t testfile_count = 0; int option_verbose = 0; int function_id = 0; /* Other Local variables */ int arg_index = 1; const char *next_arg; - int testfile_index, ret, i, cnt; - int total_errors = 0, total_tests = 0, total_skipped = 0; + size_t testfile_index, i, cnt; + int ret; + unsigned total_errors = 0, total_tests = 0, total_skipped = 0; FILE *file; char buf[5000]; char *params[50]; @@ -403,6 +516,8 @@ int execute_tests( int argc , const char ** argv ) #if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) int stdout_fd = -1; #endif /* __unix__ || __APPLE__ __MACH__ */ + const char *outcome_file_name = getenv( "MBEDTLS_TEST_OUTCOME_FILE" ); + FILE *outcome_file = NULL; #if defined(MBEDTLS_MEMORY_BUFFER_ALLOC_C) && \ !defined(TEST_SUITE_MEMORY_BUFFER_ALLOC) @@ -410,6 +525,15 @@ int execute_tests( int argc , const char ** argv ) mbedtls_memory_buffer_alloc_init( alloc_buf, sizeof( alloc_buf ) ); #endif + if( outcome_file_name != NULL ) + { + outcome_file = fopen( outcome_file_name, "a" ); + if( outcome_file == NULL ) + { + mbedtls_fprintf( stderr, "Unable to open outcome file. Continuing anyway.\n" ); + } + } + /* * The C standard doesn't guarantee that all-bits-0 is the representation * of a NULL pointer. We do however use that in our code for initializing @@ -473,7 +597,7 @@ int execute_tests( int argc , const char ** argv ) testfile_index < testfile_count; testfile_index++ ) { - int unmet_dep_count = 0; + size_t unmet_dep_count = 0; char *unmet_dependencies[20]; test_filename = test_files[ testfile_index ]; @@ -505,6 +629,7 @@ int execute_tests( int argc , const char ** argv ) mbedtls_fprintf( stdout, "." ); mbedtls_fprintf( stdout, " " ); fflush( stdout ); + write_outcome_entry( outcome_file, argv[0], buf ); total_tests++; @@ -585,6 +710,9 @@ int execute_tests( int argc , const char ** argv ) } + write_outcome_result( outcome_file, + unmet_dep_count, unmet_dependencies, + ret, &test_info ); if( unmet_dep_count > 0 || ret == DISPATCH_UNSUPPORTED_SUITE ) { total_skipped++; @@ -645,7 +773,7 @@ int execute_tests( int argc , const char ** argv ) } else if( ret == DISPATCH_TEST_FN_NOT_FOUND ) { - mbedtls_fprintf( stderr, "FAILED: FATAL TEST FUNCTION NOT FUND\n" ); + mbedtls_fprintf( stderr, "FAILED: FATAL TEST FUNCTION NOT FOUND\n" ); fclose( file ); mbedtls_exit( 2 ); } @@ -659,14 +787,17 @@ int execute_tests( int argc , const char ** argv ) free( unmet_dependencies[i] ); } + if( outcome_file != NULL ) + fclose( outcome_file ); + mbedtls_fprintf( stdout, "\n----------------------------------------------------------------------------\n\n"); if( total_errors == 0 ) mbedtls_fprintf( stdout, "PASSED" ); else mbedtls_fprintf( stdout, "FAILED" ); - mbedtls_fprintf( stdout, " (%d / %d tests (%d skipped))\n", - total_tests - total_errors, total_tests, total_skipped ); + mbedtls_fprintf( stdout, " (%u / %u tests (%u skipped))\n", + total_tests - total_errors, total_tests, total_skipped ); #if defined(MBEDTLS_MEMORY_BUFFER_ALLOC_C) && \ !defined(TEST_SUITE_MEMORY_BUFFER_ALLOC) diff --git a/tests/suites/test_suite_version.data b/tests/suites/test_suite_version.data index a4575ab00..b6dca233b 100644 --- a/tests/suites/test_suite_version.data +++ b/tests/suites/test_suite_version.data @@ -1,8 +1,8 @@ Check compiletime library version -check_compiletime_version:"2.17.0" +check_compiletime_version:"2.19.1" Check runtime library version -check_runtime_version:"2.17.0" +check_runtime_version:"2.19.1" Check for MBEDTLS_VERSION_C check_feature:"MBEDTLS_VERSION_C":0