From f54a5de66ad36c2f7a59b26607f530fb2e2b0569 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 9 Apr 2020 18:28:14 +0200 Subject: [PATCH 1/5] Exit with a failure status if some tests failed Signed-off-by: Gilles Peskine --- tests/scripts/basic-build-test.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh index 8990dfbea..dee765cfa 100755 --- a/tests/scripts/basic-build-test.sh +++ b/tests/scripts/basic-build-test.sh @@ -220,3 +220,7 @@ make clean if [ -f "$CONFIG_BAK" ]; then mv "$CONFIG_BAK" "$CONFIG_H" fi + +if [ $TOTAL_FAIL -ne 0 ]; then + exit 1 +fi From 8bfe12b1ff40fb5fc6eb3903a1e5ba03323ebb19 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 9 Apr 2020 18:29:42 +0200 Subject: [PATCH 2/5] Note that we keep going even if some tests fail Signed-off-by: Gilles Peskine --- tests/scripts/basic-build-test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh index dee765cfa..8428799c3 100755 --- a/tests/scripts/basic-build-test.sh +++ b/tests/scripts/basic-build-test.sh @@ -75,15 +75,15 @@ make -j TEST_OUTPUT=out_${PPID} cd tests -# Step 2a - Unit Tests +# Step 2a - Unit Tests (keep going even if some tests fail) perl scripts/run-test-suites.pl -v 2 |tee unit-test-$TEST_OUTPUT echo -# Step 2b - System Tests +# Step 2b - System Tests (keep going even if some tests fail) sh ssl-opt.sh |tee sys-test-$TEST_OUTPUT echo -# Step 2c - Compatibility tests +# Step 2c - Compatibility tests (keep going even if some tests fail) sh compat.sh -m 'tls1 tls1_1 tls1_2 dtls1 dtls1_2' | \ tee compat-test-$TEST_OUTPUT OPENSSL_CMD="$OPENSSL_LEGACY" \ From ed1f6735259eddbce4473e1ffdac2aa3909ab5c9 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 9 Apr 2020 18:32:48 +0200 Subject: [PATCH 3/5] If 'make lcov' failed, exit immediately Signed-off-by: Gilles Peskine --- tests/scripts/basic-build-test.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh index 8428799c3..1862786a2 100755 --- a/tests/scripts/basic-build-test.sh +++ b/tests/scripts/basic-build-test.sh @@ -100,7 +100,15 @@ echo # Step 3 - Process the coverage report cd .. -make lcov |tee tests/cov-$TEST_OUTPUT +{ + make lcov + echo SUCCESS +} | tee tests/cov-$TEST_OUTPUT + +if [ "$(tail -n1 tests/cov-$TEST_OUTPUT)" != "SUCCESS" ]; then + echo >&2 "Fatal: 'make lcov' failed" + exit 2 +fi # Step 4 - Summarise the test report From d9701ae21edc66e8f86b24192cf22d450a68e66c Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 9 Apr 2020 18:33:34 +0200 Subject: [PATCH 4/5] Create a seedfile explicitly Running the entropy unit test creates a suitable seedfile, but this only works due to the happy accident that no prior unit test needs one (specifically, test_suite_entropy runs before test_suite_rsa). So create one explicitly, both for robustness and to keep the script closer to the version in development where the explicit seedfile creation is required. Signed-off-by: Gilles Peskine --- tests/scripts/basic-build-test.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh index 1862786a2..1a1a7b9b9 100755 --- a/tests/scripts/basic-build-test.sh +++ b/tests/scripts/basic-build-test.sh @@ -75,6 +75,10 @@ make -j TEST_OUTPUT=out_${PPID} cd tests +if [ ! -f "seedfile" ]; then + dd if=/dev/urandom of="seedfile" bs=64 count=1 +fi + # Step 2a - Unit Tests (keep going even if some tests fail) perl scripts/run-test-suites.pl -v 2 |tee unit-test-$TEST_OUTPUT echo From f25cd6ba5f98fcd30344acf2209219b512062ea6 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 9 Apr 2020 19:47:48 +0200 Subject: [PATCH 5/5] MBEDTLS_MEMORY_BACKTRACE is no longer included in the full config Signed-off-by: Gilles Peskine --- tests/scripts/basic-build-test.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh index 1a1a7b9b9..8d4604ab7 100755 --- a/tests/scripts/basic-build-test.sh +++ b/tests/scripts/basic-build-test.sh @@ -67,7 +67,6 @@ export CFLAGS=' --coverage -g3 -O0 ' make clean cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full -scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE make -j