From dd7ea389ffe2e7afc5e85ce9c2aa2b1b0d05f7f1 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 9 Apr 2020 18:28:14 +0200 Subject: [PATCH 1/5] Exit with a failure status if some tests failed Signed-off-by: Gilles Peskine --- tests/scripts/basic-build-test.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh index 97120ea84..52bcffc7a 100755 --- a/tests/scripts/basic-build-test.sh +++ b/tests/scripts/basic-build-test.sh @@ -217,3 +217,7 @@ make clean if [ -f "$CONFIG_BAK" ]; then mv "$CONFIG_BAK" "$CONFIG_H" fi + +if [ $TOTAL_FAIL -ne 0 ]; then + exit 1 +fi From 27c36f050f8c0244a62d1659aa4cc0fd1720cdea Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 9 Apr 2020 18:29:42 +0200 Subject: [PATCH 2/5] Note that we keep going even if some tests fail Signed-off-by: Gilles Peskine --- tests/scripts/basic-build-test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh index 52bcffc7a..0fc7413f2 100755 --- a/tests/scripts/basic-build-test.sh +++ b/tests/scripts/basic-build-test.sh @@ -75,15 +75,15 @@ make -j TEST_OUTPUT=out_${PPID} cd tests -# Step 2a - Unit Tests +# Step 2a - Unit Tests (keep going even if some tests fail) perl scripts/run-test-suites.pl -v 2 |tee unit-test-$TEST_OUTPUT echo -# Step 2b - System Tests +# Step 2b - System Tests (keep going even if some tests fail) sh ssl-opt.sh |tee sys-test-$TEST_OUTPUT echo -# Step 2c - Compatibility tests +# Step 2c - Compatibility tests (keep going even if some tests fail) sh compat.sh -m 'tls1 tls1_1 tls1_2 dtls1 dtls1_2' | \ tee compat-test-$TEST_OUTPUT OPENSSL_CMD="$OPENSSL_LEGACY" \ From b07541d9f008cf88d927752f8994f5c723c13cc6 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 9 Apr 2020 18:32:48 +0200 Subject: [PATCH 3/5] If 'make lcov' failed, exit immediately Signed-off-by: Gilles Peskine --- tests/scripts/basic-build-test.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh index 0fc7413f2..9c00d66c5 100755 --- a/tests/scripts/basic-build-test.sh +++ b/tests/scripts/basic-build-test.sh @@ -97,7 +97,15 @@ echo # Step 3 - Process the coverage report cd .. -make lcov |tee tests/cov-$TEST_OUTPUT +{ + make lcov + echo SUCCESS +} | tee tests/cov-$TEST_OUTPUT + +if [ "$(tail -n1 tests/cov-$TEST_OUTPUT)" != "SUCCESS" ]; then + echo >&2 "Fatal: 'make lcov' failed" + exit 2 +fi # Step 4 - Summarise the test report From 9cc203a7eaff6faca88ca8c64479a386fc7b3c9a Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 9 Apr 2020 18:33:34 +0200 Subject: [PATCH 4/5] Create a seedfile explicitly Running the entropy unit test creates a suitable seedfile, but this only works due to the happy accident that no prior unit test needs one (specifically, test_suite_entropy runs before test_suite_rsa). So create one explicitly, both for robustness and to keep the script closer to the version in development where the explicit seedfile creation is required. Signed-off-by: Gilles Peskine --- tests/scripts/basic-build-test.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh index 9c00d66c5..40db80c29 100755 --- a/tests/scripts/basic-build-test.sh +++ b/tests/scripts/basic-build-test.sh @@ -75,6 +75,10 @@ make -j TEST_OUTPUT=out_${PPID} cd tests +if [ ! -f "seedfile" ]; then + dd if=/dev/urandom of="seedfile" bs=64 count=1 +fi + # Step 2a - Unit Tests (keep going even if some tests fail) perl scripts/run-test-suites.pl -v 2 |tee unit-test-$TEST_OUTPUT echo From 39b610227b31887536d3ab72dfb55337c77a9348 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 9 Apr 2020 19:47:48 +0200 Subject: [PATCH 5/5] MBEDTLS_MEMORY_BACKTRACE is no longer included in the full config Signed-off-by: Gilles Peskine --- tests/scripts/basic-build-test.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh index 40db80c29..836466616 100755 --- a/tests/scripts/basic-build-test.sh +++ b/tests/scripts/basic-build-test.sh @@ -67,7 +67,6 @@ export CFLAGS=' --coverage -g3 -O0 ' make clean cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.pl full -scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE make -j