From 6d6ee98c140e70fb8b07301e270081eb1fe91d13 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 9 Apr 2020 18:28:14 +0200 Subject: [PATCH 1/7] Exit with a failure status if some tests failed Signed-off-by: Gilles Peskine --- tests/scripts/basic-build-test.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh index 6419f05e4..7490777ec 100755 --- a/tests/scripts/basic-build-test.sh +++ b/tests/scripts/basic-build-test.sh @@ -224,3 +224,7 @@ make clean if [ -f "$CONFIG_BAK" ]; then mv "$CONFIG_BAK" "$CONFIG_H" fi + +if [ $TOTAL_FAIL -ne 0 ]; then + exit 1 +fi From ca51b474dc2966576a98a59d094647423fb0dd35 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 9 Apr 2020 18:29:42 +0200 Subject: [PATCH 2/7] Note that we keep going even if some tests fail Signed-off-by: Gilles Peskine --- tests/scripts/basic-build-test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh index 7490777ec..a2f12f890 100755 --- a/tests/scripts/basic-build-test.sh +++ b/tests/scripts/basic-build-test.sh @@ -79,15 +79,15 @@ if [ ! -f "seedfile" ]; then dd if=/dev/urandom of="seedfile" bs=32 count=1 fi -# Step 2a - Unit Tests +# Step 2a - Unit Tests (keep going even if some tests fail) perl scripts/run-test-suites.pl -v 2 |tee unit-test-$TEST_OUTPUT echo -# Step 2b - System Tests +# Step 2b - System Tests (keep going even if some tests fail) sh ssl-opt.sh |tee sys-test-$TEST_OUTPUT echo -# Step 2c - Compatibility tests +# Step 2c - Compatibility tests (keep going even if some tests fail) sh compat.sh -m 'tls1 tls1_1 tls1_2 dtls1 dtls1_2' | \ tee compat-test-$TEST_OUTPUT OPENSSL_CMD="$OPENSSL_LEGACY" \ From 5757d542619bc84e1a1b2ff9eecb35195fbc1cbd Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 9 Apr 2020 18:32:48 +0200 Subject: [PATCH 3/7] If 'make lcov' failed, exit immediately Signed-off-by: Gilles Peskine --- tests/scripts/basic-build-test.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh index a2f12f890..e248c2267 100755 --- a/tests/scripts/basic-build-test.sh +++ b/tests/scripts/basic-build-test.sh @@ -104,7 +104,15 @@ echo # Step 3 - Process the coverage report cd .. -make lcov |tee tests/cov-$TEST_OUTPUT +{ + make lcov + echo SUCCESS +} | tee tests/cov-$TEST_OUTPUT + +if [ "$(tail -n1 tests/cov-$TEST_OUTPUT)" != "SUCCESS" ]; then + echo >&2 "Fatal: 'make lcov' failed" + exit 2 +fi # Step 4 - Summarise the test report From bfcb6e16ab669814478b1de63e48d4ff6b408711 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 9 Apr 2020 18:33:34 +0200 Subject: [PATCH 4/7] Create a large enough seedfile The seedfile needs to have the size of the entropy accumulator, which is 64 bytes (512 bits) since the entropy accumulator uses SHA-512 and the seed size needs to be the same as the hash output (or larger). We used to enable MBEDTLS_ENTROPY_FORCE_SHA256 in the full config, so the entropy accumulator was 256 bits (32 bytes), and therefore a 32-byte seedfile worked. But we no longer turn on this option in the full config, so the 32-byte seedfile no longer works. Signed-off-by: Gilles Peskine --- tests/scripts/basic-build-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh index e248c2267..bbfcf25c8 100755 --- a/tests/scripts/basic-build-test.sh +++ b/tests/scripts/basic-build-test.sh @@ -76,7 +76,7 @@ make -j TEST_OUTPUT=out_${PPID} cd tests if [ ! -f "seedfile" ]; then - dd if=/dev/urandom of="seedfile" bs=32 count=1 + dd if=/dev/urandom of="seedfile" bs=64 count=1 fi # Step 2a - Unit Tests (keep going even if some tests fail) From 40be51ffe32f729dbaf8da8876faa60d83caaa09 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 9 Apr 2020 18:50:08 +0200 Subject: [PATCH 5/7] Add a few echo statements to make the logs easier to follow Signed-off-by: Gilles Peskine --- tests/scripts/basic-build-test.sh | 39 +++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh index bbfcf25c8..1b92e4183 100755 --- a/tests/scripts/basic-build-test.sh +++ b/tests/scripts/basic-build-test.sh @@ -78,28 +78,43 @@ cd tests if [ ! -f "seedfile" ]; then dd if=/dev/urandom of="seedfile" bs=64 count=1 fi +echo # Step 2a - Unit Tests (keep going even if some tests fail) +echo '################ Unit tests ################' perl scripts/run-test-suites.pl -v 2 |tee unit-test-$TEST_OUTPUT +echo '^^^^^^^^^^^^^^^^ Unit tests ^^^^^^^^^^^^^^^^' echo # Step 2b - System Tests (keep going even if some tests fail) +echo +echo '################ ssl-opt.sh ################' sh ssl-opt.sh |tee sys-test-$TEST_OUTPUT +echo '^^^^^^^^^^^^^^^^ ssl-opt.sh ^^^^^^^^^^^^^^^^' echo # Step 2c - Compatibility tests (keep going even if some tests fail) -sh compat.sh -m 'tls1 tls1_1 tls1_2 dtls1 dtls1_2' | \ - tee compat-test-$TEST_OUTPUT -OPENSSL_CMD="$OPENSSL_LEGACY" \ - sh compat.sh -m 'ssl3' |tee -a compat-test-$TEST_OUTPUT -OPENSSL_CMD="$OPENSSL_LEGACY" \ - GNUTLS_CLI="$GNUTLS_LEGACY_CLI" \ - GNUTLS_SERV="$GNUTLS_LEGACY_SERV" \ - sh compat.sh -e '^$' -f 'NULL\|DES\|RC4\|ARCFOUR' | \ - tee -a compat-test-$TEST_OUTPUT -OPENSSL_CMD="$OPENSSL_NEXT" \ - sh compat.sh -e '^$' -f 'ARIA\|CHACHA' | \ - tee -a compat-test-$TEST_OUTPUT +echo '################ compat.sh ################' +{ + echo '#### compat.sh: Default versions' + sh compat.sh -m 'tls1 tls1_1 tls1_2 dtls1 dtls1_2' + echo + + echo '#### compat.sh: legacy (SSLv3)' + OPENSSL_CMD="$OPENSSL_LEGACY" sh compat.sh -m 'ssl3' + echo + + echo '#### compat.sh: legacy (null, DES, RC4)' + OPENSSL_CMD="$OPENSSL_LEGACY" \ + GNUTLS_CLI="$GNUTLS_LEGACY_CLI" GNUTLS_SERV="$GNUTLS_LEGACY_SERV" \ + sh compat.sh -e '^$' -f 'NULL\|DES\|RC4\|ARCFOUR' + echo + + echo '#### compat.sh: next (ARIA, ChaCha)' + OPENSSL_CMD="$OPENSSL_NEXT" sh compat.sh -e '^$' -f 'ARIA\|CHACHA' + echo +} | tee compat-test-$TEST_OUTPUT +echo '^^^^^^^^^^^^^^^^ compat.sh ^^^^^^^^^^^^^^^^' echo # Step 3 - Process the coverage report From b3d0bab10b32d113deeeba4704da0f184d0d1a78 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 9 Apr 2020 19:47:48 +0200 Subject: [PATCH 6/7] MBEDTLS_MEMORY_BACKTRACE is no longer included in the full config Signed-off-by: Gilles Peskine --- tests/scripts/basic-build-test.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh index 1b92e4183..08c141052 100755 --- a/tests/scripts/basic-build-test.sh +++ b/tests/scripts/basic-build-test.sh @@ -68,7 +68,6 @@ export LDFLAGS=' --coverage' make clean cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.py full -scripts/config.py unset MBEDTLS_MEMORY_BACKTRACE make -j From c877c24ed0514ad0e398d0fd6ca7db3593bef740 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 10 Apr 2020 00:18:55 +0200 Subject: [PATCH 7/7] Enable extra features that have tests The "full" configuration excludes some deprecated or experimental features. Enable the ones that have tests, don't have extra requirements and don't turn off some other feature. Signed-off-by: Gilles Peskine --- tests/scripts/basic-build-test.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/scripts/basic-build-test.sh b/tests/scripts/basic-build-test.sh index 08c141052..aca2f11fb 100755 --- a/tests/scripts/basic-build-test.sh +++ b/tests/scripts/basic-build-test.sh @@ -68,6 +68,10 @@ export LDFLAGS=' --coverage' make clean cp "$CONFIG_H" "$CONFIG_BAK" scripts/config.py full +# Enable some deprecated or experimental features that are not in the +# full config, but are compatible with it and have tests. +scripts/config.py set MBEDTLS_SSL_PROTO_SSL3 +scripts/config.py set MBEDTLS_PSA_CRYPTO_SE_C make -j