From 67ffdafde6e17f345e1a8560971bd6d7ce018c24 Mon Sep 17 00:00:00 2001
From: Gilles Peskine <Gilles.Peskine@arm.com>
Date: Mon, 16 Sep 2019 15:55:46 +0200
Subject: [PATCH] all.sh --outcome-file creates an outcome file

By default, remove the outcome file before starting. With
--append-outcome, append to the existing outcome file if there is one.
---
 tests/scripts/all.sh | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh
index a97189a90..4d31b0ecd 100755
--- a/tests/scripts/all.sh
+++ b/tests/scripts/all.sh
@@ -117,11 +117,14 @@ pre_initialize_variables () {
     CONFIG_H='include/mbedtls/config.h'
     CONFIG_BAK="$CONFIG_H.bak"
 
+    append_outcome=0
     MEMORY=0
     FORCE=0
     KEEP_GOING=0
 
+    : ${MBEDTLS_TEST_OUTCOME_FILE=}
     : ${MBEDTLS_TEST_PLATFORM="$(uname -s | tr -c \\n0-9A-Za-z _)-$(uname -m | tr -c \\n0-9A-Za-z _)"}
+    export MBEDTLS_TEST_OUTCOME_FILE
     export MBEDTLS_TEST_PLATFORM
 
     # Default commands, can be overridden by the environment
@@ -193,14 +196,18 @@ General options:
   -f|--force            Force the tests to overwrite any modified files.
   -k|--keep-going       Run all tests and report errors at the end.
   -m|--memory           Additional optional memory tests.
+     --append-outcome   Append to the outcome file (if used).
      --armcc            Run ARM Compiler builds (on by default).
      --except           Exclude the COMPONENTs listed on the command line,
                         instead of running only those.
+     --no-append-outcome    Write a new outcome file and analyze it (default).
      --no-armcc         Skip ARM Compiler builds.
      --no-force         Refuse to overwrite modified files (default).
      --no-keep-going    Stop at the first error (default).
      --no-memory        No additional memory tests (default).
      --out-of-source-dir=<path>  Directory used for CMake out-of-source build tests.
+     --outcome-file=<path>  File where test outcomes are written (not done if
+                            empty; default: \$MBEDTLS_TEST_OUTCOME_FILE).
      --random-seed      Use a random seed value for randomized tests (default).
   -r|--release-test     Run this script in release mode. This fixes the seed value to 1.
   -s|--seed             Integer seed value to use for this test run.
@@ -326,6 +333,7 @@ pre_parse_command_line () {
 
     while [ $# -gt 0 ]; do
         case "$1" in
+            --append-outcome) append_outcome=1;;
             --armcc) no_armcc=;;
             --armc5-bin-dir) shift; ARMC5_BIN_DIR="$1";;
             --armc6-bin-dir) shift; ARMC6_BIN_DIR="$1";;
@@ -340,6 +348,7 @@ pre_parse_command_line () {
             --list-all-components) printf '%s\n' $ALL_COMPONENTS; exit;;
             --list-components) printf '%s\n' $SUPPORTED_COMPONENTS; exit;;
             --memory|-m) MEMORY=1;;
+            --no-append-outcome) append_outcome=0;;
             --no-armcc) no_armcc=1;;
             --no-force) FORCE=0;;
             --no-keep-going) KEEP_GOING=0;;
@@ -347,6 +356,7 @@ pre_parse_command_line () {
             --openssl) shift; OPENSSL="$1";;
             --openssl-legacy) shift; OPENSSL_LEGACY="$1";;
             --openssl-next) shift; OPENSSL_NEXT="$1";;
+            --outcome-file) shift; MBEDTLS_TEST_OUTCOME_FILE="$1";;
             --out-of-source-dir) shift; OUT_OF_SOURCE_DIR="$1";;
             --random-seed) unset SEED;;
             --release-test|-r) SEED=1;;
@@ -488,11 +498,22 @@ not() {
     ! "$@"
 }
 
+pre_prepare_outcome_file () {
+    case "$MBEDTLS_TEST_OUTCOME_FILE" in
+      [!/]*) MBEDTLS_TEST_OUTCOME_FILE="$PWD/$MBEDTLS_TEST_OUTCOME_FILE";;
+    esac
+    if [ -n "$MBEDTLS_TEST_OUTCOME_FILE" ] && [ "$append_outcome" -eq 0 ]; then
+        rm -f "$MBEDTLS_TEST_OUTCOME_FILE"
+    fi
+}
+
 pre_print_configuration () {
     msg "info: $0 configuration"
     echo "MEMORY: $MEMORY"
     echo "FORCE: $FORCE"
+    echo "MBEDTLS_TEST_OUTCOME_FILE: ${MBEDTLS_TEST_OUTCOME_FILE:-(none)}"
     echo "SEED: ${SEED-"UNSET"}"
+    echo
     echo "OPENSSL: $OPENSSL"
     echo "OPENSSL_LEGACY: $OPENSSL_LEGACY"
     echo "OPENSSL_NEXT: $OPENSSL_NEXT"
@@ -640,6 +661,8 @@ component_test_large_ecdsa_key_signature () {
 component_test_default_out_of_box () {
     msg "build: make, default config (out-of-box)" # ~1min
     make
+    # Disable fancy stuff
+    unset MBEDTLS_TEST_OUTCOME_FILE
 
     msg "test: main suites make, default config (out-of-box)" # ~10s
     make test
@@ -1448,6 +1471,7 @@ else
         "$@"
     }
 fi
+pre_prepare_outcome_file
 pre_print_configuration
 pre_check_tools
 cleanup