added valgrind tests to testme.sh and travis.yml

.travis.yml

@@ -1,35 +1,133 @@
-language: c
+#############################################################################
+#                                                                           #
+#      Travis-ci test-suite for LibTomMath                                  #
+#      (https://github.com/libtom/libtommath.git)                           #
+#                                                                           #
+#############################################################################
 
-install:
+# Compilation failures are in gcc_errors_*.log
-    - sudo apt-get update -qq
+# Failed tests in test_*.log
-    - sudo apt-get install gcc-multilib
+# Files do not exist in case of success
+after_failure:
+  - cat test_*.log
+  - cat gcc_errors_*.log
 
-matrix:
+# In case of a Travis error a success might get signaled
-  fast_finish: true
+# even without any test run. This file also keeps any notes
+# printed from the tests which might come handy from time
+# to time.
+# Valgrid will print its output to stderr which will not show up
+# in test_*.log. testme.sh accepts one additional option to
+# valgrind and "--valgrind-options=--log-fd=1" sends the output
+# of Valgrind to stdout instead.
+after_success:
+  - cat test_*.log
+
+# Tests restricted to the following branches of LTM.
 branches:
   only:
     - master
     - develop
     - /^release\/.*$/
 
-compiler:
+# Additional installs are Valgrind for the memory-tests
-  - gcc
+# and gcc-multilib for the compilation of the different
-  - clang
+# architectures.
-script:
+install:
-  - ./testme.sh --with-cc=$CC ${BUILDOPTIONS}
+  - sudo apt-get update -qq
-env:
+  - sudo apt-get install valgrind
-  - |
+  - sudo apt-get install gcc-multilib
-    BUILDOPTIONS="--test-vs-mtest=333333"
-  - |
-    BUILDOPTIONS="--test-vs-mtest=333333 --mtest-real-rand"
-  - |
-    BUILDOPTIONS="--with-low-mp"
-  - |
-    BUILDOPTIONS="--with-m64 --with-m32 --with-mx32"
 
-after_failure:
+# The language is C and it will load the respective dependencies
-  - cat test_*.log
+language: c
-  - cat gcc_errors_*.log
 
+# The actual workspace. Will run the individual jobs in parallel
+# which also means that the jobs must be able to run in parallel.
+# Either specify sets which will be combined or, as in this case,
+# specify all builds individually. The number of jobs is currently
+# restricted to 200 jobs at most.
+matrix:
+  # Will mark as finished if all of the remaining tests are allowed to fail
+  # or one test has failed already.
+  fast_finish: true
+
+  # The individual jobs
+  include:
+    # The environment given to the programs in the build
+    # We have only one program and the variable $BUILDOPTIONS
+    # has only the options to that program: testme.sh
+
+    # GCC for the 32-bit architecture (no valgrind yet)
+    - env: BUILDOPTIONS='--with-cc=gcc --with-m32'
+      addons:
+        apt:
+          packages:
+            - libc6-dev-i386
+      sudo: required
+
+    # clang for the 32-bit architecture (no valgrind yet)
+    - env: BUILDOPTIONS='--with-cc=clang --with-m32'
+      addons:
+        apt:
+          packages:
+            - libc6-dev-i386
+      sudo: required
+
+    # GCC for the x64_32 architecture (32-bit longs and 32-bit pointers)
+    # TODO: Probably not possible to run anything in x32 in Travis
+    #       but needs to be checked to be sure.
+    - env: BUILDOPTIONS='--with-cc=gcc --with-mx32'
+      addons:
+        apt:
+          packages:
+            - libc6-dev-x32
+      sudo: required
+
+    # clang for the x64_32 architecture
+    - env:  BUILDOPTIONS='--with-cc=clang --with-mx32'
+      addons:
+        apt:
+          packages:
+            - libc6-dev-x32
+      sudo: required
+
+    # GCC for the x86-64 architecture (64-bit longs and 64-bit pointers)
+    - env:  BUILDOPTIONS='--with-cc=gcc --with-m64 --with-valgrind'
+    # clang for x86-64 architecture (64-bit longs and 64-bit pointers)
+    - env:  BUILDOPTIONS='--with-cc=clang --with-m64 --with-valgrind'
+
+    # GCC for the x86-64 architecture with restricted limb sizes
+    # formerly started with the option "--with-low-mp" to testme.sh
+    # but testing all three in one run took to long and timed out.
+    - env:  BUILDOPTIONS='--with-cc=gcc --cflags=-DMP_8BIT  --with-valgrind'
+    - env:  BUILDOPTIONS='--with-cc=gcc --cflags=-DMP_16BIT --with-valgrind'
+    - env:  BUILDOPTIONS='--with-cc=gcc --cflags=-DMP_32BIT --with-valgrind'
+
+    # clang for the x86-64 architecture with restricted limb sizes
+    - env:  BUILDOPTIONS='--with-cc=clang --cflags=-DMP_8BIT  --with-valgrind'
+    - env:  BUILDOPTIONS='--with-cc=clang --cflags=-DMP_16BIT --with-valgrind'
+    - env:  BUILDOPTIONS='--with-cc=clang --cflags=-DMP_32BIT --with-valgrind'
+
+    # GCC for the x86-64 architecture testing against a different Bigint-implementation
+    # with 333333 different inputs.
+    - env:  BUILDOPTIONS='--with-cc=gcc --test-vs-mtest=333333 --with-valgrind'
+    - env:  BUILDOPTIONS='--with-cc=clang --test-vs-mtest=333333 --with-valgrind'
+
+    # clang for the x86-64 architecture testing against a different Bigint-implementation
+    # with a better random source.
+    - env:  BUILDOPTIONS='--with-cc=gcc --test-vs-mtest=333333 --mtest-real-rand --with-valgrind'
+    - env:  BUILDOPTIONS='--with-cc=clang --test-vs-mtest=333333 --mtest-real-rand --with-valgrind'
+
+# Notifications go to
+# An email address is also possible.
 notifications:
   irc: "chat.freenode.net#libtom-notifications"
+
+# The actual script the jobs run.
+# Because of a default timeout of 10 minutes it was necessary to use
+# a Travis tool to extend that timeout to 40 minutes. 50 minutes
+# seem to be the max and 20 the default if travis_wait is called without
+# any options.
+script:
+  - travis_wait 40 sh -c "./testme.sh  ${BUILDOPTIONS}"
+

testme.sh

@@ -21,40 +21,63 @@ _help()
 {
   echo "Usage options for $(basename $0) [--with-cc=arg [other options]]"
   echo
-  echo "Executing this script without any parameter will only run the default configuration"
+  echo "Executing this script without any parameter will only run the default"
-  echo "that has automatically been determined for the architecture you're running."
+  echo "configuration that has automatically been determined for the"
+  echo "architecture you're running."
   echo
   echo "    --with-cc=*             The compiler(s) to use for the tests"
-  echo "        This is an option that will be iterated."
+  echo "                            This is an option that will be iterated."
   echo
   echo "    --test-vs-mtest=*       Run test vs. mtest for '*' operations."
-  echo "        Only the first of each options will be taken into account."
+  echo "                            Only the first of each options will be"
+  echo "                            taken into account."
   echo
-  echo "To be able to specify options a compiler has to be given."
+  echo "To be able to specify options a compiler has to be given with"
-  echo "All options will be tested with all MP_xBIT configurations."
+  echo "the option --with-cc=compilername"
+  echo "All other options will be tested with all MP_xBIT configurations."
   echo
-  echo "    --with-{m64,m32,mx32}   The architecture(s) to build and test for,"
+  echo "    --with-{m64,m32,mx32}   The architecture(s) to build and test"
-  echo "                            e.g. --with-mx32."
+  echo "                            for, e.g. --with-mx32."
-  echo "        This is an option that will be iterated, multiple selections are possible."
+  echo "                            This is an option that will be iterated,"
-  echo "        The mx32 architecture is not supported by clang and will not be executed."
+  echo "                            multiple selections are possible."
+  echo "                            The mx32 architecture is not supported"
+  echo "                            by clang and will not be executed."
   echo
   echo "    --cflags=*              Give an option to the compiler,"
   echo "                            e.g. --cflags=-g"
-  echo "        This is an option that will always be passed as parameter to CC."
+  echo "                            This is an option that will always be"
+  echo "                            passed as parameter to CC."
   echo
   echo "    --make-option=*         Give an option to make,"
   echo "                            e.g. --make-option=\"-f makefile.shared\""
-  echo "        This is an option that will always be passed as parameter to make."
+  echo "                            This is an option that will always be"
+  echo "                            passed as parameter to make."
   echo
   echo "    --with-low-mp           Also build&run tests with -DMP_{8,16,32}BIT."
   echo
   echo "    --mtest-real-rand       Use real random data when running mtest."
   echo
+  echo "    --with-valgrind"
+  echo "    --with-valgrind=*       Run in valgrind (slow!)."
+  echo
+  echo "    --valgrind-options       Additional Valgrind options"
+  echo "                             Some of the options like e.g.:"
+  echo "                             --track-origins=yes add a lot of extra"
+  echo "                             runtime and may trigger the 30 minutes"
+  echo "                             timeout."
+  echo
   echo "Godmode:"
   echo
-  echo "    --all                   Choose all architectures and gcc and clang as compilers"
+  echo "    --all                   Choose all architectures and gcc and clang"
+  echo "                            as compilers but does not run valgrind."
   echo
+  echo "    -h"
   echo "    --help                  This message"
+  echo
+  echo "    -v"
+  echo "    --version               Prints the version. It is just the number"
+  echo "                            of git commits to this file, no deeper"
+  echo "                            meaning attached"
   exit 0
 }
 
@@ -97,6 +120,22 @@ _runtest()
   $_timeout ./test > test_${suffix}.log || _die "running tests" $?
 }
 
+# This is not much more of a C&P of _runtest with a different timeout
+# and the additional valgrind call.
+# TODO: merge
+_runvalgrind()
+{
+  make clean > /dev/null
+  _make "$1" "$2" "test_standalone"
+  local _timeout=""
+  # 30 minutes? Yes. Had it at 20 minutes and the Valgrind run needed over 25 minutes.
+  # A bit too close for comfort.
+  which timeout >/dev/null && _timeout="timeout --foreground 1800"
+  echo -e "\rRun test $1 $2 inside valgrind"
+  $_timeout $VALGRIND_BIN $VALGRIND_OPTS ./test > test_${suffix}.log || _die "running tests" $?
+}
+
+
 _banner()
 {
   echo "uname="$(uname -a)
@@ -121,6 +160,13 @@ CFLAGS=""
 WITH_LOW_MP=""
 TEST_VS_MTEST=""
 MTEST_RAND=""
+# timed with an AMD A8-6600K
+# 25 minutes
+#VALGRIND_OPTS=" --track-origins=yes --leak-check=full --show-leak-kinds=all --error-exitcode=1 "
+# 9 minutes (14 minutes with --test-vs-mtest=333333 --mtest-real-rand)
+VALGRIND_OPTS=" --leak-check=full --show-leak-kinds=all --error-exitcode=1 "
+#VALGRIND_OPTS=""
+VALGRIND_BIN=""
 
 while [ $# -gt 0 ];
 do
@@ -134,6 +180,17 @@ do
     --cflags=*)
       CFLAGS="$CFLAGS ${1#*=}"
     ;;
+    --valgrind-options=*)
+      VALGRIND_OPTS="$VALGRIND_OPTS ${1#*=}"
+    ;;
+    --with-valgrind*)
+      if [[ ${1#*d} != "" ]]
+      then
+        VALGRIND_BIN="${1#*=}"
+      else
+        VALGRIND_BIN="valgrind"
+      fi
+    ;;
     --make-option=*)
       MAKE_OPTIONS="$MAKE_OPTIONS ${1#*=}"
     ;;
@@ -158,6 +215,10 @@ do
     --help | -h)
       _help
     ;;
+    --version | -v)
+      echo $(git rev-list HEAD --count -- testme.sh) || echo "Unknown. Please run in original libtommath git repository."
+      exit 0
+    ;;
     *)
       echo "Ignoring option ${1}"
     ;;
@@ -173,10 +234,16 @@ then
 elif [[ "$COMPILERS" == "" ]]
 then
   _banner gcc
-  _runtest "gcc" ""
+  if [[ "$VALGRIND_BIN" != "" ]]
+  then
+    _runvalgrind "gcc" ""
+  else
+    _runtest "gcc" ""
+  fi
   _exit
 fi
 
+
 archflags=( $ARCHFLAGS )
 compilers=( $COMPILERS )
 
@@ -200,7 +267,7 @@ then
    alive_pid=$!
    _timeout=""
    which timeout >/dev/null && _timeout="timeout --foreground 900"
-   $_TIMEOUT ./mtest/mtest $TEST_VS_MTEST | ./test > test.log
+   $_TIMEOUT./mtest/mtest $TEST_VS_MTEST |  $VALGRIND_BIN $VALGRIND_OPTS  ./test > test.log
    disown $alive_pid
    kill $alive_pid 2>/dev/null
    head -n 5 test.log
@@ -232,12 +299,20 @@ do
       echo "clang -mx32 tests skipped"
       continue
     fi
-
+    if [[ "$VALGRIND_BIN" != "" ]]
-    _runtest "$i $a" "$CFLAGS"
+    then
-    [ "$WITH_LOW_MP" != "1" ] && continue
+      _runvalgrind "$i $a" "$CFLAGS"
-    _runtest "$i $a" "-DMP_8BIT $CFLAGS"
+      [ "$WITH_LOW_MP" != "1" ] && continue
-    _runtest "$i $a" "-DMP_16BIT $CFLAGS"
+      _runvalgrind "$i $a" "-DMP_8BIT $CFLAGS"
-    _runtest "$i $a" "-DMP_32BIT $CFLAGS"
+      _runvalgrind "$i $a" "-DMP_16BIT $CFLAGS"
+      _runvalgrind "$i $a" "-DMP_32BIT $CFLAGS"
+    else
+      _runtest "$i $a" "$CFLAGS"
+      [ "$WITH_LOW_MP" != "1" ] && continue
+      _runtest "$i $a" "-DMP_8BIT $CFLAGS"
+      _runtest "$i $a" "-DMP_16BIT $CFLAGS"
+      _runtest "$i $a" "-DMP_32BIT $CFLAGS"
+    fi
   done
 done