support: Add support for delayed test failure reporting

The new functions support_record_failure records a test failure,
but does not terminate the process.  The macros TEST_VERIFY
and TEST_VERIFY_EXIT check that a condition is true.

ChangeLog

@@ -1,3 +1,23 @@
+2016-12-28  Florian Weimer  <fweimer@redhat.com>
+
+	* support/Makefile (libsupport-routines): Add
+	support_test_verify_impl, support_record_failure, xfork, xwaitpid.
+	(tests): Add tst-support_record_failure.
+	(tests-special): tst-support_record_failure-2.
+	(tst-support_record_failure-2.out): Depend on
+	tst-support_record_failure-2.sh and tst-support_record_failure.
+	* support/check.h (TEST_VERIFY, TEST_VERIFY_EXIT): Define.
+	(support_test_verify_impl, support_record_failure)
+	(support_report_failure, support_report_failure_reset): Declare.
+	* support/support_test_main.c (adjust_exit_status): New function.
+	(support_test_main): Call it to incorporate record test failures.
+	* support/support_record_failure.c: New file.
+	* support/tst-support_record_failure.c: Likewise.
+	* support/tst-support_record_failure-2.sh: Likewise.
+	* support/xunistd.h: Likewise.
+	* support/xfork.c: Likewise.
+	* support/xwaitpid.c: Likewise.
+
 2016-12-27  Steve Ellcey  <sellcey@caviumnetworks.com>
 
 	* scripts/check-c++-types.sh: Add comments.

support/Makefile

@@ -30,11 +30,14 @@ libsupport-routines = \
   ignore_stderr \
   oom_error \
   set_fortify_handler \
+  support_record_failure \
   support_test_main \
+  support_test_verify_impl \
   temp_file \
   write_message \
   xasprintf \
   xcalloc \
+  xfork \
   xmalloc \
   xpthread_barrier_destroy \
   xpthread_barrier_init \
@@ -51,6 +54,7 @@ libsupport-routines = \
   xpthread_spin_lock \
   xpthread_spin_unlock \
   xrealloc \
+  xwaitpid \
 
 libsupport-static-only-routines := $(libsupport-routines)
 # Only build one variant of the library.
@@ -59,6 +63,18 @@ ifeq ($(build-shared),yes)
 libsupport-inhibit-o += .o
 endif
 
-tests = README-testing
+tests = \
+  README-testing \
+  tst-support_record_failure \
+
+tests-special = \
+  $(objpfx)tst-support_record_failure-2.out
+
+$(objpfx)tst-support_record_failure-2.out: tst-support_record_failure-2.sh \
+  $(objpfx)tst-support_record_failure
+	$(SHELL) $< $(common-objpfx) '$(test-program-prefix-before-env)' \
+	  '$(run-program-env)' '$(test-program-prefix-after-env)' \
+	  > $@; \
+	$(evaluate-test)
 
 include ../Rules

support/check.h

@@ -1,4 +1,4 @@
-/* Macros for reporting test results.
+/* Functionality for reporting test results.
    Copyright (C) 2016 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
@@ -35,6 +35,25 @@ __BEGIN_DECLS
 #define FAIL_EXIT1(...) \
   support_exit_failure_impl (1, __FILE__, __LINE__, __VA_ARGS__)
 
+/* Record a test failure (but continue executing) if EXPR evaluates to
+   false.  */
+#define TEST_VERIFY(expr)                                       \
+  ({                                                            \
+    if (expr)                                                   \
+      ;                                                         \
+    else                                                        \
+      support_test_verify_impl (-1, __FILE__, __LINE__, #expr); \
+  })
+
+/* Record a test failure and exit if EXPR evaluates to false.  */
+#define TEST_VERIFY_EXIT(expr)                                  \
+  ({                                                            \
+    if (expr)                                                   \
+      ;                                                         \
+    else                                                        \
+      support_test_verify_impl (1, __FILE__, __LINE__, #expr);  \
+  })
+
 int support_print_failure_impl (const char *file, int line,
                                 const char *format, ...)
   __attribute__ ((nonnull (1), format (printf, 3, 4)));
@@ -42,7 +61,24 @@ void support_exit_failure_impl (int exit_status,
                                 const char *file, int line,
                                 const char *format, ...)
   __attribute__ ((noreturn, nonnull (2), format (printf, 4, 5)));
+void support_test_verify_impl (int status, const char *file, int line,
+                               const char *expr);
 
+/* Record a test failure.  This function returns and does not
+   terminate the process.  The failure counter is stored in a shared
+   memory mapping, so that failures reported in child processes are
+   visible to the parent process and test driver.  This function
+   depends on initialization by an ELF constructor, so it can only be
+   invoked after the test driver has run.  Note that this function
+   does not support reporting failures from a DSO.  */
+void support_record_failure (void);
+
+/* Internal function called by the test driver.  */
+int support_report_failure (int status)
+  __attribute__ ((weak, warn_unused_result));
+
+/* Internal function used to test the failure recording framework.  */
+void support_record_failure_reset (void);
 
 __END_DECLS
 

support/support_record_failure.c

@@ -0,0 +1,106 @@
+/* Global test failure counter.
+   Copyright (C) 2016 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <support/check.h>
+#include <support/support.h>
+#include <support/test-driver.h>
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+/* This structure keeps track of test failures.  The counter is
+   incremented on each failure.  The failed member is set to true if a
+   failure is detected, so that even if the counter wraps around to
+   zero, the failure of a test can be detected.
+
+   The init constructor function below puts *state on a shared
+   annonymous mapping, so that failure reports from subprocesses
+   propagate to the parent process.  */
+struct test_failures
+{
+  unsigned counter;
+  unsigned failed;
+};
+static struct test_failures *state;
+
+static __attribute__ ((constructor)) void
+init (void)
+{
+  void *ptr = mmap (NULL, sizeof (*state), PROT_READ | PROT_WRITE,
+                    MAP_ANONYMOUS | MAP_SHARED, -1, 0);
+  if (ptr == MAP_FAILED)
+    {
+      printf ("error: could not map %zu bytes: %m\n", sizeof (*state));
+      exit (1);
+    }
+  /* Zero-initialization of the struct is sufficient.  */
+  state = ptr;
+}
+
+void
+support_record_failure (void)
+{
+  if (state == NULL)
+    {
+      write_message
+        ("error: support_record_failure called without initialization\n");
+      _exit (1);
+    }
+  /* Relaxed MO is sufficient because we are only interested in the
+     values themselves, in isolation.  */
+  __atomic_store_n (&state->failed, 1, __ATOMIC_RELEASE);
+  __atomic_add_fetch (&state->counter, 1, __ATOMIC_RELEASE);
+}
+
+int
+support_report_failure (int status)
+{
+  if (state == NULL)
+    {
+      write_message
+        ("error: support_report_failure called without initialization\n");
+      return 1;
+    }
+
+  /* Relaxed MO is sufficient because acquire test result reporting
+     assumes that exiting from the main thread happens before the
+     error reporting via support_record_failure, which requires some
+     form of external synchronization.  */
+  bool failed = __atomic_load_n (&state->failed, __ATOMIC_RELAXED);
+  if (failed)
+    printf ("error: %u test failures\n",
+            __atomic_load_n (&state->counter, __ATOMIC_RELAXED));
+
+  if ((status == 0 || status == EXIT_UNSUPPORTED) && failed)
+    /* If we have a recorded failure, it overrides a non-failure
+       report from the test function.  */
+    status = 1;
+  return status;
+}
+
+void
+support_record_failure_reset (void)
+{
+  /* Only used for testing the test framework, with external
+     synchronization, but use release MO for consistency.  */
+  __atomic_store_n (&state->failed, 0, __ATOMIC_RELAXED);
+  __atomic_add_fetch (&state->counter, 0, __ATOMIC_RELAXED);
+}

support/support_test_main.c

@@ -17,6 +17,7 @@
    <http://www.gnu.org/licenses/>.  */
 
 #include <support/test-driver.h>
+#include <support/check.h>
 #include <support/temp_file-internal.h>
 
 #include <assert.h>
@@ -164,6 +165,17 @@ static bool test_main_called;
 
 const char *test_dir = NULL;
 
+
+/* If test failure reporting has been linked in, it may contribute
+   additional test failures.  */
+static int
+adjust_exit_status (int status)
+{
+  if (support_report_failure != NULL)
+    return support_report_failure (status);
+  return status;
+}
+
 int
 support_test_main (int argc, char **argv, const struct test_config *config)
 {
@@ -300,7 +312,7 @@ support_test_main (int argc, char **argv, const struct test_config *config)
 
   /* If we are not expected to fork run the function immediately.  */
   if (direct)
-    return run_test_function (argc, argv, config);
+    return adjust_exit_status (run_test_function (argc, argv, config));
 
   /* Set up the test environment:
      - prevent core dumps
@@ -363,8 +375,8 @@ support_test_main (int argc, char **argv, const struct test_config *config)
       if (config->expected_status == 0)
         {
           if (config->expected_signal == 0)
-            /* Simply exit with the return value of the test.  */
-            return WEXITSTATUS (status);
+            /* Exit with the return value of the test.  */
+            return adjust_exit_status (WEXITSTATUS (status));
           else
             {
               printf ("Expected signal '%s' from child, got none\n",
@@ -382,7 +394,7 @@ support_test_main (int argc, char **argv, const struct test_config *config)
               exit (1);
             }
         }
-      return 0;
+      return adjust_exit_status (0);
     }
   /* Process was killed by timer or other signal.  */
   else
@@ -401,6 +413,6 @@ support_test_main (int argc, char **argv, const struct test_config *config)
           exit (1);
         }
 
-      return 0;
+      return adjust_exit_status (0);
     }
 }

support/support_test_verify_impl.c

@@ -0,0 +1,33 @@
+/* Implementation of the TEST_VERIFY and TEST_VERIFY_EXIT macros.
+   Copyright (C) 2016 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <support/check.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+
+void
+support_test_verify_impl (int status, const char *file, int line,
+                          const char *expr)
+{
+  support_record_failure ();
+  printf ("FAIL %s:%d: not true: %s\n", file, line, expr);
+  if (status >= 0)
+    exit (status);
+
+}

support/tst-support_record_failure-2.sh

@@ -0,0 +1,66 @@
+#!/bin/sh
+# Test failure recording (with and without --direct).
+# Copyright (C) 2016 Free Software Foundation, Inc.
+# This file is part of the GNU C Library.
+
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, see
+# <http://www.gnu.org/licenses/>.  */
+
+set -e
+
+common_objpfx=$1; shift
+test_program_prefix_before_env=$1; shift
+run_program_env=$1; shift
+test_program_prefix_after_env=$1; shift
+
+run_test () {
+    expected_status="$1"
+    expected_output="$2"
+    shift 2
+    args="${common_objpfx}support/tst-support_record_failure $*"
+    echo "running: $args"
+    set +e
+    output="$(${test_program_prefix_before_env} \
+		 ${run_program} ${test_program_prefix_after_env} $args)"
+    status=$?
+    set -e
+    echo "  exit status: $status"
+    if test "$output" != "$expected_output" ; then
+	echo "error: unexpected ouput: $output"
+	exit 1
+    fi
+    if test "$status" -ne "$expected_status" ; then
+	echo "error: exit status $expected_status expected"
+	exit 1
+    fi
+}
+
+different_status () {
+    direct="$1"
+    run_test 1 "error: 1 test failures" $direct --status=0
+    run_test 1 "error: 1 test failures" $direct --status=1
+    run_test 2 "error: 1 test failures" $direct --status=2
+    run_test 1 "error: 1 test failures" $direct --status=77
+    run_test 2 "FAIL tst-support_record_failure.c:108: not true: false
+error: 1 test failures" $direct --test-verify
+}
+
+different_status
+different_status --direct
+
+run_test 1 "FAIL tst-support_record_failure.c:113: not true: false
+error: 1 test failures" --test-verify-exit
+# --direct does not print the summary error message if exit is called.
+run_test 1 "FAIL tst-support_record_failure.c:113: not true: false" \
+	 --direct --test-verify-exit

support/tst-support_record_failure.c

@@ -0,0 +1,150 @@
+/* Test support_record_failure state sharing.
+   Copyright (C) 2016 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <support/check.h>
+#include <support/support.h>
+#include <support/test-driver.h>
+#include <support/xunistd.h>
+
+#include <getopt.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+static int exit_status_with_failure = -1;
+static bool test_verify;
+static bool test_verify_exit;
+enum
+  {
+    OPT_STATUS = 10001,
+    OPT_TEST_VERIFY,
+    OPT_TEST_VERIFY_EXIT,
+  };
+#define CMDLINE_OPTIONS                                                 \
+  { "status", required_argument, NULL, OPT_STATUS },                    \
+  { "test-verify", no_argument, NULL, OPT_TEST_VERIFY },                \
+  { "test-verify-exit", no_argument, NULL, OPT_TEST_VERIFY_EXIT },
+static void
+cmdline_process (int c)
+{
+  switch (c)
+    {
+    case OPT_STATUS:
+      exit_status_with_failure = atoi (optarg);
+      break;
+    case OPT_TEST_VERIFY:
+      test_verify = true;
+      break;
+    case OPT_TEST_VERIFY_EXIT:
+      test_verify_exit = true;
+      break;
+    }
+}
+#define CMDLINE_PROCESS cmdline_process
+
+static void
+check_failure_reporting (int phase, int zero, int unsupported)
+{
+  int status = support_report_failure (0);
+  if (status != zero)
+    {
+      printf ("real-error (phase %d): support_report_failure (0) == %d\n",
+              phase, status);
+      exit (1);
+    }
+  status = support_report_failure (1);
+  if (status != 1)
+    {
+      printf ("real-error (phase %d): support_report_failure (1) == %d\n",
+              phase, status);
+      exit (1);
+    }
+  status = support_report_failure (2);
+  if (status != 2)
+    {
+      printf ("real-error (phase %d): support_report_failure (2) == %d\n",
+              phase, status);
+      exit (1);
+    }
+  status = support_report_failure (EXIT_UNSUPPORTED);
+  if (status != unsupported)
+    {
+      printf ("real-error (phase %d): "
+              "support_report_failure (EXIT_UNSUPPORTED) == %d\n",
+              phase, status);
+      exit (1);
+    }
+}
+
+static int
+do_test (void)
+{
+  if (exit_status_with_failure >= 0)
+    {
+      /* External invocation with requested error status.  Used by
+         tst-support_report_failure-2.sh.  */
+      support_record_failure ();
+      return exit_status_with_failure;
+    }
+  TEST_VERIFY (true);
+  TEST_VERIFY_EXIT (true);
+  if (test_verify)
+    {
+      TEST_VERIFY (false);
+      return 2; /* Expected exit status.  */
+    }
+  if (test_verify_exit)
+    {
+      TEST_VERIFY_EXIT (false);
+      return 3; /* Not reached.  Expected exit status is 1.  */
+    }
+
+  printf ("info: This test tests the test framework.\n"
+          "info: It reports some expected errors on stdout.\n");
+
+  /* Check that the status is passed through unchanged.  */
+  check_failure_reporting (1, 0, EXIT_UNSUPPORTED);
+
+  /* Check state propagation from a subprocess.  */
+  pid_t pid = xfork ();
+  if (pid == 0)
+    {
+      support_record_failure ();
+      _exit (0);
+    }
+  int status;
+  xwaitpid (pid, &status, 0);
+  if (status != 0)
+    {
+      printf ("real-error: incorrect status from subprocess: %d\n", status);
+      return 1;
+    }
+  check_failure_reporting (2, 1, 1);
+
+  /* Also test directly in the parent process.  */
+  support_record_failure_reset ();
+  check_failure_reporting (3, 0, EXIT_UNSUPPORTED);
+  support_record_failure ();
+  check_failure_reporting (4, 1, 1);
+
+  /* We need to mask the failure above.  */
+  support_record_failure_reset ();
+  return 0;
+}
+
+#include <support/test-driver.c>

support/xfork.c

@@ -0,0 +1,34 @@
+/* fork with error checking.
+   Copyright (C) 2016 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <support/xunistd.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+
+pid_t
+xfork (void)
+{
+  pid_t result = fork ();
+  if (result < 0)
+    {
+      printf ("error: fork: %m\n");
+      exit (1);
+    }
+  return result;
+}

support/xunistd.h

@@ -0,0 +1,35 @@
+/* POSIX-specific extra functions.
+   Copyright (C) 2016 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* These wrapper functions use POSIX types and therefore cannot be
+   declared in <support/support.h>.  */
+
+#ifndef SUPPORT_XUNISTD_H
+#define SUPPORT_XUNISTD_H
+
+#include <unistd.h>
+#include <sys/cdefs.h>
+
+__BEGIN_DECLS
+
+pid_t xfork (void);
+pid_t xwaitpid (pid_t, int *status, int flags);
+
+__END_DECLS
+
+#endif /* SUPPORT_XUNISTD_H */

support/xwaitpid.c

@@ -0,0 +1,35 @@
+/* waitpid with error checking.
+   Copyright (C) 2016 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <support/xunistd.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/wait.h>
+
+int
+xwaitpid (int pid, int *status, int flags)
+{
+  pid_t result = waitpid (pid, status, flags);
+  if (result < 0)
+    {
+      printf ("error: waitpid: %m\n");
+      exit (1);
+    }
+  return result;
+}