rt: Set shm_open as a non cancellation point (BZ #18243)

This patch changes shm_open to not act as a cancellation point.
Cancellation is disable at start and reenable in function exit.
It fixes BZ #18243.

Tested on x86_64 and i686.

	[BZ #18243]
	* rt/Makefile (test): Add tst-shm-cancel.
	* rt/tst-shm-cancel.c: New file.
	* sysdeps/posix/shm_open.c: Disable asynchronous cancellation.

ChangeLog

@@ -1,5 +1,10 @@
 2016-09-15  Adhemerval Zanella  <adhemerval.zanella@linaro.org>
 
+	[BZ #18243]
+	* rt/Makefile (test): Add tst-shm-cancel.
+	* rt/tst-shm-cancel.c: New file.
+	* sysdeps/posix/shm_open.c: Disable asynchronous cancellation.
+
 	[BZ #18243]
 	* nptl/pthreadP.h (__pthread_testcancel): Add prototype and hidden_proto.
 	* nptl/pthread_testcancel.c (pthread_cancel): Add internal aliais

rt/Makefile

@@ -53,7 +53,7 @@ tests := tst-shm tst-clock tst-clock_nanosleep tst-timer tst-timer2 \
 	 tst-timer3 tst-timer4 tst-timer5 \
 	 tst-cpuclock1 tst-cpuclock2 \
 	 tst-cputimer1 tst-cputimer2 tst-cputimer3 \
-	 tst-clock2
+	 tst-clock2 tst-shm-cancel
 
 extra-libs := librt
 extra-libs-others := $(extra-libs)

rt/tst-shm-cancel.c

@@ -0,0 +1,130 @@
+/* Test for shm_open cancellation handling: BZ #18243.
+   Copyright (C) 2016 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <pthread.h>
+#include <sys/mman.h>
+#include <semaphore.h>
+#include <stdio.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <errno.h>
+#include <stdlib.h>
+
+static sem_t sem;	/* Use to sync with thread start.  */
+static const char shm_name[] = "/glibc-shm_open-cancel";
+
+static void
+remove_shm (int status, void *arg)
+{
+  shm_unlink (shm_name);
+}
+
+static void *
+tf (void *arg)
+{
+  pthread_setcancelstate (PTHREAD_CANCEL_DISABLE, 0);
+
+  if (sem_wait (&sem) != 0)
+    {
+      printf ("error: sem_wait failed: %m");
+      exit (1);
+    }
+
+  if (pthread_setcancelstate (PTHREAD_CANCEL_ENABLE, 0) != 0)
+    {
+      printf ("error: pthread_setcancelstate failed: %m");
+      exit (1);
+    }
+
+  /* Neither sem_unlink or sem_open should act on thread cancellation.  */
+  shm_unlink (shm_name);
+  on_exit (remove_shm, NULL);
+
+  int fd = shm_open (shm_name, O_CREAT, 0600);
+  if (fd == -1)
+    {
+      int exit_code;
+      if (errno == ENOSYS || errno == EACCES)
+	exit_code = 77;
+      else
+	exit_code = 1;
+      exit (exit_code);
+    }
+
+  if (pthread_setcancelstate (PTHREAD_CANCEL_DISABLE, 0) != 0)
+    {
+      printf ("error: pthread_setcancelstate failed: %m");
+      exit (1);
+    }
+
+  if (close (fd) != 0)
+    {
+      printf ("error: pthread_setcancelstate failed: %m");
+      exit (1);
+    }
+
+  return NULL;
+}
+
+static int
+do_test (void)
+{
+  pthread_t td;
+
+  if (sem_init (&sem, 0, 0))
+    {
+      printf ("error: sem_init failed: %m\n");
+      exit (1);
+    }
+
+  if (pthread_create (&td, NULL, tf, NULL) != 0)
+    {
+      printf ("error: pthread_create failed: %m\n");
+      exit (1);
+    }
+
+  if (pthread_cancel (td) != 0)
+    {
+      printf ("error: pthread_cancel failed: %m\n");
+      exit (1);
+    }
+
+  if (sem_post (&sem) != 0)
+    {
+      printf ("error: sem_post failed: %m\n");
+      exit (1);
+    }
+
+  void *r;
+  if (pthread_join (td, &r) != 0)
+    {
+      printf ("error: pthread_join failed: %m\n");
+      exit (1);
+    }
+
+  if (r == PTHREAD_CANCELED)
+    {
+      puts ("error: pthread_join returned PTHREAD_CANCELED");
+      exit (1);
+    }
+
+  return 0;
+}
+
+#define TEST_FUNCTION do_test ()
+#include <test-skeleton.c>

sysdeps/posix/shm_open.c

@@ -40,6 +40,11 @@ shm_open (const char *name, int oflag, mode_t mode)
 # ifdef O_CLOEXEC
   oflag |= O_CLOEXEC;
 # endif
+
+  /* Disable asynchronous cancellation.  */
+  int state;
+  pthread_setcancelstate (PTHREAD_CANCEL_DISABLE, &state);
+
   int fd = open (shm_name, oflag, mode);
   if (fd == -1 && __glibc_unlikely (errno == EISDIR))
     /* It might be better to fold this error with EINVAL since
@@ -70,6 +75,8 @@ shm_open (const char *name, int oflag, mode_t mode)
     }
 # endif
 
+  pthread_setcancelstate (state, NULL);
+
   return fd;
 }