Patch 2/4 of the effort to make TLS access async-signal-safe.

Add a signal-safe malloc replacement.

2013-12-18  Andrew Hunter  <ahh@google.com>

	* sysdeps/generic/ldsodefs.h (__signal_safe_memalign): New prototype.
	(__signal_safe_malloc, __signal_safe_free): Likewise.
	(__signal_safe_realloc, __signal_safe_calloc): Likewise.
	* elf/dl-misc.c (__signal_safe_allocator_header): New struct.
	(__signal_safe_memalign, __signal_safe_malloc): New function.
	(__signal_safe_free, __signal_safe_realloc): Likewise.
	(__signal_safe_calloc): Likewise.
	* elf/dl-tls.c (allocate_dtv, _dl_clear_dtv): Call signal-safe
	functions.
	(_dl_deallocate_tls, _dl_update_slotinfo): Likewise.

ChangeLog

@@ -1,3 +1,16 @@
+2013-12-18  Andrew Hunter  <ahh@google.com>
+
+	* sysdeps/generic/ldsodefs.h (__signal_safe_memalign): New prototype.
+	(__signal_safe_malloc, __signal_safe_free): Likewise.
+	(__signal_safe_realloc, __signal_safe_calloc): Likewise.
+	* elf/dl-misc.c (__signal_safe_allocator_header): New struct.
+	(__signal_safe_memalign, __signal_safe_malloc): New function.
+	(__signal_safe_free, __signal_safe_realloc): Likewise.
+	(__signal_safe_calloc): Likewise.
+	* elf/dl-tls.c (allocate_dtv, _dl_clear_dtv): Call signal-safe
+	functions.
+	(_dl_deallocate_tls, _dl_update_slotinfo): Likewise.
+
 2013-12-18  Andrew Hunter  <ahh@google.com>
 
 	* elf/Versions (ld): Add _dl_clear_dtv.

elf/dl-misc.c

@@ -19,6 +19,7 @@
 #include <assert.h>
 #include <fcntl.h>
 #include <ldsodefs.h>
+#include <libc-symbols.h>
 #include <limits.h>
 #include <link.h>
 #include <stdarg.h>
@@ -364,3 +365,134 @@ _dl_higher_prime_number (unsigned long int n)
 
   return *low;
 }
+
+/* To support accessing TLS variables from signal handlers, we need an
+   async signal safe memory allocator.  These routines are never
+   themselves invoked reentrantly (all calls to them are surrounded by
+   signal masks) but may be invoked concurrently from many threads.
+   The current implementation is not particularly performant nor space
+   efficient, but it will be used rarely (and only in binaries that use
+   dlopen.)  The API matches that of malloc() and friends.  */
+
+struct __signal_safe_allocator_header
+{
+  size_t size;
+  void *start;
+};
+
+void *weak_function
+__signal_safe_memalign (size_t boundary, size_t size)
+{
+  struct __signal_safe_allocator_header *header;
+  if (boundary < sizeof (*header))
+    boundary = sizeof (*header);
+
+  /* Boundary must be a power of two.  */
+  if (boundary & (boundary - 1) == 0)
+    return NULL;
+
+  size_t pg = GLRO (dl_pagesize);
+  size_t padded_size;
+  if (boundary <= pg)
+    {
+      /* We'll get a pointer certainly aligned to boundary, so just
+	 add one more boundary-sized chunk to hold the header.  */
+      padded_size = roundup (size, boundary) + boundary;
+    }
+  else
+    {
+      /* If we want K pages aligned to a J-page boundary, K+J+1 pages
+	 contains at least one such region that isn't directly at the start
+	 (so we can place the header.)	This is wasteful, but you're the one
+	 who wanted 64K-aligned TLS.  */
+      padded_size = roundup (size, pg) + boundary + pg;
+    }
+
+
+  size_t actual_size = roundup (padded_size, pg);
+  void *actual = mmap (NULL, actual_size, PROT_READ | PROT_WRITE,
+		       MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+  if (actual == MAP_FAILED)
+    return NULL;
+
+  if (boundary <= pg)
+    {
+      header = actual + boundary - sizeof (*header);
+    }
+  else
+    {
+      intptr_t actual_pg = ((intptr_t) actual) / pg;
+      intptr_t boundary_pg = boundary / pg;
+      intptr_t start_pg = actual_pg + boundary_pg;
+      start_pg -= start_pg % boundary_pg;
+      if (start_pg > (actual_pg + 1))
+	{
+	  int ret = munmap (actual, (start_pg - actual_pg - 1) * pg);
+	  assert (ret == 0);
+	  actual = (void *) ((start_pg - 1) * pg);
+	}
+      char *start = (void *) (start_pg * pg);
+      header = start - sizeof (*header);
+
+    }
+  header->size = actual_size;
+  header->start = actual;
+  void *ptr = header;
+  ptr += sizeof (*header);
+  if (((intptr_t) ptr) % boundary != 0)
+    _dl_fatal_printf ("__signal_safe_memalign produced incorrect alignment\n");
+  return ptr;
+}
+
+void * weak_function
+__signal_safe_malloc (size_t size)
+{
+  return __signal_safe_memalign (1, size);
+}
+
+void weak_function
+__signal_safe_free (void *ptr)
+{
+  if (ptr == NULL)
+    return;
+
+  struct __signal_safe_allocator_header *header = ((char *) ptr) - sizeof (*header);
+  int ret = munmap (header->start, header->size);
+
+  assert (ret == 0);
+}
+
+void * weak_function
+__signal_safe_realloc (void *ptr, size_t size)
+{
+  if (size == 0)
+    {
+      __signal_safe_free (ptr);
+      return NULL;
+    }
+  if (ptr == NULL)
+    return __signal_safe_malloc (size);
+
+  struct __signal_safe_allocator_header *header = ((char *) ptr) - sizeof (*header);
+  size_t old_size = header->size;
+  if (old_size - sizeof (*header) >= size)
+    return ptr;
+
+  void *new_ptr = __signal_safe_malloc (size);
+  if (new_ptr == NULL)
+    return NULL;
+
+  memcpy (new_ptr, ptr, old_size);
+  __signal_safe_free (ptr);
+
+  return new_ptr;
+}
+
+void * weak_function
+__signal_safe_calloc (size_t nmemb, size_t size)
+{
+  void *ptr = __signal_safe_malloc (nmemb * size);
+  if (ptr == NULL)
+    return NULL;
+  return memset (ptr, 0, nmemb * size);
+}

elf/dl-tls.c

@@ -293,7 +293,7 @@ allocate_dtv (void *result)
      initial set of modules.  This should avoid in most cases expansions
      of the dtv.  */
   dtv_length = GL(dl_tls_max_dtv_idx) + DTV_SURPLUS;
-  dtv = calloc (dtv_length + 2, sizeof (dtv_t));
+  dtv = __signal_safe_calloc (dtv_length + 2, sizeof (dtv_t));
   if (dtv != NULL)
     {
       /* This is the initial length of the dtv.  */
@@ -470,7 +470,7 @@ _dl_clear_dtv (dtv_t *dtv)
   for (size_t cnt = 0; cnt < dtv[-1].counter; ++cnt)
     if (! dtv[1 + cnt].pointer.is_static
 	&& dtv[1 + cnt].pointer.val != TLS_DTV_UNALLOCATED)
-      free (dtv[1 + cnt].pointer.val);
+      __signal_safe_free (dtv[1 + cnt].pointer.val);
   memset (dtv, '\0', (dtv[-1].counter + 1) * sizeof (dtv_t));
 }
 
@@ -491,11 +491,11 @@ _dl_deallocate_tls (void *tcb, bool dealloc_tcb)
   for (size_t cnt = 0; cnt < dtv[-1].counter; ++cnt)
     if (! dtv[1 + cnt].pointer.is_static
 	&& dtv[1 + cnt].pointer.val != TLS_DTV_UNALLOCATED)
-      free (dtv[1 + cnt].pointer.val);
+      __signal_safe_free (dtv[1 + cnt].pointer.val);
 
   /* The array starts with dtv[-1].  */
   if (dtv != GL(dl_initial_dtv))
-    free (dtv - 1);
+    __signal_safe_free (dtv - 1);
 
   if (dealloc_tcb)
     {
@@ -537,8 +537,7 @@ static void *
 allocate_and_init (struct link_map *map)
 {
   void *newp;
-
-  newp = __libc_memalign (map->l_tls_align, map->l_tls_blocksize);
+  newp = __signal_safe_memalign (map->l_tls_align, map->l_tls_blocksize);
   if (newp == NULL)
     oom ();
 
@@ -608,25 +607,27 @@ _dl_update_slotinfo (unsigned long int req_modid)
 	      if (gen <= dtv[0].counter)
 		continue;
 
+	      size_t modid = total + cnt;
+
 	      /* If there is no map this means the entry is empty.  */
 	      struct link_map *map = listp->slotinfo[cnt].map;
 	      if (map == NULL)
 		{
 		  /* If this modid was used at some point the memory
 		     might still be allocated.  */
-		  if (! dtv[total + cnt].pointer.is_static
-		      && dtv[total + cnt].pointer.val != TLS_DTV_UNALLOCATED)
+		  if (dtv[-1].counter >= modid
+		      && !dtv[modid].pointer.is_static
+		      && dtv[modid].pointer.val != TLS_DTV_UNALLOCATED)
 		    {
-		      free (dtv[total + cnt].pointer.val);
-		      dtv[total + cnt].pointer.val = TLS_DTV_UNALLOCATED;
+		      __signal_safe_free (dtv[modid].pointer.val);
+		      dtv[modid].pointer.val = TLS_DTV_UNALLOCATED;
 		    }
 
 		  continue;
 		}
 
+	      assert (modid == map->l_tls_modid);
 	      /* Check whether the current dtv array is large enough.  */
-	      size_t modid = map->l_tls_modid;
-	      assert (total + cnt == modid);
 	      if (dtv[-1].counter < modid)
 		{
 		  /* Reallocate the dtv.  */
@@ -640,17 +641,18 @@ _dl_update_slotinfo (unsigned long int req_modid)
 		    {
 		      /* This is the initial dtv that was allocated
 			 during rtld startup using the dl-minimal.c
-			 malloc instead of the real malloc.  We can't
+			 malloc instead of the real allocator.  We can't
 			 free it, we have to abandon the old storage.  */
 
-		      newp = malloc ((2 + newsize) * sizeof (dtv_t));
+		      newp = __signal_safe_malloc (
+					(2 + newsize) * sizeof (dtv_t));
 		      if (newp == NULL)
 			oom ();
 		      memcpy (newp, &dtv[-1], (2 + oldsize) * sizeof (dtv_t));
 		    }
 		  else
 		    {
-		      newp = realloc (&dtv[-1],
+		      newp = __signal_safe_realloc (&dtv[-1],
 				      (2 + newsize) * sizeof (dtv_t));
 		      if (newp == NULL)
 			oom ();
@@ -680,7 +682,7 @@ _dl_update_slotinfo (unsigned long int req_modid)
 		   deallocate even if it is this dtv entry we are
 		   supposed to load.  The reason is that we call
 		   memalign and not malloc.  */
-		free (dtv[modid].pointer.val);
+		__signal_safe_free (dtv[modid].pointer.val);
 
 	      /* This module is loaded dynamically- We defer memory
 		 allocation.  */

sysdeps/generic/ldsodefs.h

@@ -994,6 +994,12 @@ rtld_hidden_proto (_dl_allocate_tls_init)
 extern void _dl_clear_dtv (dtv_t *dtv) internal_function;
 rtld_hidden_proto (_dl_clear_dtv)
 
+extern void *__signal_safe_memalign (size_t boundary, size_t size);
+extern void *__signal_safe_malloc (size_t size);
+extern void __signal_safe_free (void *ptr);
+extern void *__signal_safe_realloc (void *ptr, size_t size);
+extern void *__signal_safe_calloc (size_t nmemb, size_t size);
+
 /* Deallocate memory allocated with _dl_allocate_tls.  */
 extern void _dl_deallocate_tls (void *tcb, bool dealloc_tcb) internal_function;
 rtld_hidden_proto (_dl_deallocate_tls)