Fix for test "malloc_usable_size: expected 7 but got 11"

[BZ #17581] The checking chain of unused chunks was terminated by a hash of
    the block pointer, which was sometimes confused with the chunk length byte.
    We now avoid using a length byte equal to the magic byte.

ChangeLog

@@ -1,3 +1,12 @@
+2015-05-19  James Lemke  <jwlemke@codesourcery.com>
+
+	[BZ #17581]
+	* malloc/hooks.c
+	(magicbyte): Convert to a function and avoid returning 0x01.
+	(mem2mem_check): Avoid using a length byte equal to the magic byte.
+	(mem2chunk_check): Fix unsigned comparisons to zero.
+	Hoist defs of sz and magic.
+
 2015-05-19  Richard Henderson  <rth@redhat.com>
 
 	* soft-fp/op-common.h (_FP_FROM_INT): Don't write to R.

NEWS

@@ -11,14 +11,14 @@ Version 2.22
 
   4719, 6792, 13064, 14094, 14841, 14906, 15319, 15467, 15790, 15969, 16159,
   16339, 16351, 16352, 16512, 16560, 16704, 16783, 16850, 17090, 17195,
-  17269, 17523, 17542, 17569, 17588, 17596, 17620, 17621, 17628, 17631,
+  17269, 17523, 17542, 17569, 17581, 17588, 17596, 17620, 17621, 17628,
-  17692, 17711, 17715, 17776, 17779, 17792, 17836, 17912, 17916, 17930,
+  17631, 17692, 17711, 17715, 17776, 17779, 17792, 17836, 17912, 17916,
-  17932, 17944, 17949, 17964, 17965, 17967, 17969, 17978, 17987, 17991,
+  17930, 17932, 17944, 17949, 17964, 17965, 17967, 17969, 17978, 17987,
-  17996, 17998, 17999, 18007, 18019, 18020, 18029, 18030, 18032, 18036,
+  17991, 17996, 17998, 17999, 18007, 18019, 18020, 18029, 18030, 18032,
-  18038, 18039, 18042, 18043, 18046, 18047, 18068, 18080, 18093, 18100,
+  18036, 18038, 18039, 18042, 18043, 18046, 18047, 18068, 18080, 18093,
-  18104, 18110, 18111, 18125, 18128, 18138, 18185, 18196, 18197, 18206,
+  18100, 18104, 18110, 18111, 18125, 18128, 18138, 18185, 18196, 18197,
-  18210, 18211, 18217, 18220, 18221, 18247, 18287, 18319, 18333, 18346,
+  18206, 18210, 18211, 18217, 18220, 18221, 18247, 18287, 18319, 18333,
-  18397, 18409, 18418.
+  18346, 18397, 18409, 18418.
 
 * Cache information can be queried via sysconf() function on s390 e.g. with
   _SC_LEVEL1_ICACHE_SIZE as argument.

malloc/hooks.c

@@ -88,11 +88,22 @@ __malloc_check_init (void)
    overruns.  The goal here is to avoid obscure crashes due to invalid
    usage, unlike in the MALLOC_DEBUG code. */
 
-#define MAGICBYTE(p) ((((size_t) p >> 3) ^ ((size_t) p >> 11)) & 0xFF)
+static unsigned char
+magicbyte (const void *p)
+{
+  unsigned char magic;
 
-/* Visualize the chunk as being partitioned into blocks of 256 bytes from the
+  magic = (((uintptr_t) p >> 3) ^ ((uintptr_t) p >> 11)) & 0xFF;
-   highest address of the chunk, downwards.  The beginning of each block tells
+  /* Do not return 1.  See the comment in mem2mem_check().  */
-   us the size of the previous block, up to the actual size of the requested
+  if (magic == 1)
+    ++magic;
+  return magic;
+}
+
+
+/* Visualize the chunk as being partitioned into blocks of 255 bytes from the
+   highest address of the chunk, downwards.  The end of each block tells
+   us the size of that block, up to the actual size of the requested
    memory.  Our magic byte is right at the end of the requested size, so we
    must reach it with this iteration, otherwise we have witnessed a memory
    corruption.  */
@@ -101,7 +112,7 @@ malloc_check_get_size (mchunkptr p)
 {
   size_t size;
   unsigned char c;
-  unsigned char magic = MAGICBYTE (p);
+  unsigned char magic = magicbyte (p);
 
   assert (using_malloc_checking == 1);
 
@@ -123,32 +134,36 @@ malloc_check_get_size (mchunkptr p)
 }
 
 /* Instrument a chunk with overrun detector byte(s) and convert it
-   into a user pointer with requested size sz. */
+   into a user pointer with requested size req_sz. */
 
 static void *
 internal_function
-mem2mem_check (void *ptr, size_t sz)
+mem2mem_check (void *ptr, size_t req_sz)
 {
   mchunkptr p;
   unsigned char *m_ptr = ptr;
-  size_t i;
+  size_t max_sz, block_sz, i;
+  unsigned char magic;
 
   if (!ptr)
     return ptr;
 
   p = mem2chunk (ptr);
-  for (i = chunksize (p) - (chunk_is_mmapped (p) ? 2 * SIZE_SZ + 1 : SIZE_SZ + 1);
+  magic = magicbyte (p);
-       i > sz;
+  max_sz = chunksize (p) - 2 * SIZE_SZ;
-       i -= 0xFF)
+  if (!chunk_is_mmapped (p))
+    max_sz += SIZE_SZ;
+  for (i = max_sz - 1; i > req_sz; i -= block_sz)
     {
-      if (i - sz < 0x100)
+      block_sz = MIN (i - req_sz, 0xff);
-        {
+      /* Don't allow the magic byte to appear in the chain of length bytes.
-          m_ptr[i] = (unsigned char) (i - sz);
+         For the following to work, magicbyte cannot return 0x01.  */
-          break;
+      if (block_sz == magic)
+        --block_sz;
+
+      m_ptr[i] = block_sz;
     }
-      m_ptr[i] = 0xFF;
+  m_ptr[req_sz] = magic;
-    }
-  m_ptr[sz] = MAGICBYTE (p);
   return (void *) m_ptr;
 }
 
@@ -167,11 +182,12 @@ mem2chunk_check (void *mem, unsigned char **magic_p)
     return NULL;
 
   p = mem2chunk (mem);
+  sz = chunksize (p);
+  magic = magicbyte (p);
   if (!chunk_is_mmapped (p))
     {
       /* Must be a chunk in conventional heap memory. */
       int contig = contiguous (&main_arena);
-      sz = chunksize (p);
       if ((contig &&
            ((char *) p < mp_.sbrk_base ||
             ((char *) p + sz) >= (mp_.sbrk_base + main_arena.system_mem))) ||
@@ -181,10 +197,9 @@ mem2chunk_check (void *mem, unsigned char **magic_p)
                                next_chunk (prev_chunk (p)) != p)))
         return NULL;
 
-      magic = MAGICBYTE (p);
       for (sz += SIZE_SZ - 1; (c = ((unsigned char *) p)[sz]) != magic; sz -= c)
         {
-          if (c <= 0 || sz < (c + 2 * SIZE_SZ))
+          if (c == 0 || sz < (c + 2 * SIZE_SZ))
             return NULL;
         }
     }
@@ -202,13 +217,12 @@ mem2chunk_check (void *mem, unsigned char **magic_p)
            offset < 0x2000) ||
           !chunk_is_mmapped (p) || (p->size & PREV_INUSE) ||
           ((((unsigned long) p - p->prev_size) & page_mask) != 0) ||
-          ((sz = chunksize (p)), ((p->prev_size + sz) & page_mask) != 0))
+          ((p->prev_size + sz) & page_mask) != 0)
         return NULL;
 
-      magic = MAGICBYTE (p);
       for (sz -= 1; (c = ((unsigned char *) p)[sz]) != magic; sz -= c)
         {
-          if (c <= 0 || sz < (c + 2 * SIZE_SZ))
+          if (c == 0 || sz < (c + 2 * SIZE_SZ))
             return NULL;
         }
     }