Check for integer overflow in cache size computation in strcoll

strcoll is implemented using a cache for indices and weights of
collation sequences in the strings so that subsequent passes do not
have to search through collation data again.  For very large string
inputs, the cache size computation could overflow.  In such a case,
use the fallback function that does not cache indices and weights of
collation sequences.

Fixes CVE-2012-4412.

ChangeLog

@@ -1,5 +1,11 @@
 2013-09-23  Siddhesh Poyarekar  <siddhesh@redhat.com>
 
+	[BZ #14547]
+	* string/tst-strcoll-overflow.c: New test case.
+	* string/Makefile (xtests): Add tst-strcoll-overflow.
+	* string/strcoll_l.c (STRCOLL): Skip allocating memory for
+	cache if string sizes may cause integer overflow.
+
 	[BZ #14547]
 	* string/strcoll_l.c (coll_seq): New members rule, idx,
 	save_idx and back_us.

NEWS

@@ -14,6 +14,12 @@ Version 2.19
   15867, 15886, 15887, 15890, 15892, 15893, 15895, 15897, 15905, 15909,
   15919, 15921, 15923, 15939, 15963, 15966.
 
+* CVE-2012-4412 The strcoll implementation caches indices and rules for
+  large collation sequences to optimize multiple passes.  This cache
+  computation may overflow for large collation sequences and may cause a
+  stack or buffer overflow.  This is now fixed to use a slower algorithm
+  which does not use a cache if there is an integer overflow.
+
 * CVE-2012-4424 The strcoll implementation uses malloc to cache indices and
   rules for large collation sequences to optimize multiple passes and falls
   back to alloca if malloc fails, resulting in a possible stack overflow.

string/Makefile

@@ -57,6 +57,8 @@ tests		:= tester inl-tester noinl-tester testcopy test-ffs	\
 tests-ifunc := $(strop-tests:%=test-%-ifunc)
 tests += $(tests-ifunc)
 
+xtests = tst-strcoll-overflow
+
 include ../Rules
 
 tester-ENV = LANGUAGE=C

string/strcoll_l.c

@@ -524,7 +524,15 @@ STRCOLL (const STRING_TYPE *s1, const STRING_TYPE *s2, __locale_t l)
   memset (&seq1, 0, sizeof (seq1));
   seq2 = seq1;
 
-  if (! __libc_use_alloca ((s1len + s2len) * (sizeof (int32_t) + 1)))
+  size_t size_max = SIZE_MAX / (sizeof (int32_t) + 1);
+
+  if (MIN (s1len, s2len) > size_max
+      || MAX (s1len, s2len) > size_max - MIN (s1len, s2len))
+    {
+      /* If the strings are long enough to cause overflow in the size request,
+         then skip the allocation and proceed with the non-cached routines.  */
+    }
+  else if (! __libc_use_alloca ((s1len + s2len) * (sizeof (int32_t) + 1)))
     {
       seq1.idxarr = (int32_t *) malloc ((s1len + s2len) * (sizeof (int32_t) + 1));
 

string/tst-strcoll-overflow.c

@@ -0,0 +1,61 @@
+/* Copyright (C) 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <locale.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* Verify that strcoll does not crash for large strings for which it cannot
+   cache weight lookup results.  The size is large enough to cause integer
+   overflows on 32-bit as well as buffer overflows on 64-bit.  The test should
+   work reasonably reliably when overcommit is disabled, but it obviously
+   depends on how much memory the system has.  There's a limitation to this
+   test in that it does not run to completion.  Actually collating such a
+   large string can take days and we can't have xcheck running that long.  For
+   that reason, we run the test for about 5 minutes and then assume that
+   everything is fine if there are no crashes.  */
+#define SIZE 0x40000000ul
+
+int
+do_test (void)
+{
+  if (setlocale (LC_COLLATE, "en_GB.UTF-8") == NULL)
+    {
+      puts ("setlocale failed, cannot test for overflow");
+      return 0;
+    }
+
+  char *p = malloc (SIZE);
+
+  if (p == NULL)
+    {
+      puts ("could not allocate memory");
+      return 1;
+    }
+
+  memset (p, 'x', SIZE - 1);
+  p[SIZE - 1] = 0;
+  printf ("%d\n", strcoll (p, p));
+  return 0;
+}
+
+#define TIMEOUT 300
+#define EXPECTED_SIGNAL SIGALRM
+#define TEST_FUNCTION do_test ()
+#include "../test-skeleton.c"