[testrunner] enable variant sharding

Testrunner runs variants of a test sequentially without taking sharding into
account. A slow test with slow variants slows down the whole test run no matter
the sharding configuration.

This CL implements a test hashing algorithm and variant sharding for test
variants.

R=machenbach@chromium.org
CC=yangguo@chromium.org,sergiyb@chromium.org

Bug: v8:8174
Change-Id: I15f8c547fa2f361fb6c53bf8d5df055d3df38d3e
Reviewed-on: https://chromium-review.googlesource.com/c/1458016
Commit-Queue: Tamer Tas <tmrts@chromium.org>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#59530}

tools/testrunner/standard_runner.py

@@ -295,10 +295,10 @@ class StandardTestRunner(base_runner.BaseTestRunner):
         loader,
         NameFilterProc(args) if args else None,
         StatusFileFilterProc(options.slow_tests, options.pass_fail_tests),
-        self._create_shard_proc(options),
         VariantProc(self._variants),
         StatusFileFilterProc(options.slow_tests, options.pass_fail_tests),
         self._create_predictable_filter(),
+        self._create_shard_proc(options),
         self._create_seed_proc(options),
         sigproc,
       ] + indicators + [

tools/testrunner/testproc/shard.py

@@ -5,10 +5,21 @@
 from . import base
 
 
+# Alphabet size determines the hashing radix. Choosing a prime number prevents
+# clustering of the hashes.
+HASHING_ALPHABET_SIZE = 2 ** 7 -1
+
+def radix_hash(capacity, key):
+  h = 0
+  for character in key:
+    h = (h * HASHING_ALPHABET_SIZE + ord(character)) % capacity
+
+  return h
+
+
 class ShardProc(base.TestProcFilter):
   """Processor distributing tests between shards.
-  It simply passes every n-th test. To be deterministic it has to be placed
-  before all processors that generate tests dynamically.
+  It hashes the unique test identifiers uses the hash to shard tests.
   """
   def __init__(self, myid, shards_count):
     """
@@ -22,9 +33,6 @@ class ShardProc(base.TestProcFilter):
 
     self._myid = myid
     self._shards_count = shards_count
-    self._last = 0
 
   def _filter(self, test):
-    res = self._last != self._myid
-    self._last = (self._last + 1) % self._shards_count
-    return res
+    return self._myid != radix_hash(self._shards_count, test.procid)

tools/testrunner/testproc/shard_unittest.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# Copyright 2019 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import sys
+import tempfile
+import unittest
+
+# Needed because the test runner contains relative imports.
+TOOLS_PATH = os.path.dirname(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__))))
+sys.path.append(TOOLS_PATH)
+
+from testrunner.testproc.shard import radix_hash
+
+
+class TestRadixHashing(unittest.TestCase):
+  def test_hash_character_by_radix(self):
+    self.assertEqual(97, radix_hash(capacity=2**32, key="a"))
+
+  def test_hash_character_by_radix_with_capacity(self):
+    self.assertEqual(6, radix_hash(capacity=7, key="a"))
+
+  def test_hash_string(self):
+    self.assertEqual(6, radix_hash(capacity=7, key="ab"))
+
+  def test_hash_test_id(self):
+    self.assertEqual(
+      5,
+      radix_hash(capacity=7,
+                 key="test262/Map/class-private-method-Variant-0-1"))
+
+  def test_hash_boundaries(self):
+    total_variants = 5
+    cases = []
+    for case in [
+      "test262/Map/class-private-method",
+      "test262/Map/class-public-method",
+      "test262/Map/object-retrieval",
+      "test262/Map/object-deletion",
+      "test262/Map/object-creation",
+      "test262/Map/garbage-collection",
+    ]:
+      for variant_index in range(total_variants):
+        cases.append("%s-Variant-%d" % (case, variant_index))
+
+    for case in cases:
+      self.assertTrue(0 <= radix_hash(capacity=7, key=case) < 7)
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/unittests/run_tests_test.py

@@ -189,17 +189,24 @@ class SystemTest(unittest.TestCase):
             '--variants=default,stress',
             '--shard-count=2',
             '--shard-run=%d' % shard,
-            'sweet/bananas',
+            'sweet/blackberries',
             'sweet/raspberries',
             infra_staging=False,
         )
         # One of the shards gets one variant of each test.
         self.assertIn('2 tests ran', result.stdout, result)
         if shard == 1:
-          self.assertIn('Done running sweet/bananas', result.stdout, result)
+          self.assertIn(
+            'Done running sweet/raspberries default', result.stdout, result)
+          self.assertIn(
+            'Done running sweet/raspberries stress', result.stdout, result)
+          self.assertEqual(0, result.returncode, result)
         else:
-          self.assertIn('Done running sweet/raspberries', result.stdout, result)
-        self.assertEqual(0, result.returncode, result)
+          self.assertIn(
+            'sweet/blackberries default: FAIL', result.stdout, result)
+          self.assertIn(
+            'sweet/blackberries stress: FAIL', result.stdout, result)
+          self.assertEqual(1, result.returncode, result)
 
   @unittest.skip("incompatible with test processors")
   def testSharded(self):