[test] Run heavy tests sequentially

This adds a new status file indicator "HEAVY" to mark tests with high
resource demands. There will be other tests running in parallel,
but only a limited number of other heavy tests. The limit is
controlled with a new parameter --max-heavy-tests and defaults to 1.

The change also marks a variety of tests as heavy that recently had
flaky timeouts. Heavy also implies slow, hence heavy tests are
executed at the beginning with a higher timeout like other slow tests.

The implementation is encapsulated in the test-processor chain. A
new processor buffers heavy tests in a queue and adds buffered tests
only if other heavy tests have ended their computation.

Bug: v8:5861
Change-Id: I89648ad0030271a3a5af588ecc9c43285b728d6d
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2905767
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Liviu Rau <liviurau@chromium.org>
Cr-Commit-Position: refs/heads/master@{#74712}
This commit is contained in:
Michael Achenbach 2021-05-21 14:08:41 +02:00 committed by V8 LUCI CQ
parent 55cbb2ce3b
commit ee56a9863e
18 changed files with 448 additions and 15 deletions

View File

@ -128,6 +128,26 @@
# pthread_rwlock_t combined with signals is broken on Mac (https://crbug.com/v8/11399).
'signals-and-mutexes/SignalsPlusSharedMutexes': [PASS, ['system == macos', SKIP]],
# Tests that need to run sequentially (e.g. due to memory consumption).
'test-accessors/HandleScopePop': [PASS, HEAVY],
'test-api/FastApiCalls': [PASS, HEAVY],
'test-api/NewStringRangeError': [PASS, HEAVY],
'test-api/Threading8': [PASS, HEAVY],
'test-lockers/LockTwiceAndUnlock': [PASS, HEAVY],
'test-run-machops/RunInt32AddWithOverflowImm': [PASS, HEAVY],
'test-run-machops/RunInt32MulAndInt32AddP': [PASS, HEAVY],
'test-run-machops/RunInt64SubWithOverflowImm': [PASS, HEAVY],
'test-serialize/ContextSerializerContext': [PASS, HEAVY],
'test-serialize/ContextSerializerCustomContext': [PASS, HEAVY],
'test-serialize/SnapshotCompression': [PASS, HEAVY],
'test-serialize/StartupSerializerOnceRunScript': [PASS, HEAVY],
'test-serialize/StartupSerializerTwiceRunScript': [PASS, HEAVY],
'test-strings/StringOOMNewStringFromOneByte': [PASS, HEAVY],
'test-strings/StringOOMNewStringFromUtf8': [PASS, HEAVY],
'test-strings/Traverse': [PASS, HEAVY],
'test-swiss-name-dictionary-csa/DeleteAtBoundaries': [PASS, HEAVY],
'test-swiss-name-dictionary-csa/SameH2': [PASS, HEAVY],
}], # ALWAYS
##############################################################################
@ -212,6 +232,32 @@
'test-log-stack-tracer/PureJSStackTrace': [SKIP],
}], # 'asan == True'
##############################################################################
['asan or tsan', {
# Tests that need to run sequentially (e.g. due to memory consumption).
'regress/regress-crbug-9161': [PASS, HEAVY],
'test-run-wasm/RunWasmInterpreter_I32Binop_Add': [PASS, HEAVY],
'test-run-wasm/RunWasmInterpreter_I32Binop_DivS': [PASS, HEAVY],
'test-run-wasm/RunWasmInterpreter_I32Binop_DivU': [PASS, HEAVY],
'test-run-wasm/RunWasmInterpreter_I32Binop_Eq': [PASS, HEAVY],
'test-run-wasm/RunWasmInterpreter_I32Binop_GeS': [PASS, HEAVY],
'test-run-wasm/RunWasmInterpreter_I32Binop_GeU': [PASS, HEAVY],
'test-run-wasm/RunWasmInterpreter_I32Binop_GtS': [PASS, HEAVY],
'test-run-wasm/RunWasmInterpreter_I32Binop_Ior': [PASS, HEAVY],
'test-run-wasm/RunWasmInterpreter_I32Binop_LeU': [PASS, HEAVY],
'test-run-wasm/RunWasmInterpreter_I32Binop_LtU': [PASS, HEAVY],
'test-run-wasm/RunWasmLiftoff_I32Binop_Add': [PASS, HEAVY],
'test-run-wasm/RunWasmLiftoff_I32Binop_DivS': [PASS, HEAVY],
'test-run-wasm/RunWasmLiftoff_I32Binop_DivU': [PASS, HEAVY],
'test-run-wasm/RunWasmLiftoff_I32Binop_Eq': [PASS, HEAVY],
'test-run-wasm/RunWasmLiftoff_I32Binop_GeS': [PASS, HEAVY],
'test-run-wasm/RunWasmLiftoff_I32Binop_GeU': [PASS, HEAVY],
'test-run-wasm/RunWasmLiftoff_I32Binop_GtS': [PASS, HEAVY],
'test-run-wasm/RunWasmLiftoff_I32Binop_Ior': [PASS, HEAVY],
'test-run-wasm/RunWasmLiftoff_I32Binop_LeU': [PASS, HEAVY],
'test-run-wasm/RunWasmLiftoff_I32Binop_LtU': [PASS, HEAVY],
}], # 'asan or tsan'
##############################################################################
['msan == True', {
# ICU upstream issues.

View File

@ -22,16 +22,16 @@
# CRASH is also a reasonable outcome).
'debug/es6/debug-scope-default-param-with-eval': [FAIL, CRASH],
# Slow tests
'debug/debug-scopes': [PASS, SLOW],
'debug/debug-stepout-scope-part*': [PASS, SLOW],
'debug/ignition/debug-step-prefix-bytecodes': [PASS, SLOW, ['mode == debug', SKIP]],
# Too slow in debug mode and on slow platforms.
'regress/regress-2318': [PASS, SLOW, ['mode == debug or (arch != ia32 and arch != x64) or asan == True or msan == True', SKIP]],
'regress/regress-2318': [PASS, HEAVY, ['mode == debug or (arch != ia32 and arch != x64) or asan == True or msan == True', SKIP]],
# forcing weak callback in asan build change break order
'debug/debug-stepin-builtin-callback': [['asan == True or msan == True', SKIP]],
# Tests that need to run sequentially (e.g. due to memory consumption).
'debug/debug-scopes': [PASS, HEAVY],
'debug/debug-stepout-scope-part*': [PASS, HEAVY],
'debug/ignition/debug-step-prefix-bytecodes': [PASS, HEAVY, ['mode == debug', SKIP]],
}], # ALWAYS
##############################################################################

View File

@ -27,6 +27,10 @@
# https://crbug.com/v8/11338
'runtime-call-stats/enable-disable': [SKIP],
# Tests that need to run sequentially (e.g. due to memory consumption).
'runtime/console-messages-limits': [PASS, HEAVY],
'runtime/regression-732717': [PASS, HEAVY],
}], # ALWAYS
##############################################################################

View File

@ -125,10 +125,8 @@
'es6/promise-all-overflow-2': [PASS, SLOW, ['arch != x64', SKIP]],
'es6/typedarray-construct-offset-not-smi': [PASS, SLOW],
'harmony/promise-any-overflow-2': [PASS, SLOW, ['arch != x64', SKIP]],
'harmony/sharedarraybuffer-worker-gc-stress': [PASS, SLOW],
'harmony/futex': [PASS, SLOW],
'harmony/regexp-property-script-extensions': [PASS, SLOW],
'ignition/regress-672027': [PASS, SLOW],
'large-object-literal-slow-elements': [PASS, SLOW],
'math-floor-of-div': [PASS, SLOW],
'md5': [PASS, SLOW],
@ -136,7 +134,6 @@
'regress/regress-1122': [PASS, SLOW],
'regress/regress-605470': [PASS, SLOW],
'regress/regress-655573': [PASS, SLOW],
'regress/regress-1034322': [PASS, SLOW, NO_VARIANTS, ['mode != release', SKIP]],
'regress/regress-1200351': [PASS, SLOW],
'regress/regress-crbug-808192': [PASS, SLOW, NO_VARIANTS, ['arch not in (ia32, x64)', SKIP]],
'regress/regress-crbug-918301': [PASS, SLOW, NO_VARIANTS, ['mode != release or dcheck_always_on', SKIP], ['(arch == arm or arch == arm64) and simulator_run', SKIP], ['tsan', SKIP]],
@ -169,7 +166,7 @@
# OOM with too many isolates/memory objects (https://crbug.com/1010272)
# Predictable tests fail due to race between postMessage and GrowMemory
'regress/wasm/regress-1010272': [PASS, NO_VARIANTS, ['system == android', SKIP], ['predictable', SKIP]],
'regress/wasm/regress-1010272': [PASS, HEAVY, NO_VARIANTS, ['system == android', SKIP], ['predictable', SKIP]],
# Only makes sense in the no_i18n variant.
'es6/unicode-regexp-ignore-case-noi18n':
@ -177,6 +174,28 @@
# Needs to be adapted after changes to Function constructor. chromium:1065094
'cross-realm-filtering': [SKIP],
# Tests that need to run sequentially (e.g. due to memory consumption).
'compiler/array-subclass': [PASS, HEAVY],
'compiler/regress-crbug-11564': [PASS, HEAVY],
'd8/d8-worker-shutdown*': [PASS, HEAVY],
'es6/large-classes-*': [PASS, HEAVY],
'harmony/sharedarraybuffer-stress': [PASS, HEAVY],
'harmony/sharedarraybuffer-worker-gc-stress': [PASS, HEAVY],
'ignition/regress-672027': [PASS, HEAVY],
'json2': [PASS, HEAVY],
'regress/regress-500980': [PASS, HEAVY],
'regress/regress-599414-array-concat-fast-path': [PASS, HEAVY],
'regress/regress-678917': [PASS, HEAVY],
'regress/regress-752764': [PASS, HEAVY],
'regress/regress-779407': [PASS, HEAVY],
'regress/regress-852258': [PASS, HEAVY],
'regress/regress-862433': [PASS, HEAVY],
'regress/regress-1034322': [PASS, HEAVY, NO_VARIANTS, ['mode != release', SKIP]],
'regress/regress-crbug-119926': [PASS, HEAVY],
'regress/regress-crbug-941743': [PASS, HEAVY],
'regress/regress-crbug-1191886': [PASS, HEAVY],
'wasm/externref-globals': [PASS, HEAVY],
}], # ALWAYS
##############################################################################
@ -547,6 +566,9 @@
# https://bugs.chromium.org/p/v8/issues/detail?id=7102
# Flaky due to huge string allocation.
'regress/regress-748069': [SKIP],
# Tests that need to run sequentially (e.g. due to memory consumption).
'wasm/asm-wasm': [PASS, HEAVY],
}], # 'asan == True'
##############################################################################

View File

@ -741,6 +741,15 @@
'built-ins/ArrayBuffer/length-is-too-large-throws': [SKIP],
'built-ins/SharedArrayBuffer/allocation-limit': [SKIP],
'built-ins/SharedArrayBuffer/length-is-too-large-throws': [SKIP],
# Tests that need to run sequentially (e.g. due to memory consumption).
'annexB/built-ins/RegExp/RegExp-leading-escape-BMP': [PASS, HEAVY],
'annexB/built-ins/RegExp/RegExp-trailing-escape-BMP': [PASS, HEAVY],
'built-ins/decodeURI/*': [PASS, HEAVY],
'built-ins/decodeURIComponent/*': [PASS, HEAVY],
'built-ins/RegExp/property-escapes/generated/*': [PASS, HEAVY],
'language/comments/S7.4_A5': [PASS, HEAVY],
'language/comments/S7.4_A6': [PASS, HEAVY],
}], # asan == True or msan == True or tsan == True
['system == android', {

View File

@ -67,6 +67,10 @@
# https://crbug.com/v8/9380
# The test is broken and needs to be fixed to use separate isolates.
'BackingStoreTest.RacyGrowWasmMemoryInPlace': [SKIP],
# Tests that need to run sequentially (e.g. due to memory consumption).
'MachineOperatorReducerTest.Word32EqualWithShiftedMaskedValueAndConstant': [PASS, HEAVY],
'SequentialUnmapperTest.UnmapOnTeardown': [PASS, HEAVY],
}], # tsan == True
##############################################################################

View File

@ -35,6 +35,12 @@
'proposals/memory64/data': [FAIL],
'proposals/memory64/elem': [FAIL],
'proposals/memory64/imports': [FAIL],
# Tests that need to run sequentially (e.g. due to memory consumption).
'proposals/simd/simd_f32x4*': [PASS, HEAVY],
'proposals/simd/simd_f64x2*': [PASS, HEAVY],
'f32*': [PASS, HEAVY],
'f64*': [PASS, HEAVY],
}], # ALWAYS
['arch == arm and not simulator_run', {

View File

@ -46,6 +46,7 @@ FAIL_OK = "FAIL_OK"
FAIL_SLOPPY = "FAIL_SLOPPY"
# Modifiers
HEAVY = "HEAVY"
SKIP = "SKIP"
SLOW = "SLOW"
NO_VARIANTS = "NO_VARIANTS"
@ -54,8 +55,8 @@ FAIL_PHASE_ONLY = "FAIL_PHASE_ONLY"
ALWAYS = "ALWAYS"
KEYWORDS = {}
for key in [SKIP, FAIL, PASS, CRASH, SLOW, FAIL_OK, NO_VARIANTS, FAIL_SLOPPY,
ALWAYS, FAIL_PHASE_ONLY]:
for key in [SKIP, FAIL, PASS, CRASH, HEAVY, SLOW, FAIL_OK, NO_VARIANTS,
FAIL_SLOPPY, ALWAYS, FAIL_PHASE_ONLY]:
KEYWORDS[key] = key
# Support arches, modes to be written as keywords instead of strings.

View File

@ -214,9 +214,13 @@ class TestCase(object):
return (statusfile.SKIP in self._statusfile_outcomes and
not self.suite.test_config.run_skipped)
@property
def is_heavy(self):
return statusfile.HEAVY in self._statusfile_outcomes
@property
def is_slow(self):
return statusfile.SLOW in self._statusfile_outcomes
return self.is_heavy or statusfile.SLOW in self._statusfile_outcomes
@property
def is_fail_ok(self):

View File

@ -25,6 +25,7 @@ from testrunner.testproc.execution import ExecutionProc
from testrunner.testproc.filter import StatusFileFilterProc, NameFilterProc
from testrunner.testproc.loader import LoadProc
from testrunner.testproc.seed import SeedProc
from testrunner.testproc.sequence import SequenceProc
from testrunner.testproc.variant import VariantProc
@ -122,6 +123,8 @@ class StandardTestRunner(base_runner.BaseTestRunner):
'generation.')
# Extra features.
parser.add_option('--max-heavy-tests', default=1, type='int',
help='Maximum number of heavy tests run in parallel')
parser.add_option('--time', help='Print timing information after running',
default=False, action='store_true')
@ -306,6 +309,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
self._create_predictable_filter(),
self._create_shard_proc(options),
self._create_seed_proc(options),
SequenceProc(options.max_heavy_tests),
sigproc,
] + indicators + [
results,

View File

@ -0,0 +1,59 @@
# Copyright 2021 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from collections import deque
from . import base
class SequenceProc(base.TestProc):
"""Processor ensuring heavy tests are sent sequentially into the execution
pipeline.
The class keeps track of the number of tests in the pipeline marked heavy
and permits only a configurable amount. An excess amount is queued and sent
as soon as other heavy tests return.
"""
def __init__(self, max_heavy):
"""Initialize the processor.
Args:
max_heavy: The maximum number of heavy tests that will be sent further
down the pipeline simultaneously.
"""
super(SequenceProc, self).__init__()
assert max_heavy > 0
self.max_heavy = max_heavy
self.n_heavy = 0
self.buffer = deque()
def next_test(self, test):
if test.is_heavy:
if self.n_heavy < self.max_heavy:
# Enough space to send more heavy tests. Check if the test is not
# filtered otherwise.
used = self._send_test(test)
if used:
self.n_heavy += 1
return used
else:
# Too many tests in the pipeline. Buffer the test and indicate that
# this test didn't end up in the execution queue (i.e. test loader
# will try to send more tests).
self.buffer.append(test)
return False
else:
return self._send_test(test)
def result_for(self, test, result):
if test.is_heavy:
# A heavy test finished computing. Try to send one from the buffer.
self.n_heavy -= 1
while self.buffer:
next_test = self.buffer.popleft()
if self._send_test(next_test):
self.n_heavy += 1
break
self._send_result(test, result)

View File

@ -0,0 +1,162 @@
#!/usr/bin/env python
# Copyright 2021 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
Test integrating the sequence processor into a simple test pipeline.
"""
import os
import sys
import unittest
# Needed because the test runner contains relative imports.
TOOLS_PATH = os.path.dirname(os.path.dirname(os.path.dirname(
os.path.abspath(__file__))))
sys.path.append(TOOLS_PATH)
from testrunner.testproc import base
from testrunner.testproc.loader import LoadProc
from testrunner.testproc.sequence import SequenceProc
class FakeExecutionProc(base.TestProc):
"""Simulates the pipeline sink consuming and running the tests.
Test execution is simulated for each test by calling run().
"""
def __init__(self):
super(FakeExecutionProc, self).__init__()
self.tests = []
def next_test(self, test):
self.tests.append(test)
return True
def run(self):
test = self.tests.pop()
self._send_result(test, test.n)
class FakeResultObserver(base.TestProcObserver):
"""Observer to track all results sent back through the pipeline."""
def __init__(self):
super(FakeResultObserver, self).__init__()
self.tests = set([])
def _on_result_for(self, test, result):
self.tests.add(test.n)
class FakeTest(object):
"""Simple test representation to differentiate light/heavy tests."""
def __init__(self, n, is_heavy):
self.n = n
self.is_heavy = is_heavy
self.keep_output = False
class TestSequenceProc(unittest.TestCase):
def _test(self, tests, batch_size, max_heavy):
# Set up a simple processing pipeline:
# Loader -> observe results -> sequencer -> execution.
loader = LoadProc(iter(tests))
results = FakeResultObserver()
sequence_proc = SequenceProc(max_heavy)
execution = FakeExecutionProc()
loader.connect_to(results)
results.connect_to(sequence_proc)
sequence_proc.connect_to(execution)
# Fill the execution queue (with the number of tests potentially
# executed in parallel).
loader.load_initial_tests(batch_size)
# Simulate the execution test by test.
while execution.tests:
# Assert the invariant of maximum heavy tests executed simultaneously.
self.assertLessEqual(
sum(int(test.is_heavy) for test in execution.tests), max_heavy)
# As in the real pipeline, running a test and returning its result
# will add another test into the pipeline.
execution.run()
# Ensure that all tests are processed and deliver results.
self.assertEqual(set(test.n for test in tests), results.tests)
def test_wrong_usage(self):
self.assertRaises(lambda: SequenceProc(0))
def test_no_tests(self):
self._test([], 1, 1)
def test_large_batch_light(self):
self._test([
FakeTest(0, False),
FakeTest(1, False),
FakeTest(2, False),
], 4, 1)
def test_small_batch_light(self):
self._test([
FakeTest(0, False),
FakeTest(1, False),
FakeTest(2, False),
], 2, 1)
def test_large_batch_heavy(self):
self._test([
FakeTest(0, True),
FakeTest(1, True),
FakeTest(2, True),
], 4, 1)
def test_small_batch_heavy(self):
self._test([
FakeTest(0, True),
FakeTest(1, True),
FakeTest(2, True),
], 2, 1)
def test_large_batch_mixed(self):
self._test([
FakeTest(0, True),
FakeTest(1, False),
FakeTest(2, True),
FakeTest(3, False),
], 4, 1)
def test_small_batch_mixed(self):
self._test([
FakeTest(0, True),
FakeTest(1, False),
FakeTest(2, True),
FakeTest(3, False),
], 2, 1)
def test_large_batch_more_heavy(self):
self._test([
FakeTest(0, True),
FakeTest(1, True),
FakeTest(2, True),
FakeTest(3, False),
FakeTest(4, True),
FakeTest(5, True),
FakeTest(6, False),
], 4, 2)
def test_small_batch_more_heavy(self):
self._test([
FakeTest(0, True),
FakeTest(1, True),
FakeTest(2, True),
FakeTest(3, False),
FakeTest(4, True),
FakeTest(5, True),
FakeTest(6, False),
], 2, 2)
if __name__ == '__main__':
unittest.main()

View File

@ -42,8 +42,11 @@ class VariantProc(base.TestProcProducer):
return self._try_send_new_subtest(test, gen)
def _result_for(self, test, subtest, result):
gen = self._next_variant[test.procid]
if not self._try_send_new_subtest(test, gen):
# The generator might have been removed after cycling through all subtests
# below. If some of the subtests are heavy, they get buffered and return
# their results later.
gen = self._next_variant.get(test.procid)
if not gen or not self._try_send_new_subtest(test, gen):
self._send_result(test, None)
def _try_send_new_subtest(self, test, variants_gen):

View File

@ -183,6 +183,19 @@ class SystemTest(unittest.TestCase):
# self.assertIn('sweet/bananas', result.stderr, result)
self.assertEqual(0, result.returncode, result)
def testPassHeavy(self):
"""Test running with some tests marked heavy."""
with temp_base(baseroot='testroot3') as basedir:
result = run_tests(
basedir,
'--progress=verbose',
'--variants=nooptimization',
'-j2',
'sweet',
)
self.assertIn('7 tests ran', result.stdout, result)
self.assertEqual(0, result.returncode, result)
def testShardedProc(self):
with temp_base() as basedir:
for shard in [1, 2]:

View File

@ -0,0 +1,16 @@
# Copyright 2021 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
Dummy d8 replacement. Just passes all test.
"""
# for py2/py3 compatibility
from __future__ import print_function
import sys
args = ' '.join(sys.argv[1:])
print(args)
sys.exit(0)

View File

@ -0,0 +1,15 @@
# Copyright 2021 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
[
[ALWAYS, {
'raspberries': [PASS, HEAVY],
'strawberries': [PASS, HEAVY],
'blackberries': [PASS, HEAVY],
}],
['variant == nooptimization', {
'cherries': [PASS, HEAVY],
}],
]

View File

@ -0,0 +1,36 @@
# Copyright 2021 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
Dummy test suite extension with some fruity tests.
"""
from testrunner.local import testsuite
from testrunner.objects import testcase
class TestLoader(testsuite.TestLoader):
def _list_test_filenames(self):
return [
'bananas', 'apples', 'cherries', 'mangoes', 'strawberries',
'blackberries', 'raspberries',
]
class TestSuite(testsuite.TestSuite):
def _test_loader_class(self):
return TestLoader
def _test_class(self):
return TestCase
class TestCase(testcase.D8TestCase):
def get_shell(self):
return 'd8_mocked.py'
def _get_files_params(self):
return [self.name]
def GetSuite(*args, **kwargs):
return TestSuite(*args, **kwargs)

View File

@ -0,0 +1,29 @@
{
"current_cpu": "x64",
"dcheck_always_on": false,
"is_android": false,
"is_asan": false,
"is_cfi": false,
"is_clang": true,
"is_component_build": false,
"is_debug": false,
"is_full_debug": false,
"is_gcov_coverage": false,
"is_ubsan_vptr": false,
"is_msan": false,
"is_tsan": false,
"target_cpu": "x64",
"v8_current_cpu": "x64",
"v8_enable_i18n_support": true,
"v8_enable_verify_predictable": false,
"v8_target_cpu": "x64",
"v8_enable_concurrent_marking": true,
"v8_enable_verify_csa": false,
"v8_enable_lite_mode": false,
"v8_enable_pointer_compression": true,
"v8_enable_pointer_compression_shared_cage": true,
"v8_control_flow_integrity": false,
"v8_enable_single_generation": false,
"v8_enable_third_party_heap": false,
"v8_enable_webassembly": true
}