diff --git a/tools/testrunner/local/execution.py b/tools/testrunner/local/execution.py index d18007629c..59aacb0457 100644 --- a/tools/testrunner/local/execution.py +++ b/tools/testrunner/local/execution.py @@ -37,7 +37,12 @@ from . import command from . import perfdata from . import statusfile from . import utils -from pool import Pool +from . pool import Pool +from ..objects import predictable +from ..testproc.execution import ExecutionProc +from ..testproc.loader import LoadProc +from ..testproc.progress import VerboseProgressIndicator, ResultsTracker +from ..testproc.rerun import RerunProc # Base dir of the v8 checkout. @@ -210,6 +215,9 @@ class Runner(object): return not has_unexpected_output def Run(self, jobs): + if self.context.infra_staging: + return self._RunTestProc(jobs) + self.indicator.Starting() self._RunInternal(jobs) self.indicator.Done() @@ -219,6 +227,47 @@ class Runner(object): return 2 return 0 + def _RunTestProc(self, jobs): + print '>>> Running with test processors' + procs = [] + indicators = self.indicator.ToProgressIndicatorProcs() + + # TODO(majeski): Implement all indicators and remove this filter. + indicators = filter(None, indicators) + + loader = LoadProc() + procs.append(loader) + + results = ResultsTracker(count_subtests=False) + procs.append(results) + + procs += indicators + + if self.context.rerun_failures_count: + procs.append(RerunProc( + self.context.rerun_failures_count, + self.context.rerun_failures_max + )) + + execproc = ExecutionProc(jobs, self.context) + procs.append(execproc) + + for i in xrange(0, len(procs) - 1): + procs[i].connect_to(procs[i + 1]) + + loader.load_tests(self.tests) + for indicator in indicators: + indicator.starting() + execproc.start() + for indicator in indicators: + indicator.finished() + + if results.failed: + return 1 + if results.remaining: + return 2 + return 0 + def _RunInternal(self, jobs): pool = Pool(jobs) test_map = {} diff --git a/tools/testrunner/local/progress.py b/tools/testrunner/local/progress.py index 792f5b69fc..cdd2806958 100644 --- a/tools/testrunner/local/progress.py +++ b/tools/testrunner/local/progress.py @@ -34,6 +34,7 @@ import time from . import junit_output from . import statusfile +from ..testproc import progress as progress_proc class ProgressIndicator(object): @@ -66,6 +67,11 @@ class ProgressIndicator(object): 'negative': negative_marker, } + def ToProgressIndicatorProc(self): + print ('Warning: %s is not available as a processor' % + self.__class__.__name__) + return None + class IndicatorNotifier(object): """Holds a list of progress indicators and notifies them all on events.""" @@ -75,6 +81,9 @@ class IndicatorNotifier(object): def Register(self, indicator): self.indicators.append(indicator) + def ToProgressIndicatorProcs(self): + return [i.ToProgressIndicatorProc() for i in self.indicators] + # Forge all generic event-dispatching methods in IndicatorNotifier, which are # part of the ProgressIndicator interface. @@ -144,6 +153,9 @@ class VerboseProgressIndicator(SimpleProgressIndicator): print 'Still working...' sys.stdout.flush() + def ToProgressIndicatorProc(self): + return progress_proc.VerboseProgressIndicator() + class DotsProgressIndicator(SimpleProgressIndicator): @@ -302,6 +314,10 @@ class JsonTestProgressIndicator(ProgressIndicator): self.results = [] self.tests = [] + def ToProgressIndicatorProc(self): + return progress_proc.JsonTestProgressIndicator( + self.json_test_results, self.arch, self.mode, self.random_seed) + def Done(self): complete_results = [] if os.path.exists(self.json_test_results): diff --git a/tools/testrunner/objects/context.py b/tools/testrunner/objects/context.py index da8f8373d8..a3dd56d2dd 100644 --- a/tools/testrunner/objects/context.py +++ b/tools/testrunner/objects/context.py @@ -30,7 +30,7 @@ class Context(): def __init__(self, arch, mode, shell_dir, mode_flags, verbose, timeout, isolates, command_prefix, extra_flags, noi18n, random_seed, no_sorting, rerun_failures_count, rerun_failures_max, no_harness, - use_perf_data, sancov_dir): + use_perf_data, sancov_dir, infra_staging=False): self.arch = arch self.mode = mode self.shell_dir = shell_dir @@ -48,3 +48,4 @@ class Context(): self.no_harness = no_harness self.use_perf_data = use_perf_data self.sancov_dir = sancov_dir + self.infra_staging = infra_staging diff --git a/tools/testrunner/objects/testcase.py b/tools/testrunner/objects/testcase.py index aceb7186e6..5789e7b849 100644 --- a/tools/testrunner/objects/testcase.py +++ b/tools/testrunner/objects/testcase.py @@ -53,11 +53,23 @@ class TestCase(object): self.run = 1 # The nth time this test is executed. self.cmd = None + # Fields used by the test processors. + self.origin = None # Test that this test is subtest of. + self.processor = None # Processor that created this subtest. + self.procid = '%s/%s' % (self.suite.name, self.name) # unique id + self._statusfile_outcomes = None self._expected_outcomes = None # optimization: None == [statusfile.PASS] self._statusfile_flags = None self._prepare_outcomes() + def create_subtest(self, processor, subtest_id): + subtest = copy.copy(self) + subtest.origin = self + subtest.processor = processor + subtest.procid += '.%s' % subtest_id + return subtest + def create_variant(self, variant, flags): """Makes a shallow copy of the object and updates variant, variant_flags and all fields that depend on it, e.g. expected outcomes. @@ -68,6 +80,7 @@ class TestCase(object): else: other.variant_flags = self.variant_flags + flags other.variant = variant + other.procid += '[%s]' % variant other._prepare_outcomes(variant != self.variant) diff --git a/tools/testrunner/standard_runner.py b/tools/testrunner/standard_runner.py index 451d36be5a..e28f1d2045 100755 --- a/tools/testrunner/standard_runner.py +++ b/tools/testrunner/standard_runner.py @@ -380,7 +380,8 @@ class StandardTestRunner(base_runner.BaseTestRunner): options.rerun_failures_max, options.no_harness, use_perf_data=not options.swarming, - sancov_dir=self.sancov_dir) + sancov_dir=self.sancov_dir, + infra_staging=options.infra_staging) # TODO(all): Combine "simulator" and "simulator_run". # TODO(machenbach): In GN we can derive simulator run from diff --git a/tools/testrunner/testproc/__init__.py b/tools/testrunner/testproc/__init__.py new file mode 100644 index 0000000000..4433538556 --- /dev/null +++ b/tools/testrunner/testproc/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2018 the V8 project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. diff --git a/tools/testrunner/testproc/base.py b/tools/testrunner/testproc/base.py new file mode 100644 index 0000000000..3fda3271ca --- /dev/null +++ b/tools/testrunner/testproc/base.py @@ -0,0 +1,72 @@ +# Copyright 2018 the V8 project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +class TestProc(object): + def __init__(self): + self._prev_proc = None + self._next_proc = None + + def connect_to(self, next_proc): + next_proc._prev_proc = self + self._next_proc = next_proc + + def next_test(self, test): + raise NotImplementedError() + + def result_for(self, test, result, is_last): + raise NotImplementedError() + + ### Communication + def _send_test(self, test): + return self._next_proc.next_test(test) + + def _send_result(self, test, result, is_last=True): + return self._prev_proc.result_for(test, result, is_last=is_last) + + + +class TestProcObserver(TestProc): + def next_test(self, test): + self._on_next_test(test) + self._send_test(test) + + def result_for(self, test, result, is_last): + self._on_result_for(test, result, is_last) + self._send_result(test, result, is_last) + + def _on_next_test(self, test): + pass + + def _on_result_for(self, test, result, is_last): + pass + + +class TestProcProducer(TestProc): + def __init__(self, name): + super(TestProcProducer, self).__init__() + self._name = name + + def next_test(self, test): + return self._next_test(test) + + def result_for(self, subtest, result, is_last): + test = self._get_subtest_origin(subtest) + self._result_for(test, subtest, result, is_last) + + ### Implementation + def _next_test(self, test): + raise NotImplementedError() + + def _result_for(self, test, subtest, result, is_last): + raise NotImplementedError() + + ### Managing subtests + def _create_subtest(self, test, subtest_id): + return test.create_subtest(self, '%s-%s' % (self._name, subtest_id)) + + def _get_subtest_origin(self, subtest): + while subtest.processor and subtest.processor is not self: + subtest = subtest.origin + return subtest.origin diff --git a/tools/testrunner/testproc/execution.py b/tools/testrunner/testproc/execution.py new file mode 100644 index 0000000000..fce1294a70 --- /dev/null +++ b/tools/testrunner/testproc/execution.py @@ -0,0 +1,81 @@ +# Copyright 2018 the V8 project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import collections +import traceback + +from . import base +from ..local import pool + + +# Global function for multiprocessing, because pickling a static method doesn't +# work on Windows. +def run_job(job): + return job.run() + + +JobResult = collections.namedtuple('JobResult', ['id', 'result']) + + +class Job(object): + def __init__(self, test_id, cmd, outproc): + self.test_id = test_id + self.cmd = cmd + self.outproc = outproc + + def run(self): + output = self.cmd.execute() + return JobResult(self.test_id, self.outproc.process(output)) + + +class ExecutionProc(base.TestProc): + def __init__(self, jobs, context): + super(ExecutionProc, self).__init__() + self._pool = pool.Pool(jobs) + self._context = context + self._tests = {} + + def connect_to(self, next_proc): + assert False, 'ExecutionProc cannot be connected to anything' + + def start(self): + try: + it = self._pool.imap_unordered( + fn=run_job, + gen=[], + process_context_fn=None, + process_context_args=None, + ) + for pool_result in it: + if pool_result.heartbeat: + continue + + job_result = pool_result.value + test_id, result = job_result + + test = self._tests[test_id] + del self._tests[test_id] + self._send_result(test, result) + except KeyboardInterrupt: + raise + except: + traceback.print_exc() + raise + finally: + self._pool.terminate() + + def next_test(self, test): + test_id = test.procid + self._tests[test_id] = test + + # TODO(majeski): Don't modify test. It's currently used in the progress + # indicator. + test.cmd = test.get_command(self._context) + + # TODO(majeski): Needs factory for outproc as in local/execution.py + outproc = test.output_proc + self._pool.add([Job(test_id, test.cmd, outproc)]) + + def result_for(self, test, result, is_last): + assert False, 'ExecutionProc cannot receive results' diff --git a/tools/testrunner/testproc/loader.py b/tools/testrunner/testproc/loader.py new file mode 100644 index 0000000000..cd4f015c97 --- /dev/null +++ b/tools/testrunner/testproc/loader.py @@ -0,0 +1,20 @@ +# Copyright 2018 the V8 project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from . import base + + +class LoadProc(base.TestProc): + def load_tests(self, tests): + loaded = set() + for test in tests: + if test.procid in loaded: + print 'Warning: %s already obtained' % test.procid + continue + + loaded.add(test.procid) + self._send_test(test) + + def result_for(self, test, result, is_last): + pass diff --git a/tools/testrunner/testproc/progress.py b/tools/testrunner/testproc/progress.py new file mode 100644 index 0000000000..9697aa9215 --- /dev/null +++ b/tools/testrunner/testproc/progress.py @@ -0,0 +1,194 @@ +# Copyright 2018 the V8 project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import json +import os +import sys + +from . import base + + +def print_failure_header(test): + if test.output_proc.negative: + negative_marker = '[negative] ' + else: + negative_marker = '' + print "=== %(label)s %(negative)s===" % { + 'label': test, + 'negative': negative_marker, + } + + +class ResultsTracker(base.TestProcObserver): + def __init__(self, count_subtests): + super(ResultsTracker, self).__init__() + self.failed = 0 + self.remaining = 0 + self.total = 0 + self.count_subtests = count_subtests + + def _on_next_test(self, test): + self.total += 1 + self.remaining += 1 + # TODO(majeski): If count_subtests is set get number of subtests from the + # next proc. + + def _on_result_for(self, test, result, is_last): + if not is_last and not self.count_subtests: + return + + self.remaining -= 1 + if result.has_unexpected_output: + self.failed += 1 + + +class ProgressIndicator(base.TestProcObserver): + def starting(self): + pass + + def finished(self): + pass + + +class SimpleProgressIndicator(ProgressIndicator): + def __init__(self): + super(SimpleProgressIndicator, self).__init__() + + self._failed = [] + self._total = 0 + + def _on_next_test(self, test): + # TODO(majeski): Collect information about subtests, e.g. for each test + # we create multiple variants. + self._total += 1 + + def _on_result_for(self, test, result, is_last): + if result.has_unexpected_output: + self._failed.append((test, result.output)) + + def starting(self): + print 'Running %i tests' % self._total + + def finished(self): + crashed = 0 + print + for test, output in self._failed: + print_failure_header(test) + if output.stderr: + print "--- stderr ---" + print output.stderr.strip() + if output.stdout: + print "--- stdout ---" + print output.stdout.strip() + print "Command: %s" % test.cmd.to_string() + if output.HasCrashed(): + print "exit code: %d" % output.exit_code + print "--- CRASHED ---" + crashed += 1 + if output.HasTimedOut(): + print "--- TIMEOUT ---" + if len(self._failed) == 0: + print "===" + print "=== All tests succeeded" + print "===" + else: + print + print "===" + print "=== %i tests failed" % len(self._failed) + if crashed > 0: + print "=== %i tests CRASHED" % crashed + print "===" + + +class VerboseProgressIndicator(SimpleProgressIndicator): + def _on_result_for(self, test, result, is_last): + super(VerboseProgressIndicator, self)._on_result_for(test, result, is_last) + if result.has_unexpected_output: + if result.output.HasCrashed(): + outcome = 'CRASH' + else: + outcome = 'FAIL' + else: + outcome = 'pass' + print 'Done running %s: %s' % (test, outcome) + sys.stdout.flush() + + +class JsonTestProgressIndicator(ProgressIndicator): + def __init__(self, json_test_results, arch, mode, random_seed): + super(JsonTestProgressIndicator, self).__init__() + self.json_test_results = json_test_results + self.arch = arch + self.mode = mode + self.random_seed = random_seed + self.results = [] + self.tests = [] + + def _on_result_for(self, test, result, is_last): + output = result.output + # Buffer all tests for sorting the durations in the end. + self.tests.append((test, output.duration)) + + # TODO(majeski): Previously we included reruns here. If we still want this + # json progress indicator should be placed just before execution. + if not result.has_unexpected_output: + # Omit tests that run as expected. + return + + self.results.append({ + "name": str(test), + "flags": test.cmd.args, + "command": test.cmd.to_string(relative=True), + "run": -100, # TODO(majeski): do we need this? + "stdout": output.stdout, + "stderr": output.stderr, + "exit_code": output.exit_code, + "result": test.output_proc.get_outcome(output), + "expected": test.expected_outcomes, + "duration": output.duration, + + # TODO(machenbach): This stores only the global random seed from the + # context and not possible overrides when using random-seed stress. + "random_seed": self.random_seed, + "target_name": test.get_shell(), + "variant": test.variant, + }) + + def finished(self): + complete_results = [] + if os.path.exists(self.json_test_results): + with open(self.json_test_results, "r") as f: + # Buildbot might start out with an empty file. + complete_results = json.loads(f.read() or "[]") + + duration_mean = None + if self.tests: + # Get duration mean. + duration_mean = ( + sum(duration for (_, duration) in self.tests) / + float(len(self.tests))) + + # Sort tests by duration. + self.tests.sort(key=lambda (_, duration): duration, reverse=True) + slowest_tests = [ + { + "name": str(test), + "flags": test.cmd.args, + "command": test.cmd.to_string(relative=True), + "duration": duration, + "marked_slow": test.is_slow, + } for (test, duration) in self.tests[:20] + ] + + complete_results.append({ + "arch": self.arch, + "mode": self.mode, + "results": self.results, + "slowest_tests": slowest_tests, + "duration_mean": duration_mean, + "test_total": len(self.tests), + }) + + with open(self.json_test_results, "w") as f: + f.write(json.dumps(complete_results)) diff --git a/tools/testrunner/testproc/rerun.py b/tools/testrunner/testproc/rerun.py new file mode 100644 index 0000000000..d4755661ac --- /dev/null +++ b/tools/testrunner/testproc/rerun.py @@ -0,0 +1,50 @@ +# Copyright 2018 the V8 project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from . import base + + +class RerunProc(base.TestProcProducer): + def __init__(self, rerun_max, rerun_max_total=None): + super(RerunProc, self).__init__('Rerun') + self._rerun = dict() + self._rerun_max = rerun_max + self._rerun_total_left = rerun_max_total + + def _next_test(self, test): + self._init_test(test) + self._send_next_subtest(test) + + def _result_for(self, test, subtest, result, is_last): + # Rerun processor cannot be placed before any processor that produces more + # than one subtest per test. + # TODO(majeski): Introduce constraints and check them during pipeline + # creation to avoid asserts like that. + assert is_last + + if self._needs_rerun(test, result): + self._rerun[test.procid] += 1 + if self._rerun_total_left is not None: + self._rerun_total_left -= 1 + self._send_next_subtest(test) + else: + self._finalize_test(test) + self._send_result(test, result) + + def _init_test(self, test): + self._rerun[test.procid] = 0 + + def _needs_rerun(self, test, result): + # TODO(majeski): Limit reruns count for slow tests. + return ((self._rerun_total_left is None or self._rerun_total_left > 0) and + self._rerun[test.procid] < self._rerun_max and + result.has_unexpected_output) + + def _send_next_subtest(self, test): + run = self._rerun[test.procid] + subtest = self._create_subtest(test, str(run + 1)) + self._send_test(subtest) + + def _finalize_test(self, test): + del self._rerun[test.procid] diff --git a/tools/unittests/run_tests_test.py b/tools/unittests/run_tests_test.py index 3d06b123fb..2e52b8a186 100755 --- a/tools/unittests/run_tests_test.py +++ b/tools/unittests/run_tests_test.py @@ -95,10 +95,12 @@ def capture(): sys.stderr = olderr -def run_tests(basedir, *args): +def run_tests(basedir, *args, **kwargs): """Executes the test runner with captured output.""" with capture() as (stdout, stderr): sys_args = ['--command-prefix', sys.executable] + list(args) + if kwargs.get('infra_staging', False): + sys_args.append('--infra-staging') code = standard_runner.StandardTestRunner( basedir=basedir).execute(sys_args) return Result(stdout.getvalue(), stderr.getvalue(), code) @@ -194,7 +196,10 @@ class SystemTest(unittest.TestCase): self.assertIn('Done running sweet/raspberries', result.stdout, result) self.assertEqual(0, result.returncode, result) - def testFail(self): + def testFailProc(self): + self.testFail(infra_staging=True) + + def testFail(self, infra_staging=False): """Test running only failing tests in two variants.""" with temp_base() as basedir: result = run_tests( @@ -203,12 +208,16 @@ class SystemTest(unittest.TestCase): '--progress=verbose', '--variants=default,stress', 'sweet/strawberries', + infra_staging=infra_staging, ) self.assertIn('Running 2 tests', result.stdout, result) self.assertIn('Done running sweet/strawberries: FAIL', result.stdout, result) self.assertEqual(1, result.returncode, result) - def testFailWithRerunAndJSON(self): + def testFailWithRerunAndJSONProc(self): + self.testFailWithRerunAndJSON(infra_staging=True) + + def testFailWithRerunAndJSON(self, infra_staging=False): """Test re-running a failing test and output to json.""" with temp_base() as basedir: json_path = os.path.join(basedir, 'out.json') @@ -221,11 +230,17 @@ class SystemTest(unittest.TestCase): '--random-seed=123', '--json-test-results', json_path, 'sweet/strawberries', + infra_staging=infra_staging, ) self.assertIn('Running 1 tests', result.stdout, result) self.assertIn('Done running sweet/strawberries: FAIL', result.stdout, result) - # We run one test, which fails and gets re-run twice. - self.assertIn('3 tests failed', result.stdout, result) + if not infra_staging: + # We run one test, which fails and gets re-run twice. + self.assertIn('3 tests failed', result.stdout, result) + else: + # With test processors we don't count reruns as separated failures. + # TODO(majeski): fix it. + self.assertIn('1 tests failed', result.stdout, result) self.assertEqual(0, result.returncode, result) # Check relevant properties of the json output. @@ -246,7 +261,11 @@ class SystemTest(unittest.TestCase): replace_variable_data(data) json_output['duration_mean'] = 1 - with open(os.path.join(TEST_DATA_ROOT, 'expected_test_results1.json')) as f: + suffix = '' + if infra_staging: + suffix = '-proc' + expected_results_name = 'expected_test_results1%s.json' % suffix + with open(os.path.join(TEST_DATA_ROOT, expected_results_name)) as f: expected_test_results = json.load(f) # TODO(majeski): Previously we only reported the variant flags in the @@ -303,12 +322,19 @@ class SystemTest(unittest.TestCase): self.assertIn('Running 0 tests', result.stdout, result) self.assertEqual(0, result.returncode, result) - def testDefault(self): + def testDefaultProc(self): + self.testDefault(infra_staging=True) + + def testDefault(self, infra_staging=False): """Test using default test suites, though no tests are run since they don't exist in a test setting. """ with temp_base() as basedir: - result = run_tests(basedir, '--mode=Release') + result = run_tests( + basedir, + '--mode=Release', + infra_staging=infra_staging, + ) self.assertIn('Warning: no tests were run!', result.stdout, result) self.assertEqual(0, result.returncode, result) @@ -403,7 +429,10 @@ class SystemTest(unittest.TestCase): self.assertIn('(no source available)', result.stdout, result) self.assertEqual(0, result.returncode, result) - def testPredictable(self): + def testPredictableProc(self): + self.testPredictable(infra_staging=True) + + def testPredictable(self, infra_staging=False): """Test running a test in verify-predictable mode. The test will fail because of missing allocation output. We verify that and @@ -417,6 +446,7 @@ class SystemTest(unittest.TestCase): '--progress=verbose', '--variants=default', 'sweet/bananas', + infra_staging=infra_staging, ) self.assertIn('Running 1 tests', result.stdout, result) self.assertIn('Done running sweet/bananas: FAIL', result.stdout, result) diff --git a/tools/unittests/testdata/expected_test_results1-proc.json b/tools/unittests/testdata/expected_test_results1-proc.json new file mode 100644 index 0000000000..c459048bc6 --- /dev/null +++ b/tools/unittests/testdata/expected_test_results1-proc.json @@ -0,0 +1,43 @@ +{ + "arch": "x64", + "duration_mean": 1, + "mode": "release", + "results": [ + { + "command": "/usr/bin/python out/Release/d8_mocked.py --random-seed=123 strawberries --nohard-abort", + "duration": 1, + "exit_code": 1, + "expected": [ + "PASS" + ], + "flags": [ + "--random-seed=123", + "strawberries", + "--nohard-abort" + ], + "name": "sweet/strawberries", + "random_seed": 123, + "result": "FAIL", + "run": -100, + "stderr": "", + "stdout": "--random-seed=123 strawberries --nohard-abort\n", + "target_name": "d8_mocked.py", + "variant": "default" + } + ], + "slowest_tests": [ + { + "command": "/usr/bin/python out/Release/d8_mocked.py --random-seed=123 strawberries --nohard-abort", + "duration": 1, + "flags": [ + "--random-seed=123", + "strawberries", + "--nohard-abort" + ], + "marked_slow": true, + "name": "sweet/strawberries" + } + ], + "test_total": 1 +} +