AuroraMiddleware/v8
1
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

[test] Remove output from testcase.

Browse Source 
Bug: v8:6917
Cq-Include-Trybots: master.tryserver.v8:v8_linux_noi18n_rel_ng
Change-Id: I5ca840db75766413659cb96fd28b922712040cec
Reviewed-on: https://chromium-review.googlesource.com/823843
Commit-Queue: Michał Majewski <majeski@google.com>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Sergiy Byelozyorov <sergiyb@chromium.org>
Cr-Commit-Position: refs/heads/master@{#50078}
This commit is contained in:
Michal Majewski 2017-12-13 14:53:52 +01:00 committed by Commit Bot
parent 5760d76ef7
commit af192c8905
12 changed files with 75 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
test/inspector/testcfg.py
View File

@ -47,7 +47,7 @@ class TestSuite(testsuite.TestSuite):
string == "Warning: unknown flag --enable-slow-asserts." or
string == "Try --help for options")
def IsFailureOutput(self, test):
def IsFailureOutput(self, test, output):
file_name = os.path.join(self.root, test.path) + EXPECTED_SUFFIX
with file(file_name, "r") as expected:
expected_lines = expected.readlines()
@ -66,7 +66,7 @@ class TestSuite(testsuite.TestSuite):
def ActBlockIterator():
"""Iterates over blocks of actual output lines."""
lines = test.output.stdout.splitlines()
lines = output.stdout.splitlines()
start_index = 0
found_eqeq = False
for index, line in enumerate(lines):

5
test/message/testcfg.py
View File

@ -81,11 +81,10 @@ class TestSuite(testsuite.TestSuite):
path = head
return False
def IsFailureOutput(self, test):
output = test.output
def IsFailureOutput(self, test, output):
testpath = test.path
expected_fail = self._GetExpectedFail(test)
fail = test.output.exit_code != 0
fail = output.exit_code != 0
if expected_fail != fail:
return True
expected_path = os.path.join(self.root, testpath + ".out")

3
test/mkgrokdump/testcfg.py
View File

@ -19,8 +19,7 @@ class TestSuite(testsuite.TestSuite):
def _test_class(self):
return TestCase
def IsFailureOutput(self, test):
output = test.output
def IsFailureOutput(self, test, output):
v8_path = os.path.dirname(os.path.dirname(os.path.abspath(self.root)))
expected_path = os.path.join(v8_path, "tools", "v8heapconst.py")
with open(expected_path) as f:

6
test/mozilla/testcfg.py
View File

@ -86,10 +86,10 @@ class TestSuite(testsuite.TestSuite):
def IsNegativeTest(self, test):
return test.path.endswith("-n")
def IsFailureOutput(self, test):
if test.output.exit_code != 0:
def IsFailureOutput(self, test, output):
if output.exit_code != 0:
return True
return "FAILED!" in test.output.stdout
return "FAILED!" in output.stdout
class TestCase(testcase.TestCase):

3
test/test262/testcfg.py
View File

@ -190,8 +190,7 @@ class TestSuite(testsuite.TestSuite):
def _test_class(self):
return TestCase
def IsFailureOutput(self, test):
output = test.output
def IsFailureOutput(self, test, output):
test_record = test.test_record
if output.exit_code != 0:
return True

6
test/webkit/testcfg.py
View File

@ -73,8 +73,8 @@ class TestSuite(testsuite.TestSuite):
string == "Warning: unknown flag --enable-slow-asserts." or
string == "Try --help for options")
def IsFailureOutput(self, test):
if super(TestSuite, self).IsFailureOutput(test):
def IsFailureOutput(self, test, output):
if super(TestSuite, self).IsFailureOutput(test, output):
return True
file_name = os.path.join(self.root, test.path) + "-expected.txt"
with file(file_name, "r") as expected:
@ -94,7 +94,7 @@ class TestSuite(testsuite.TestSuite):
def ActBlockIterator():
"""Iterates over blocks of actual output lines."""
lines = test.output.stdout.splitlines()
lines = output.stdout.splitlines()
start_index = 0
found_eqeq = False
for index, line in enumerate(lines):

2
tools/testrunner/deopt_fuzzer.py
View File

@ -332,7 +332,7 @@ class DeoptFuzzer(base_runner.BaseTestRunner):
for s in suites:
test_results = {}
for t in s.tests:
for line in t.output.stdout.splitlines():
for line in runner.outputs[t].stdout.splitlines():
if line.startswith("=== Stress deopt counter: "):
test_results[t.path] = MAX_DEOPT - int(line.split(" ")[-1])
for t in s.tests:

12
tools/testrunner/gc_fuzzer.py
View File

@ -213,13 +213,15 @@ class GCFuzzer(base_runner.BaseTestRunner):
for s in suites:
for t in s.tests:
# Skip failed tests.
if s.HasUnexpectedOutput(t):
if s.HasUnexpectedOutput(t, runner.outputs[t]):
print '%s failed, skipping' % t.path
continue
max_limit = self._get_max_limit_reached(t)
max_limit = self._get_max_limit_reached(runner.outputs[t])
if max_limit:
test_results[t.path] = max_limit
runner = None
if options.dump_results_file:
with file("%s.%d.txt" % (options.dump_results_file, time.time()),
"w") as f:
@ -308,7 +310,7 @@ class GCFuzzer(base_runner.BaseTestRunner):
# incremental marking limit (0-100).
# Skips values >=100% since they already trigger incremental marking.
@staticmethod
def _get_max_limit_reached(test):
def _get_max_limit_reached(output):
def is_im_line(l):
return 'IncrementalMarking' in l and '% of the memory limit reached' in l
@ -318,10 +320,10 @@ class GCFuzzer(base_runner.BaseTestRunner):
def percent_str_to_float(s):
return float(s[:-1])
if not (test.output and test.output.stdout):
if not (output and output.stdout):
return None
im_lines = filter(is_im_line, test.output.stdout.splitlines())
im_lines = filter(is_im_line, output.stdout.splitlines())
percents_str = map(line_to_percent, im_lines)
percents = map(percent_str_to_float, percents_str)

16
tools/testrunner/local/execution.py
View File

@ -37,7 +37,6 @@ from . import command
from . import perfdata
from . import statusfile
from . import utils
from ..objects import output
from pool import Pool
@ -119,6 +118,10 @@ class Runner(object):
self.perf_failures = False
self.printed_allocations = False
self.tests = [t for s in suites for t in s.tests]
# TODO(majeski): Pass outputs dynamically instead of keeping them in the
# runner.
self.outputs = {t: None for t in self.tests}
self.suite_names = [s.name for s in suites]
# Always pre-sort by status file, slowest tests first.
@ -174,19 +177,20 @@ class Runner(object):
# Rerun this test.
test.duration = None
test.output = None
test.run += 1
pool.add([TestJob(test.id, test.cmd, test.run)])
self.remaining += 1
self.total += 1
def _ProcessTest(self, test, result, pool):
test.output = result[1]
output = result[1]
self.outputs[test] = output
test.duration = result[2]
has_unexpected_output = test.suite.HasUnexpectedOutput(test, self.context)
has_unexpected_output = test.suite.HasUnexpectedOutput(
test, output, self.context)
if has_unexpected_output:
self.failed.append(test)
if test.output.HasCrashed():
if output.HasCrashed():
self.crashed += 1
else:
self.succeeded += 1
@ -194,7 +198,7 @@ class Runner(object):
# For the indicator, everything that happens after the first run is treated
# as unexpected even if it flakily passes in order to include it in the
# output.
self.indicator.HasRun(test, has_unexpected_output or test.run > 1)
self.indicator.HasRun(test, output, has_unexpected_output or test.run > 1)
if has_unexpected_output:
# Rerun test failures after the indicator has processed the results.
self._VerbosePrint("Attempting to rerun test after failure.")

61
tools/testrunner/local/progress.py
View File

@ -50,7 +50,7 @@ class ProgressIndicator(object):
def Done(self):
pass
def HasRun(self, test, has_unexpected_output):
def HasRun(self, test, output, has_unexpected_output):
pass
def Heartbeat(self):
@ -100,18 +100,19 @@ class SimpleProgressIndicator(ProgressIndicator):
def Done(self):
print
for failed in self.runner.failed:
output = self.runner.outputs[failed]
self.PrintFailureHeader(failed)
if failed.output.stderr:
if output.stderr:
print "--- stderr ---"
print failed.output.stderr.strip()
if failed.output.stdout:
print output.stderr.strip()
if output.stdout:
print "--- stdout ---"
print failed.output.stdout.strip()
print output.stdout.strip()
print "Command: %s" % failed.cmd.to_string()
if failed.output.HasCrashed():
print "exit code: %d" % failed.output.exit_code
if output.HasCrashed():
print "exit code: %d" % output.exit_code
print "--- CRASHED ---"
if failed.output.HasTimedOut():
if output.HasTimedOut():
print "--- TIMEOUT ---"
if len(self.runner.failed) == 0:
print "==="
@ -128,9 +129,9 @@ class SimpleProgressIndicator(ProgressIndicator):
class VerboseProgressIndicator(SimpleProgressIndicator):
def HasRun(self, test, has_unexpected_output):
def HasRun(self, test, output, has_unexpected_output):
if has_unexpected_output:
if test.output.HasCrashed():
if output.HasCrashed():
outcome = 'CRASH'
else:
outcome = 'FAIL'
@ -146,15 +147,15 @@ class VerboseProgressIndicator(SimpleProgressIndicator):
class DotsProgressIndicator(SimpleProgressIndicator):
def HasRun(self, test, has_unexpected_output):
def HasRun(self, test, output, has_unexpected_output):
total = self.runner.succeeded + len(self.runner.failed)
if (total > 1) and (total % 50 == 1):
sys.stdout.write('\n')
if has_unexpected_output:
if test.output.HasCrashed():
if output.HasCrashed():
sys.stdout.write('C')
sys.stdout.flush()
elif test.output.HasTimedOut():
elif output.HasTimedOut():
sys.stdout.write('T')
sys.stdout.flush()
else:
@ -178,22 +179,22 @@ class CompactProgressIndicator(ProgressIndicator):
self.PrintProgress('Done')
print "" # Line break.
def HasRun(self, test, has_unexpected_output):
def HasRun(self, test, output, has_unexpected_output):
self.PrintProgress(str(test))
if has_unexpected_output:
self.ClearLine(self.last_status_length)
self.PrintFailureHeader(test)
stdout = test.output.stdout.strip()
stdout = output.stdout.strip()
if len(stdout):
print self.templates['stdout'] % stdout
stderr = test.output.stderr.strip()
stderr = output.stderr.strip()
if len(stderr):
print self.templates['stderr'] % stderr
print "Command: %s" % test.cmd.to_string()
if test.output.HasCrashed():
print "exit code: %d" % test.output.exit_code
if output.HasCrashed():
print "exit code: %d" % output.exit_code
print "--- CRASHED ---"
if test.output.HasTimedOut():
if output.HasTimedOut():
print "--- TIMEOUT ---"
def Truncate(self, string, length):
@ -269,19 +270,19 @@ class JUnitTestProgressIndicator(ProgressIndicator):
if self.outfile != sys.stdout:
self.outfile.close()
def HasRun(self, test, has_unexpected_output):
def HasRun(self, test, output, has_unexpected_output):
fail_text = ""
if has_unexpected_output:
stdout = test.output.stdout.strip()
stdout = output.stdout.strip()
if len(stdout):
fail_text += "stdout:\n%s\n" % stdout
stderr = test.output.stderr.strip()
stderr = output.stderr.strip()
if len(stderr):
fail_text += "stderr:\n%s\n" % stderr
fail_text += "Command: %s" % self.test.cmd.to_string()
if test.output.HasCrashed():
fail_text += "exit code: %d\n--- CRASHED ---" % test.output.exit_code
if test.output.HasTimedOut():
if output.HasCrashed():
fail_text += "exit code: %d\n--- CRASHED ---" % output.exit_code
if output.HasTimedOut():
fail_text += "--- TIMEOUT ---"
self.outputter.HasRunTest(
test_name=str(test),
@ -340,7 +341,7 @@ class JsonTestProgressIndicator(ProgressIndicator):
with open(self.json_test_results, "w") as f:
f.write(json.dumps(complete_results))
def HasRun(self, test, has_unexpected_output):
def HasRun(self, test, output, has_unexpected_output):
# Buffer all tests for sorting the durations in the end.
self.tests.append(test)
if not has_unexpected_output:
@ -353,9 +354,9 @@ class JsonTestProgressIndicator(ProgressIndicator):
"flags": test.cmd.args,
"command": test.cmd.to_string(relative=True),
"run": test.run,
"stdout": test.output.stdout,
"stderr": test.output.stderr,
"exit_code": test.output.exit_code,
"stdout": output.stdout,
"stderr": output.stderr,
"exit_code": output.exit_code,
"result": test.suite.GetOutcome(test),
"expected": test.suite.GetExpectedOutcomes(test),
"duration": test.duration,
@ -393,7 +394,7 @@ class FlakinessTestProgressIndicator(ProgressIndicator):
"version": 3,
}, f)
def HasRun(self, test, has_unexpected_output):
def HasRun(self, test, output, has_unexpected_output):
key = test.get_id()
outcome = test.suite.GetOutcome(test)
assert outcome in ["PASS", "FAIL", "CRASH", "TIMEOUT"]

22
tools/testrunner/local/testsuite.py
View File

@ -276,36 +276,36 @@ class TestSuite(object):
return self._outcomes_cache[cache_key]
def IsFailureOutput(self, testcase):
return testcase.output.exit_code != 0
def IsFailureOutput(self, testcase, output):
return output.exit_code != 0
def IsNegativeTest(self, testcase):
return False
def HasFailed(self, testcase, ctx=None):
def HasFailed(self, testcase, output, ctx=None):
if ctx and ctx.predictable:
# Only check the exit code of the predictable_wrapper in
# verify-predictable mode.
execution_failed = testcase.output.exit_code != 0
execution_failed = output.exit_code != 0
else:
execution_failed = self.IsFailureOutput(testcase)
execution_failed = self.IsFailureOutput(testcase, output)
if self.IsNegativeTest(testcase):
return not execution_failed
else:
return execution_failed
def GetOutcome(self, testcase, ctx=None):
if testcase.output.HasCrashed():
def GetOutcome(self, testcase, output, ctx=None):
if output.HasCrashed():
return statusfile.CRASH
elif testcase.output.HasTimedOut():
elif output.HasTimedOut():
return statusfile.TIMEOUT
elif self.HasFailed(testcase, ctx):
elif self.HasFailed(testcase, output, ctx):
return statusfile.FAIL
else:
return statusfile.PASS
def HasUnexpectedOutput(self, testcase, ctx=None):
return (self.GetOutcome(testcase, ctx)
def HasUnexpectedOutput(self, testcase, output, ctx=None):
return (self.GetOutcome(testcase, output, ctx)
not in self.GetExpectedOutcomes(testcase))
def _create_test(self, path, **kwargs):

4
tools/testrunner/objects/testcase.py
View File

@ -45,7 +45,6 @@ class TestCase(object):
self.variant = None # name of the used testing variant
self.variant_flags = [] # list of strings, flags specific to this test
self.output = None
self.id = None # int, used to map result back to TestCase instance
self.duration = None # assigned during execution
self.run = 1 # The nth time this test is executed.
@ -198,6 +197,9 @@ class TestCase(object):
(other.suite.name, other.name, other.variant_flags)
)
def __hash__(self):
return hash((self.suite.name, self.name, ''.join(self.variant_flags)))
def __str__(self):
return self.suite.name + '/' + self.name