From ba584456e667cd4f9d2400a9883f351a3c95b799 Mon Sep 17 00:00:00 2001 From: "machenbach@chromium.org" Date: Thu, 3 Jul 2014 09:33:22 +0000 Subject: [PATCH] Fix result status of rerun flaky tests. Tests that pass on reruns where wrongly treated as failures. Now the result state can include any of (PASS, FAIL, CRASH, TIMEOUT) BUG=374134 LOG=n R=jkummerow@chromium.org Review URL: https://codereview.chromium.org/363883003 git-svn-id: https://v8.googlecode.com/svn/branches/bleeding_edge@22186 ce2b1a6d-e550-0410-aec6-3dcde31c8c00 --- tools/testrunner/local/progress.py | 3 ++- tools/testrunner/local/testsuite.py | 17 +++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/tools/testrunner/local/progress.py b/tools/testrunner/local/progress.py index 3167028186..2c9f650013 100644 --- a/tools/testrunner/local/progress.py +++ b/tools/testrunner/local/progress.py @@ -322,6 +322,7 @@ class JsonTestProgressIndicator(ProgressIndicator): # Omit tests that pass on the first run, but collect output of tests # that pass when rerun. return + self.results.append({ "name": test.GetLabel(), "flags": test.flags, @@ -331,7 +332,7 @@ class JsonTestProgressIndicator(ProgressIndicator): "stdout": test.output.stdout, "stderr": test.output.stderr, "exit_code": test.output.exit_code, - "result": "CRASH" if test.output.HasCrashed() else "FAIL", + "result": test.suite.GetOutcome(test), }) diff --git a/tools/testrunner/local/testsuite.py b/tools/testrunner/local/testsuite.py index ff51196a56..0fd3f3a300 100644 --- a/tools/testrunner/local/testsuite.py +++ b/tools/testrunner/local/testsuite.py @@ -190,18 +190,19 @@ class TestSuite(object): else: return execution_failed - def HasUnexpectedOutput(self, testcase): + def GetOutcome(self, testcase): if testcase.output.HasCrashed(): - outcome = statusfile.CRASH + return statusfile.CRASH elif testcase.output.HasTimedOut(): - outcome = statusfile.TIMEOUT + return statusfile.TIMEOUT elif self.HasFailed(testcase): - outcome = statusfile.FAIL + return statusfile.FAIL else: - outcome = statusfile.PASS - if not testcase.outcomes: - return outcome != statusfile.PASS - return not outcome in testcase.outcomes + return statusfile.PASS + + def HasUnexpectedOutput(self, testcase): + outcome = self.GetOutcome(testcase) + return not outcome in (testcase.outcomes or [statusfile.PASS]) def StripOutputForTransmit(self, testcase): if not self.HasUnexpectedOutput(testcase):