diff --git a/tools/run-tests.py b/tools/run-tests.py index cfca80f64b..61f91f0a5e 100755 --- a/tools/run-tests.py +++ b/tools/run-tests.py @@ -173,6 +173,8 @@ def BuildOptions(): help=("Quick check mode (skip slow/flaky tests)")) result.add_option("--report", help="Print a summary of the tests to be run", default=False, action="store_true") + result.add_option("--json-test-results", + help="Path to a file for storing json results.") result.add_option("--shard-count", help="Split testsuites into this number of shards", default=1, type="int") @@ -463,6 +465,9 @@ def Execute(arch, mode, args, options, suites, workspace): if options.junitout: progress_indicator = progress.JUnitTestProgressIndicator( progress_indicator, options.junitout, options.junittestsuite) + if options.json_test_results: + progress_indicator = progress.JsonTestProgressIndicator( + progress_indicator, options.json_test_results, arch, mode) run_networked = not options.no_network if not run_networked: diff --git a/tools/testrunner/local/progress.py b/tools/testrunner/local/progress.py index 03116ee768..870dcc6b0b 100644 --- a/tools/testrunner/local/progress.py +++ b/tools/testrunner/local/progress.py @@ -26,11 +26,17 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import json +import os import sys import time from . import junit_output + +ABS_PATH_PREFIX = os.getcwd() + os.sep + + def EscapeCommand(command): parts = [] for part in command: @@ -277,6 +283,55 @@ class JUnitTestProgressIndicator(ProgressIndicator): fail_text) +class JsonTestProgressIndicator(ProgressIndicator): + + def __init__(self, progress_indicator, json_test_results, arch, mode): + self.progress_indicator = progress_indicator + self.json_test_results = json_test_results + self.arch = arch + self.mode = mode + self.results = [] + + def Starting(self): + self.progress_indicator.runner = self.runner + self.progress_indicator.Starting() + + def Done(self): + self.progress_indicator.Done() + complete_results = [] + if os.path.exists(self.json_test_results): + with open(self.json_test_results, "r") as f: + # Buildbot might start out with an empty file. + complete_results = json.loads(f.read() or "[]") + + complete_results.append({ + "arch": self.arch, + "mode": self.mode, + "results": self.results, + }) + + with open(self.json_test_results, "w") as f: + f.write(json.dumps(complete_results)) + + def AboutToRun(self, test): + self.progress_indicator.AboutToRun(test) + + def HasRun(self, test, has_unexpected_output): + self.progress_indicator.HasRun(test, has_unexpected_output) + if not has_unexpected_output: + return + self.results.append({ + "name": test.GetLabel(), + "flags": test.flags, + "command": EscapeCommand(self.runner.GetCommand(test)).replace( + ABS_PATH_PREFIX, ""), + "stdout": test.output.stdout, + "stderr": test.output.stderr, + "exit_code": test.output.exit_code, + "result": "CRASH" if test.output.HasCrashed() else "FAIL", + }) + + PROGRESS_INDICATORS = { 'verbose': VerboseProgressIndicator, 'dots': DotsProgressIndicator,