2014-09-02 09:17:26 +00:00
|
|
|
#!/usr/bin/env python
|
|
|
|
# Copyright 2014 the V8 project authors. All rights reserved.
|
|
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
|
|
# found in the LICENSE file.
|
|
|
|
|
|
|
|
"""
|
|
|
|
Performance runner for d8.
|
|
|
|
|
|
|
|
Call e.g. with tools/run-perf.py --arch ia32 some_suite.json
|
|
|
|
|
|
|
|
The suite json format is expected to be:
|
|
|
|
{
|
|
|
|
"path": <relative path chunks to perf resources and main file>,
|
|
|
|
"name": <optional suite name, file name is default>,
|
|
|
|
"archs": [<architecture name for which this suite is run>, ...],
|
|
|
|
"binary": <name of binary to run, default "d8">,
|
|
|
|
"flags": [<flag to d8>, ...],
|
2014-11-28 10:34:28 +00:00
|
|
|
"test_flags": [<flag to the test file>, ...],
|
2014-09-02 09:17:26 +00:00
|
|
|
"run_count": <how often will this suite run (optional)>,
|
|
|
|
"run_count_XXX": <how often will this suite run for arch XXX (optional)>,
|
2014-11-28 11:35:37 +00:00
|
|
|
"resources": [<js file to be moved to android device>, ...]
|
2014-09-02 09:17:26 +00:00
|
|
|
"main": <main js perf runner file>,
|
|
|
|
"results_regexp": <optional regexp>,
|
|
|
|
"results_processor": <optional python results processor script>,
|
|
|
|
"units": <the unit specification for the performance dashboard>,
|
|
|
|
"tests": [
|
|
|
|
{
|
|
|
|
"name": <name of the trace>,
|
|
|
|
"results_regexp": <optional more specific regexp>,
|
|
|
|
"results_processor": <optional python results processor script>,
|
|
|
|
"units": <the unit specification for the performance dashboard>,
|
|
|
|
}, ...
|
|
|
|
]
|
|
|
|
}
|
|
|
|
|
|
|
|
The tests field can also nest other suites in arbitrary depth. A suite
|
|
|
|
with a "main" file is a leaf suite that can contain one more level of
|
|
|
|
tests.
|
|
|
|
|
|
|
|
A suite's results_regexp is expected to have one string place holder
|
|
|
|
"%s" for the trace name. A trace's results_regexp overwrites suite
|
|
|
|
defaults.
|
|
|
|
|
|
|
|
A suite's results_processor may point to an optional python script. If
|
|
|
|
specified, it is called after running the tests like this (with a path
|
|
|
|
relatve to the suite level's path):
|
|
|
|
<results_processor file> <same flags as for d8> <suite level name> <output>
|
|
|
|
|
|
|
|
The <output> is a temporary file containing d8 output. The results_regexp will
|
|
|
|
be applied to the output of this script.
|
|
|
|
|
|
|
|
A suite without "tests" is considered a performance test itself.
|
|
|
|
|
|
|
|
Full example (suite with one runner):
|
|
|
|
{
|
|
|
|
"path": ["."],
|
|
|
|
"flags": ["--expose-gc"],
|
2014-11-28 10:34:28 +00:00
|
|
|
"test_flags": ["5"],
|
2014-09-02 09:17:26 +00:00
|
|
|
"archs": ["ia32", "x64"],
|
|
|
|
"run_count": 5,
|
|
|
|
"run_count_ia32": 3,
|
|
|
|
"main": "run.js",
|
|
|
|
"results_regexp": "^%s: (.+)$",
|
|
|
|
"units": "score",
|
|
|
|
"tests": [
|
|
|
|
{"name": "Richards"},
|
|
|
|
{"name": "DeltaBlue"},
|
|
|
|
{"name": "NavierStokes",
|
|
|
|
"results_regexp": "^NavierStokes: (.+)$"}
|
|
|
|
]
|
|
|
|
}
|
|
|
|
|
|
|
|
Full example (suite with several runners):
|
|
|
|
{
|
|
|
|
"path": ["."],
|
|
|
|
"flags": ["--expose-gc"],
|
|
|
|
"archs": ["ia32", "x64"],
|
|
|
|
"run_count": 5,
|
|
|
|
"units": "score",
|
|
|
|
"tests": [
|
|
|
|
{"name": "Richards",
|
|
|
|
"path": ["richards"],
|
|
|
|
"main": "run.js",
|
|
|
|
"run_count": 3,
|
|
|
|
"results_regexp": "^Richards: (.+)$"},
|
|
|
|
{"name": "NavierStokes",
|
|
|
|
"path": ["navier_stokes"],
|
|
|
|
"main": "run.js",
|
|
|
|
"results_regexp": "^NavierStokes: (.+)$"}
|
|
|
|
]
|
|
|
|
}
|
|
|
|
|
|
|
|
Path pieces are concatenated. D8 is always run with the suite's path as cwd.
|
2014-11-28 10:34:28 +00:00
|
|
|
|
|
|
|
The test flags are passed to the js test file after '--'.
|
2014-09-02 09:17:26 +00:00
|
|
|
"""
|
|
|
|
|
2014-10-10 07:12:38 +00:00
|
|
|
from collections import OrderedDict
|
2014-09-02 09:17:26 +00:00
|
|
|
import json
|
2014-12-02 14:23:51 +00:00
|
|
|
import logging
|
2014-09-02 09:17:26 +00:00
|
|
|
import math
|
|
|
|
import optparse
|
|
|
|
import os
|
|
|
|
import re
|
2015-09-14 23:14:20 +00:00
|
|
|
import subprocess
|
2014-09-02 09:17:26 +00:00
|
|
|
import sys
|
|
|
|
|
|
|
|
from testrunner.local import commands
|
|
|
|
from testrunner.local import utils
|
|
|
|
|
|
|
|
ARCH_GUESS = utils.DefaultArch()
|
2015-06-29 15:07:15 +00:00
|
|
|
SUPPORTED_ARCHS = ["arm",
|
2014-09-02 09:17:26 +00:00
|
|
|
"ia32",
|
|
|
|
"mips",
|
|
|
|
"mipsel",
|
|
|
|
"nacl_ia32",
|
|
|
|
"nacl_x64",
|
|
|
|
"x64",
|
|
|
|
"arm64"]
|
|
|
|
|
2014-10-10 07:12:38 +00:00
|
|
|
GENERIC_RESULTS_RE = re.compile(r"^RESULT ([^:]+): ([^=]+)= ([^ ]+) ([^ ]*)$")
|
|
|
|
RESULT_STDDEV_RE = re.compile(r"^\{([^\}]+)\}$")
|
|
|
|
RESULT_LIST_RE = re.compile(r"^\[([^\]]+)\]$")
|
2015-09-15 08:15:54 +00:00
|
|
|
TOOLS_BASE = os.path.abspath(os.path.dirname(__file__))
|
2014-10-10 07:12:38 +00:00
|
|
|
|
2014-09-02 09:17:26 +00:00
|
|
|
|
2014-12-02 14:23:51 +00:00
|
|
|
def LoadAndroidBuildTools(path): # pragma: no cover
|
|
|
|
assert os.path.exists(path)
|
|
|
|
sys.path.insert(0, path)
|
|
|
|
|
2015-07-27 14:56:54 +00:00
|
|
|
from pylib.device import adb_wrapper # pylint: disable=F0401
|
2014-12-02 14:23:51 +00:00
|
|
|
from pylib.device import device_errors # pylint: disable=F0401
|
2015-07-27 14:56:54 +00:00
|
|
|
from pylib.device import device_utils # pylint: disable=F0401
|
2014-12-02 14:23:51 +00:00
|
|
|
from pylib.perf import cache_control # pylint: disable=F0401
|
|
|
|
from pylib.perf import perf_control # pylint: disable=F0401
|
2015-07-27 14:56:54 +00:00
|
|
|
global adb_wrapper
|
2014-12-02 14:23:51 +00:00
|
|
|
global cache_control
|
|
|
|
global device_errors
|
|
|
|
global device_utils
|
|
|
|
global perf_control
|
|
|
|
|
2014-09-02 09:17:26 +00:00
|
|
|
|
|
|
|
def GeometricMean(values):
|
|
|
|
"""Returns the geometric mean of a list of values.
|
|
|
|
|
|
|
|
The mean is calculated using log to avoid overflow.
|
|
|
|
"""
|
|
|
|
values = map(float, values)
|
|
|
|
return str(math.exp(sum(map(math.log, values)) / len(values)))
|
|
|
|
|
|
|
|
|
|
|
|
class Results(object):
|
|
|
|
"""Place holder for result traces."""
|
|
|
|
def __init__(self, traces=None, errors=None):
|
|
|
|
self.traces = traces or []
|
|
|
|
self.errors = errors or []
|
|
|
|
|
|
|
|
def ToDict(self):
|
|
|
|
return {"traces": self.traces, "errors": self.errors}
|
|
|
|
|
|
|
|
def WriteToFile(self, file_name):
|
|
|
|
with open(file_name, "w") as f:
|
|
|
|
f.write(json.dumps(self.ToDict()))
|
|
|
|
|
|
|
|
def __add__(self, other):
|
|
|
|
self.traces += other.traces
|
|
|
|
self.errors += other.errors
|
|
|
|
return self
|
|
|
|
|
|
|
|
def __str__(self): # pragma: no cover
|
|
|
|
return str(self.ToDict())
|
|
|
|
|
|
|
|
|
2015-07-08 12:31:29 +00:00
|
|
|
class Measurement(object):
|
|
|
|
"""Represents a series of results of one trace.
|
|
|
|
|
|
|
|
The results are from repetitive runs of the same executable. They are
|
|
|
|
gathered by repeated calls to ConsumeOutput.
|
|
|
|
"""
|
|
|
|
def __init__(self, graphs, units, results_regexp, stddev_regexp):
|
|
|
|
self.name = graphs[-1]
|
|
|
|
self.graphs = graphs
|
|
|
|
self.units = units
|
|
|
|
self.results_regexp = results_regexp
|
|
|
|
self.stddev_regexp = stddev_regexp
|
|
|
|
self.results = []
|
|
|
|
self.errors = []
|
|
|
|
self.stddev = ""
|
|
|
|
|
|
|
|
def ConsumeOutput(self, stdout):
|
|
|
|
try:
|
|
|
|
result = re.search(self.results_regexp, stdout, re.M).group(1)
|
|
|
|
self.results.append(str(float(result)))
|
|
|
|
except ValueError:
|
|
|
|
self.errors.append("Regexp \"%s\" returned a non-numeric for test %s."
|
|
|
|
% (self.results_regexp, self.name))
|
|
|
|
except:
|
|
|
|
self.errors.append("Regexp \"%s\" didn't match for test %s."
|
|
|
|
% (self.results_regexp, self.name))
|
|
|
|
|
|
|
|
try:
|
|
|
|
if self.stddev_regexp and self.stddev:
|
|
|
|
self.errors.append("Test %s should only run once since a stddev "
|
|
|
|
"is provided by the test." % self.name)
|
|
|
|
if self.stddev_regexp:
|
|
|
|
self.stddev = re.search(self.stddev_regexp, stdout, re.M).group(1)
|
|
|
|
except:
|
|
|
|
self.errors.append("Regexp \"%s\" didn't match for test %s."
|
|
|
|
% (self.stddev_regexp, self.name))
|
|
|
|
|
|
|
|
def GetResults(self):
|
|
|
|
return Results([{
|
|
|
|
"graphs": self.graphs,
|
|
|
|
"units": self.units,
|
|
|
|
"results": self.results,
|
|
|
|
"stddev": self.stddev,
|
|
|
|
}], self.errors)
|
|
|
|
|
|
|
|
|
2015-07-10 13:02:09 +00:00
|
|
|
class NullMeasurement(object):
|
|
|
|
"""Null object to avoid having extra logic for configurations that didn't
|
|
|
|
run like running without patch on trybots.
|
|
|
|
"""
|
|
|
|
def ConsumeOutput(self, stdout):
|
|
|
|
pass
|
|
|
|
|
|
|
|
def GetResults(self):
|
|
|
|
return Results()
|
|
|
|
|
|
|
|
|
|
|
|
def Unzip(iterable):
|
|
|
|
left = []
|
|
|
|
right = []
|
|
|
|
for l, r in iterable:
|
|
|
|
left.append(l)
|
|
|
|
right.append(r)
|
|
|
|
return lambda: iter(left), lambda: iter(right)
|
|
|
|
|
|
|
|
|
|
|
|
def AccumulateResults(
|
|
|
|
graph_names, trace_configs, iter_output, trybot, no_patch, calc_total):
|
2015-07-08 12:31:29 +00:00
|
|
|
"""Iterates over the output of multiple benchmark reruns and accumulates
|
|
|
|
results for a configured list of traces.
|
|
|
|
|
|
|
|
Args:
|
|
|
|
graph_names: List of names that configure the base path of the traces. E.g.
|
|
|
|
['v8', 'Octane'].
|
|
|
|
trace_configs: List of "TraceConfig" instances. Each trace config defines
|
|
|
|
how to perform a measurement.
|
|
|
|
iter_output: Iterator over the standard output of each test run.
|
2015-07-10 13:02:09 +00:00
|
|
|
trybot: Indicates that this is run in trybot mode, i.e. run twice, once
|
|
|
|
with once without patch.
|
|
|
|
no_patch: Indicates weather this is a trybot run without patch.
|
2015-07-08 12:31:29 +00:00
|
|
|
calc_total: Boolean flag to speficy the calculation of a summary trace.
|
|
|
|
Returns: A "Results" object.
|
|
|
|
"""
|
2015-07-10 13:02:09 +00:00
|
|
|
measurements = [
|
|
|
|
trace.CreateMeasurement(trybot, no_patch) for trace in trace_configs]
|
2015-07-08 12:31:29 +00:00
|
|
|
for stdout in iter_output():
|
|
|
|
for measurement in measurements:
|
|
|
|
measurement.ConsumeOutput(stdout)
|
|
|
|
|
|
|
|
res = reduce(lambda r, m: r + m.GetResults(), measurements, Results())
|
|
|
|
|
|
|
|
if not res.traces or not calc_total:
|
|
|
|
return res
|
|
|
|
|
|
|
|
# Assume all traces have the same structure.
|
|
|
|
if len(set(map(lambda t: len(t["results"]), res.traces))) != 1:
|
|
|
|
res.errors.append("Not all traces have the same number of results.")
|
|
|
|
return res
|
|
|
|
|
|
|
|
# Calculate the geometric means for all traces. Above we made sure that
|
|
|
|
# there is at least one trace and that the number of results is the same
|
|
|
|
# for each trace.
|
|
|
|
n_results = len(res.traces[0]["results"])
|
|
|
|
total_results = [GeometricMean(t["results"][i] for t in res.traces)
|
|
|
|
for i in range(0, n_results)]
|
|
|
|
res.traces.append({
|
|
|
|
"graphs": graph_names + ["Total"],
|
|
|
|
"units": res.traces[0]["units"],
|
|
|
|
"results": total_results,
|
|
|
|
"stddev": "",
|
|
|
|
})
|
|
|
|
return res
|
|
|
|
|
|
|
|
|
|
|
|
def AccumulateGenericResults(graph_names, suite_units, iter_output):
|
|
|
|
"""Iterates over the output of multiple benchmark reruns and accumulates
|
|
|
|
generic results.
|
|
|
|
|
|
|
|
Args:
|
|
|
|
graph_names: List of names that configure the base path of the traces. E.g.
|
|
|
|
['v8', 'Octane'].
|
|
|
|
suite_units: Measurement default units as defined by the benchmark suite.
|
|
|
|
iter_output: Iterator over the standard output of each test run.
|
|
|
|
Returns: A "Results" object.
|
|
|
|
"""
|
|
|
|
traces = OrderedDict()
|
|
|
|
for stdout in iter_output():
|
2015-07-10 13:02:09 +00:00
|
|
|
if stdout is None:
|
|
|
|
# The None value is used as a null object to simplify logic.
|
|
|
|
continue
|
2015-07-08 12:31:29 +00:00
|
|
|
for line in stdout.strip().splitlines():
|
|
|
|
match = GENERIC_RESULTS_RE.match(line)
|
|
|
|
if match:
|
|
|
|
stddev = ""
|
|
|
|
graph = match.group(1)
|
|
|
|
trace = match.group(2)
|
|
|
|
body = match.group(3)
|
|
|
|
units = match.group(4)
|
|
|
|
match_stddev = RESULT_STDDEV_RE.match(body)
|
|
|
|
match_list = RESULT_LIST_RE.match(body)
|
|
|
|
errors = []
|
|
|
|
if match_stddev:
|
|
|
|
result, stddev = map(str.strip, match_stddev.group(1).split(","))
|
|
|
|
results = [result]
|
|
|
|
elif match_list:
|
|
|
|
results = map(str.strip, match_list.group(1).split(","))
|
|
|
|
else:
|
|
|
|
results = [body.strip()]
|
|
|
|
|
|
|
|
try:
|
|
|
|
results = map(lambda r: str(float(r)), results)
|
|
|
|
except ValueError:
|
|
|
|
results = []
|
|
|
|
errors = ["Found non-numeric in %s" %
|
|
|
|
"/".join(graph_names + [graph, trace])]
|
|
|
|
|
|
|
|
trace_result = traces.setdefault(trace, Results([{
|
|
|
|
"graphs": graph_names + [graph, trace],
|
|
|
|
"units": (units or suite_units).strip(),
|
|
|
|
"results": [],
|
|
|
|
"stddev": "",
|
|
|
|
}], errors))
|
|
|
|
trace_result.traces[0]["results"].extend(results)
|
|
|
|
trace_result.traces[0]["stddev"] = stddev
|
|
|
|
|
|
|
|
return reduce(lambda r, t: r + t, traces.itervalues(), Results())
|
|
|
|
|
|
|
|
|
2014-09-02 09:17:26 +00:00
|
|
|
class Node(object):
|
|
|
|
"""Represents a node in the suite tree structure."""
|
|
|
|
def __init__(self, *args):
|
|
|
|
self._children = []
|
|
|
|
|
|
|
|
def AppendChild(self, child):
|
|
|
|
self._children.append(child)
|
|
|
|
|
|
|
|
|
|
|
|
class DefaultSentinel(Node):
|
|
|
|
"""Fake parent node with all default values."""
|
|
|
|
def __init__(self):
|
|
|
|
super(DefaultSentinel, self).__init__()
|
|
|
|
self.binary = "d8"
|
|
|
|
self.run_count = 10
|
2014-09-15 13:00:32 +00:00
|
|
|
self.timeout = 60
|
2014-09-02 09:17:26 +00:00
|
|
|
self.path = []
|
|
|
|
self.graphs = []
|
|
|
|
self.flags = []
|
2014-11-28 10:34:28 +00:00
|
|
|
self.test_flags = []
|
2014-09-02 09:17:26 +00:00
|
|
|
self.resources = []
|
|
|
|
self.results_regexp = None
|
|
|
|
self.stddev_regexp = None
|
|
|
|
self.units = "score"
|
|
|
|
self.total = False
|
|
|
|
|
|
|
|
|
2015-07-08 12:31:29 +00:00
|
|
|
class GraphConfig(Node):
|
2014-09-02 09:17:26 +00:00
|
|
|
"""Represents a suite definition.
|
|
|
|
|
|
|
|
Can either be a leaf or an inner node that provides default values.
|
|
|
|
"""
|
|
|
|
def __init__(self, suite, parent, arch):
|
2015-07-08 12:31:29 +00:00
|
|
|
super(GraphConfig, self).__init__()
|
2014-09-02 09:17:26 +00:00
|
|
|
self._suite = suite
|
|
|
|
|
|
|
|
assert isinstance(suite.get("path", []), list)
|
|
|
|
assert isinstance(suite["name"], basestring)
|
|
|
|
assert isinstance(suite.get("flags", []), list)
|
2014-11-28 10:34:28 +00:00
|
|
|
assert isinstance(suite.get("test_flags", []), list)
|
2014-09-02 09:17:26 +00:00
|
|
|
assert isinstance(suite.get("resources", []), list)
|
|
|
|
|
|
|
|
# Accumulated values.
|
|
|
|
self.path = parent.path[:] + suite.get("path", [])
|
|
|
|
self.graphs = parent.graphs[:] + [suite["name"]]
|
|
|
|
self.flags = parent.flags[:] + suite.get("flags", [])
|
2014-11-28 10:34:28 +00:00
|
|
|
self.test_flags = parent.test_flags[:] + suite.get("test_flags", [])
|
2014-12-04 10:37:36 +00:00
|
|
|
|
|
|
|
# Values independent of parent node.
|
|
|
|
self.resources = suite.get("resources", [])
|
2014-09-02 09:17:26 +00:00
|
|
|
|
|
|
|
# Descrete values (with parent defaults).
|
|
|
|
self.binary = suite.get("binary", parent.binary)
|
|
|
|
self.run_count = suite.get("run_count", parent.run_count)
|
|
|
|
self.run_count = suite.get("run_count_%s" % arch, self.run_count)
|
2014-09-15 13:00:32 +00:00
|
|
|
self.timeout = suite.get("timeout", parent.timeout)
|
2014-12-02 14:23:51 +00:00
|
|
|
self.timeout = suite.get("timeout_%s" % arch, self.timeout)
|
2014-09-02 09:17:26 +00:00
|
|
|
self.units = suite.get("units", parent.units)
|
|
|
|
self.total = suite.get("total", parent.total)
|
|
|
|
|
|
|
|
# A regular expression for results. If the parent graph provides a
|
|
|
|
# regexp and the current suite has none, a string place holder for the
|
|
|
|
# suite name is expected.
|
|
|
|
# TODO(machenbach): Currently that makes only sense for the leaf level.
|
|
|
|
# Multiple place holders for multiple levels are not supported.
|
|
|
|
if parent.results_regexp:
|
|
|
|
regexp_default = parent.results_regexp % re.escape(suite["name"])
|
|
|
|
else:
|
|
|
|
regexp_default = None
|
|
|
|
self.results_regexp = suite.get("results_regexp", regexp_default)
|
|
|
|
|
|
|
|
# A similar regular expression for the standard deviation (optional).
|
|
|
|
if parent.stddev_regexp:
|
|
|
|
stddev_default = parent.stddev_regexp % re.escape(suite["name"])
|
|
|
|
else:
|
|
|
|
stddev_default = None
|
|
|
|
self.stddev_regexp = suite.get("stddev_regexp", stddev_default)
|
|
|
|
|
|
|
|
|
2015-07-08 12:31:29 +00:00
|
|
|
class TraceConfig(GraphConfig):
|
|
|
|
"""Represents a leaf in the suite tree structure."""
|
2014-09-02 09:17:26 +00:00
|
|
|
def __init__(self, suite, parent, arch):
|
2015-07-08 12:31:29 +00:00
|
|
|
super(TraceConfig, self).__init__(suite, parent, arch)
|
2014-09-02 09:17:26 +00:00
|
|
|
assert self.results_regexp
|
|
|
|
|
2015-07-10 13:02:09 +00:00
|
|
|
def CreateMeasurement(self, trybot, no_patch):
|
|
|
|
if not trybot and no_patch:
|
|
|
|
# Use null object for no-patch logic if this is not a trybot run.
|
|
|
|
return NullMeasurement()
|
|
|
|
|
2015-07-08 12:31:29 +00:00
|
|
|
return Measurement(
|
|
|
|
self.graphs,
|
|
|
|
self.units,
|
|
|
|
self.results_regexp,
|
|
|
|
self.stddev_regexp,
|
|
|
|
)
|
2014-09-02 09:17:26 +00:00
|
|
|
|
|
|
|
|
2015-07-08 12:31:29 +00:00
|
|
|
class RunnableConfig(GraphConfig):
|
2014-09-02 09:17:26 +00:00
|
|
|
"""Represents a runnable suite definition (i.e. has a main file).
|
|
|
|
"""
|
|
|
|
@property
|
|
|
|
def main(self):
|
|
|
|
return self._suite.get("main", "")
|
|
|
|
|
|
|
|
def ChangeCWD(self, suite_path):
|
|
|
|
"""Changes the cwd to to path defined in the current graph.
|
|
|
|
|
|
|
|
The tests are supposed to be relative to the suite configuration.
|
|
|
|
"""
|
|
|
|
suite_dir = os.path.abspath(os.path.dirname(suite_path))
|
|
|
|
bench_dir = os.path.normpath(os.path.join(*self.path))
|
|
|
|
os.chdir(os.path.join(suite_dir, bench_dir))
|
|
|
|
|
2015-05-13 10:00:43 +00:00
|
|
|
def GetCommandFlags(self, extra_flags=None):
|
2014-12-02 14:23:51 +00:00
|
|
|
suffix = ["--"] + self.test_flags if self.test_flags else []
|
2015-05-13 10:00:43 +00:00
|
|
|
return self.flags + (extra_flags or []) + [self.main] + suffix
|
2014-12-02 14:23:51 +00:00
|
|
|
|
2015-05-13 10:00:43 +00:00
|
|
|
def GetCommand(self, shell_dir, extra_flags=None):
|
2014-09-02 09:17:26 +00:00
|
|
|
# TODO(machenbach): This requires +.exe if run on windows.
|
2015-09-14 23:14:20 +00:00
|
|
|
extra_flags = extra_flags or []
|
2015-05-13 10:00:43 +00:00
|
|
|
cmd = [os.path.join(shell_dir, self.binary)]
|
2015-09-15 08:15:54 +00:00
|
|
|
if self.binary != 'd8' and '--prof' in extra_flags:
|
2015-09-14 23:14:20 +00:00
|
|
|
print "Profiler supported only on a benchmark run with d8"
|
2015-05-13 10:00:43 +00:00
|
|
|
return cmd + self.GetCommandFlags(extra_flags=extra_flags)
|
2014-09-02 09:17:26 +00:00
|
|
|
|
2015-07-10 13:02:09 +00:00
|
|
|
def Run(self, runner, trybot):
|
2014-09-02 09:17:26 +00:00
|
|
|
"""Iterates over several runs and handles the output for all traces."""
|
2015-07-10 13:02:09 +00:00
|
|
|
stdout_with_patch, stdout_no_patch = Unzip(runner())
|
|
|
|
return (
|
|
|
|
AccumulateResults(
|
|
|
|
self.graphs,
|
|
|
|
self._children,
|
|
|
|
iter_output=stdout_with_patch,
|
|
|
|
trybot=trybot,
|
|
|
|
no_patch=False,
|
|
|
|
calc_total=self.total,
|
|
|
|
),
|
|
|
|
AccumulateResults(
|
|
|
|
self.graphs,
|
|
|
|
self._children,
|
|
|
|
iter_output=stdout_no_patch,
|
|
|
|
trybot=trybot,
|
|
|
|
no_patch=True,
|
|
|
|
calc_total=self.total,
|
|
|
|
),
|
|
|
|
)
|
2014-09-02 09:17:26 +00:00
|
|
|
|
2015-07-08 12:31:29 +00:00
|
|
|
|
|
|
|
class RunnableTraceConfig(TraceConfig, RunnableConfig):
|
2014-09-02 09:17:26 +00:00
|
|
|
"""Represents a runnable suite definition that is a leaf."""
|
|
|
|
def __init__(self, suite, parent, arch):
|
2015-07-08 12:31:29 +00:00
|
|
|
super(RunnableTraceConfig, self).__init__(suite, parent, arch)
|
2014-09-02 09:17:26 +00:00
|
|
|
|
2015-07-10 13:02:09 +00:00
|
|
|
def Run(self, runner, trybot):
|
2014-09-02 09:17:26 +00:00
|
|
|
"""Iterates over several runs and handles the output."""
|
2015-07-10 13:02:09 +00:00
|
|
|
measurement_with_patch = self.CreateMeasurement(trybot, False)
|
|
|
|
measurement_no_patch = self.CreateMeasurement(trybot, True)
|
|
|
|
for stdout_with_patch, stdout_no_patch in runner():
|
|
|
|
measurement_with_patch.ConsumeOutput(stdout_with_patch)
|
|
|
|
measurement_no_patch.ConsumeOutput(stdout_no_patch)
|
|
|
|
return (
|
|
|
|
measurement_with_patch.GetResults(),
|
|
|
|
measurement_no_patch.GetResults(),
|
|
|
|
)
|
2014-09-02 09:17:26 +00:00
|
|
|
|
|
|
|
|
2015-07-08 12:31:29 +00:00
|
|
|
class RunnableGenericConfig(RunnableConfig):
|
2014-09-02 09:17:26 +00:00
|
|
|
"""Represents a runnable suite definition with generic traces."""
|
|
|
|
def __init__(self, suite, parent, arch):
|
2015-07-08 12:31:29 +00:00
|
|
|
super(RunnableGenericConfig, self).__init__(suite, parent, arch)
|
2014-09-02 09:17:26 +00:00
|
|
|
|
2015-07-10 13:02:09 +00:00
|
|
|
def Run(self, runner, trybot):
|
|
|
|
stdout_with_patch, stdout_no_patch = Unzip(runner())
|
|
|
|
return (
|
|
|
|
AccumulateGenericResults(self.graphs, self.units, stdout_with_patch),
|
|
|
|
AccumulateGenericResults(self.graphs, self.units, stdout_no_patch),
|
|
|
|
)
|
2015-07-08 12:31:29 +00:00
|
|
|
|
|
|
|
|
|
|
|
def MakeGraphConfig(suite, arch, parent):
|
|
|
|
"""Factory method for making graph configuration objects."""
|
|
|
|
if isinstance(parent, RunnableConfig):
|
2014-09-02 09:17:26 +00:00
|
|
|
# Below a runnable can only be traces.
|
2015-07-08 12:31:29 +00:00
|
|
|
return TraceConfig(suite, parent, arch)
|
2015-04-10 09:48:54 +00:00
|
|
|
elif suite.get("main") is not None:
|
|
|
|
# A main file makes this graph runnable. Empty strings are accepted.
|
2014-09-02 09:17:26 +00:00
|
|
|
if suite.get("tests"):
|
|
|
|
# This graph has subgraphs (traces).
|
2015-07-08 12:31:29 +00:00
|
|
|
return RunnableConfig(suite, parent, arch)
|
2014-09-02 09:17:26 +00:00
|
|
|
else:
|
|
|
|
# This graph has no subgraphs, it's a leaf.
|
2015-07-08 12:31:29 +00:00
|
|
|
return RunnableTraceConfig(suite, parent, arch)
|
2014-09-02 09:17:26 +00:00
|
|
|
elif suite.get("generic"):
|
|
|
|
# This is a generic suite definition. It is either a runnable executable
|
|
|
|
# or has a main js file.
|
2015-07-08 12:31:29 +00:00
|
|
|
return RunnableGenericConfig(suite, parent, arch)
|
2014-09-02 09:17:26 +00:00
|
|
|
elif suite.get("tests"):
|
|
|
|
# This is neither a leaf nor a runnable.
|
2015-07-08 12:31:29 +00:00
|
|
|
return GraphConfig(suite, parent, arch)
|
2014-09-02 09:17:26 +00:00
|
|
|
else: # pragma: no cover
|
|
|
|
raise Exception("Invalid suite configuration.")
|
|
|
|
|
|
|
|
|
2015-07-08 12:31:29 +00:00
|
|
|
def BuildGraphConfigs(suite, arch, parent=None):
|
2014-09-02 09:17:26 +00:00
|
|
|
"""Builds a tree structure of graph objects that corresponds to the suite
|
|
|
|
configuration.
|
|
|
|
"""
|
|
|
|
parent = parent or DefaultSentinel()
|
|
|
|
|
|
|
|
# TODO(machenbach): Implement notion of cpu type?
|
2014-12-02 14:23:51 +00:00
|
|
|
if arch not in suite.get("archs", SUPPORTED_ARCHS):
|
2014-09-02 09:17:26 +00:00
|
|
|
return None
|
|
|
|
|
2015-07-08 12:31:29 +00:00
|
|
|
graph = MakeGraphConfig(suite, arch, parent)
|
2014-09-02 09:17:26 +00:00
|
|
|
for subsuite in suite.get("tests", []):
|
2015-07-08 12:31:29 +00:00
|
|
|
BuildGraphConfigs(subsuite, arch, graph)
|
2014-09-02 09:17:26 +00:00
|
|
|
parent.AppendChild(graph)
|
|
|
|
return graph
|
|
|
|
|
|
|
|
|
2014-12-03 15:34:53 +00:00
|
|
|
def FlattenRunnables(node, node_cb):
|
2014-09-02 09:17:26 +00:00
|
|
|
"""Generator that traverses the tree structure and iterates over all
|
|
|
|
runnables.
|
|
|
|
"""
|
2014-12-03 15:34:53 +00:00
|
|
|
node_cb(node)
|
2015-07-08 12:31:29 +00:00
|
|
|
if isinstance(node, RunnableConfig):
|
2014-09-02 09:17:26 +00:00
|
|
|
yield node
|
|
|
|
elif isinstance(node, Node):
|
|
|
|
for child in node._children:
|
2014-12-03 15:34:53 +00:00
|
|
|
for result in FlattenRunnables(child, node_cb):
|
2014-09-02 09:17:26 +00:00
|
|
|
yield result
|
|
|
|
else: # pragma: no cover
|
|
|
|
raise Exception("Invalid suite configuration.")
|
|
|
|
|
|
|
|
|
2014-11-28 18:39:41 +00:00
|
|
|
class Platform(object):
|
2015-05-18 15:11:05 +00:00
|
|
|
def __init__(self, options):
|
|
|
|
self.shell_dir = options.shell_dir
|
2015-07-10 13:02:09 +00:00
|
|
|
self.shell_dir_no_patch = options.shell_dir_no_patch
|
2015-05-18 15:11:05 +00:00
|
|
|
self.extra_flags = options.extra_flags.split()
|
|
|
|
|
2014-11-28 18:39:41 +00:00
|
|
|
@staticmethod
|
|
|
|
def GetPlatform(options):
|
2015-06-29 15:07:15 +00:00
|
|
|
if options.android_build_tools:
|
2014-11-28 18:39:41 +00:00
|
|
|
return AndroidPlatform(options)
|
|
|
|
else:
|
|
|
|
return DesktopPlatform(options)
|
|
|
|
|
2015-07-10 13:02:09 +00:00
|
|
|
def _Run(self, runnable, count, no_patch=False):
|
|
|
|
raise NotImplementedError() # pragma: no cover
|
|
|
|
|
|
|
|
def Run(self, runnable, count):
|
|
|
|
"""Execute the benchmark's main file.
|
|
|
|
|
|
|
|
If options.shell_dir_no_patch is specified, the benchmark is run once with
|
|
|
|
and once without patch.
|
|
|
|
Args:
|
|
|
|
runnable: A Runnable benchmark instance.
|
|
|
|
count: The number of this (repeated) run.
|
|
|
|
Returns: A tuple with the benchmark outputs with and without patch. The
|
|
|
|
latter will be None if options.shell_dir_no_patch was not
|
|
|
|
specified.
|
|
|
|
"""
|
|
|
|
stdout = self._Run(runnable, count, no_patch=False)
|
|
|
|
if self.shell_dir_no_patch:
|
|
|
|
return stdout, self._Run(runnable, count, no_patch=True)
|
|
|
|
else:
|
|
|
|
return stdout, None
|
|
|
|
|
2014-11-28 18:39:41 +00:00
|
|
|
|
|
|
|
class DesktopPlatform(Platform):
|
|
|
|
def __init__(self, options):
|
2015-05-18 15:11:05 +00:00
|
|
|
super(DesktopPlatform, self).__init__(options)
|
2014-11-28 18:39:41 +00:00
|
|
|
|
2014-12-02 14:23:51 +00:00
|
|
|
def PreExecution(self):
|
|
|
|
pass
|
2014-11-28 18:39:41 +00:00
|
|
|
|
2014-12-02 14:23:51 +00:00
|
|
|
def PostExecution(self):
|
2014-11-28 18:39:41 +00:00
|
|
|
pass
|
|
|
|
|
2014-12-03 15:34:53 +00:00
|
|
|
def PreTests(self, node, path):
|
2015-07-08 12:31:29 +00:00
|
|
|
if isinstance(node, RunnableConfig):
|
2014-12-03 15:34:53 +00:00
|
|
|
node.ChangeCWD(path)
|
2014-11-28 18:39:41 +00:00
|
|
|
|
2015-07-10 13:02:09 +00:00
|
|
|
def _Run(self, runnable, count, no_patch=False):
|
|
|
|
suffix = ' - without patch' if no_patch else ''
|
|
|
|
shell_dir = self.shell_dir_no_patch if no_patch else self.shell_dir
|
|
|
|
title = ">>> %%s (#%d)%s:" % ((count + 1), suffix)
|
2015-04-10 12:01:20 +00:00
|
|
|
try:
|
2015-05-13 10:00:43 +00:00
|
|
|
output = commands.Execute(
|
2015-07-10 13:02:09 +00:00
|
|
|
runnable.GetCommand(shell_dir, self.extra_flags),
|
2015-05-13 10:00:43 +00:00
|
|
|
timeout=runnable.timeout,
|
|
|
|
)
|
2015-07-10 13:02:09 +00:00
|
|
|
except OSError as e: # pragma: no cover
|
|
|
|
print title % "OSError"
|
2015-04-10 12:01:20 +00:00
|
|
|
print e
|
|
|
|
return ""
|
2015-07-10 13:02:09 +00:00
|
|
|
print title % "Stdout"
|
2014-11-28 18:39:41 +00:00
|
|
|
print output.stdout
|
|
|
|
if output.stderr: # pragma: no cover
|
|
|
|
# Print stderr for debugging.
|
2015-07-10 13:02:09 +00:00
|
|
|
print title % "Stderr"
|
2014-11-28 18:39:41 +00:00
|
|
|
print output.stderr
|
|
|
|
if output.timed_out:
|
|
|
|
print ">>> Test timed out after %ss." % runnable.timeout
|
2015-09-14 23:14:20 +00:00
|
|
|
if '--prof' in self.extra_flags:
|
2015-09-15 08:15:54 +00:00
|
|
|
os_prefix = {"linux": "linux", "macos": "mac"}.get(utils.GuessOS())
|
|
|
|
if os_prefix:
|
|
|
|
tick_tools = os.path.join(TOOLS_BASE, "%s-tick-processor" % os_prefix)
|
|
|
|
subprocess.check_call(tick_tools + " --only-summary", shell=True)
|
|
|
|
else: # pragma: no cover
|
2015-09-14 23:14:20 +00:00
|
|
|
print "Profiler option currently supported on Linux and Mac OS."
|
2014-11-28 18:39:41 +00:00
|
|
|
return output.stdout
|
|
|
|
|
|
|
|
|
2014-12-02 14:23:51 +00:00
|
|
|
class AndroidPlatform(Platform): # pragma: no cover
|
|
|
|
DEVICE_DIR = "/data/local/tmp/v8/"
|
|
|
|
|
2014-11-28 18:39:41 +00:00
|
|
|
def __init__(self, options):
|
2015-05-18 15:11:05 +00:00
|
|
|
super(AndroidPlatform, self).__init__(options)
|
2014-12-02 14:23:51 +00:00
|
|
|
LoadAndroidBuildTools(options.android_build_tools)
|
|
|
|
|
|
|
|
if not options.device:
|
|
|
|
# Detect attached device if not specified.
|
2015-07-27 14:56:54 +00:00
|
|
|
devices = adb_wrapper.AdbWrapper.Devices()
|
2014-12-02 14:23:51 +00:00
|
|
|
assert devices and len(devices) == 1, (
|
|
|
|
"None or multiple devices detected. Please specify the device on "
|
|
|
|
"the command-line with --device")
|
2015-07-27 14:56:54 +00:00
|
|
|
options.device = str(devices[0])
|
|
|
|
self.adb_wrapper = adb_wrapper.AdbWrapper(options.device)
|
|
|
|
self.device = device_utils.DeviceUtils(self.adb_wrapper)
|
2014-12-02 14:23:51 +00:00
|
|
|
|
|
|
|
def PreExecution(self):
|
|
|
|
perf = perf_control.PerfControl(self.device)
|
|
|
|
perf.SetHighPerfMode()
|
|
|
|
|
2014-12-03 15:34:53 +00:00
|
|
|
# Remember what we have already pushed to the device.
|
|
|
|
self.pushed = set()
|
|
|
|
|
2014-12-02 14:23:51 +00:00
|
|
|
def PostExecution(self):
|
|
|
|
perf = perf_control.PerfControl(self.device)
|
|
|
|
perf.SetDefaultPerfMode()
|
2014-12-08 09:10:38 +00:00
|
|
|
self.device.RunShellCommand(["rm", "-rf", AndroidPlatform.DEVICE_DIR])
|
2014-12-02 14:23:51 +00:00
|
|
|
|
2015-02-24 14:07:15 +00:00
|
|
|
def _PushFile(self, host_dir, file_name, target_rel=".",
|
|
|
|
skip_if_missing=False):
|
2014-12-02 14:23:51 +00:00
|
|
|
file_on_host = os.path.join(host_dir, file_name)
|
2014-12-23 12:40:32 +00:00
|
|
|
file_on_device_tmp = os.path.join(
|
|
|
|
AndroidPlatform.DEVICE_DIR, "_tmp_", file_name)
|
2014-12-04 10:37:36 +00:00
|
|
|
file_on_device = os.path.join(
|
|
|
|
AndroidPlatform.DEVICE_DIR, target_rel, file_name)
|
2014-12-31 11:16:00 +00:00
|
|
|
folder_on_device = os.path.dirname(file_on_device)
|
2014-12-03 15:34:53 +00:00
|
|
|
|
2015-02-24 14:07:15 +00:00
|
|
|
# Only attempt to push files that exist.
|
|
|
|
if not os.path.exists(file_on_host):
|
|
|
|
if not skip_if_missing:
|
|
|
|
logging.critical('Missing file on host: %s' % file_on_host)
|
|
|
|
return
|
|
|
|
|
2014-12-03 15:34:53 +00:00
|
|
|
# Only push files not yet pushed in one execution.
|
|
|
|
if file_on_host in self.pushed:
|
|
|
|
return
|
|
|
|
else:
|
|
|
|
self.pushed.add(file_on_host)
|
|
|
|
|
2014-12-23 12:40:32 +00:00
|
|
|
# Work-around for "text file busy" errors. Push the files to a temporary
|
2014-12-31 11:16:00 +00:00
|
|
|
# location and then copy them with a shell command.
|
2015-07-27 14:56:54 +00:00
|
|
|
output = self.adb_wrapper.Push(file_on_host, file_on_device_tmp)
|
2014-12-23 12:40:32 +00:00
|
|
|
# Success looks like this: "3035 KB/s (12512056 bytes in 4.025s)".
|
|
|
|
# Errors look like this: "failed to copy ... ".
|
|
|
|
if output and not re.search('^[0-9]', output.splitlines()[-1]):
|
|
|
|
logging.critical('PUSH FAILED: ' + output)
|
2015-07-27 14:56:54 +00:00
|
|
|
self.adb_wrapper.Shell("mkdir -p %s" % folder_on_device)
|
|
|
|
self.adb_wrapper.Shell("cp %s %s" % (file_on_device_tmp, file_on_device))
|
2014-12-02 14:23:51 +00:00
|
|
|
|
2015-07-10 13:02:09 +00:00
|
|
|
def _PushExecutable(self, shell_dir, target_dir, binary):
|
|
|
|
self._PushFile(shell_dir, binary, target_dir)
|
2015-02-24 14:07:15 +00:00
|
|
|
|
|
|
|
# Push external startup data. Backwards compatible for revisions where
|
|
|
|
# these files didn't exist.
|
2015-07-06 12:37:31 +00:00
|
|
|
self._PushFile(
|
2015-07-10 13:02:09 +00:00
|
|
|
shell_dir,
|
2015-07-06 12:37:31 +00:00
|
|
|
"natives_blob.bin",
|
2015-07-10 13:02:09 +00:00
|
|
|
target_dir,
|
2015-07-06 12:37:31 +00:00
|
|
|
skip_if_missing=True,
|
|
|
|
)
|
|
|
|
self._PushFile(
|
2015-07-10 13:02:09 +00:00
|
|
|
shell_dir,
|
2015-07-06 12:37:31 +00:00
|
|
|
"snapshot_blob.bin",
|
2015-07-10 13:02:09 +00:00
|
|
|
target_dir,
|
2015-07-06 12:37:31 +00:00
|
|
|
skip_if_missing=True,
|
|
|
|
)
|
2015-02-24 14:07:15 +00:00
|
|
|
|
2015-07-10 13:02:09 +00:00
|
|
|
def PreTests(self, node, path):
|
|
|
|
suite_dir = os.path.abspath(os.path.dirname(path))
|
|
|
|
if node.path:
|
|
|
|
bench_rel = os.path.normpath(os.path.join(*node.path))
|
|
|
|
bench_abs = os.path.join(suite_dir, bench_rel)
|
|
|
|
else:
|
|
|
|
bench_rel = "."
|
|
|
|
bench_abs = suite_dir
|
|
|
|
|
|
|
|
self._PushExecutable(self.shell_dir, "bin", node.binary)
|
|
|
|
if self.shell_dir_no_patch:
|
|
|
|
self._PushExecutable(
|
|
|
|
self.shell_dir_no_patch, "bin_no_patch", node.binary)
|
|
|
|
|
2015-07-08 12:31:29 +00:00
|
|
|
if isinstance(node, RunnableConfig):
|
2014-12-04 10:37:36 +00:00
|
|
|
self._PushFile(bench_abs, node.main, bench_rel)
|
2014-12-03 15:34:53 +00:00
|
|
|
for resource in node.resources:
|
2014-12-04 10:37:36 +00:00
|
|
|
self._PushFile(bench_abs, resource, bench_rel)
|
2014-12-02 14:23:51 +00:00
|
|
|
|
2015-07-10 13:02:09 +00:00
|
|
|
def _Run(self, runnable, count, no_patch=False):
|
|
|
|
suffix = ' - without patch' if no_patch else ''
|
|
|
|
target_dir = "bin_no_patch" if no_patch else "bin"
|
|
|
|
title = ">>> %%s (#%d)%s:" % ((count + 1), suffix)
|
2014-12-02 14:23:51 +00:00
|
|
|
cache = cache_control.CacheControl(self.device)
|
|
|
|
cache.DropRamCaches()
|
2015-07-06 12:37:31 +00:00
|
|
|
binary_on_device = os.path.join(
|
2015-07-10 13:02:09 +00:00
|
|
|
AndroidPlatform.DEVICE_DIR, target_dir, runnable.binary)
|
2015-05-13 10:00:43 +00:00
|
|
|
cmd = [binary_on_device] + runnable.GetCommandFlags(self.extra_flags)
|
2014-12-04 10:37:36 +00:00
|
|
|
|
|
|
|
# Relative path to benchmark directory.
|
|
|
|
if runnable.path:
|
|
|
|
bench_rel = os.path.normpath(os.path.join(*runnable.path))
|
|
|
|
else:
|
|
|
|
bench_rel = "."
|
|
|
|
|
2014-12-02 14:23:51 +00:00
|
|
|
try:
|
|
|
|
output = self.device.RunShellCommand(
|
|
|
|
cmd,
|
2014-12-04 10:37:36 +00:00
|
|
|
cwd=os.path.join(AndroidPlatform.DEVICE_DIR, bench_rel),
|
2014-12-02 14:23:51 +00:00
|
|
|
timeout=runnable.timeout,
|
|
|
|
retries=0,
|
|
|
|
)
|
|
|
|
stdout = "\n".join(output)
|
2015-07-10 13:02:09 +00:00
|
|
|
print title % "Stdout"
|
2014-12-02 14:23:51 +00:00
|
|
|
print stdout
|
|
|
|
except device_errors.CommandTimeoutError:
|
|
|
|
print ">>> Test timed out after %ss." % runnable.timeout
|
|
|
|
stdout = ""
|
|
|
|
return stdout
|
2014-11-28 18:39:41 +00:00
|
|
|
|
|
|
|
|
2014-09-02 09:17:26 +00:00
|
|
|
# TODO: Implement results_processor.
|
|
|
|
def Main(args):
|
2014-12-02 14:23:51 +00:00
|
|
|
logging.getLogger().setLevel(logging.INFO)
|
2014-09-02 09:17:26 +00:00
|
|
|
parser = optparse.OptionParser()
|
2014-11-28 18:39:41 +00:00
|
|
|
parser.add_option("--android-build-tools",
|
2015-06-29 15:07:15 +00:00
|
|
|
help="Path to chromium's build/android. Specifying this "
|
|
|
|
"option will run tests using android platform.")
|
2014-09-02 09:17:26 +00:00
|
|
|
parser.add_option("--arch",
|
|
|
|
help=("The architecture to run tests for, "
|
|
|
|
"'auto' or 'native' for auto-detect"),
|
|
|
|
default="x64")
|
|
|
|
parser.add_option("--buildbot",
|
|
|
|
help="Adapt to path structure used on buildbots",
|
|
|
|
default=False, action="store_true")
|
2014-11-28 18:39:41 +00:00
|
|
|
parser.add_option("--device",
|
|
|
|
help="The device ID to run Android tests on. If not given "
|
|
|
|
"it will be autodetected.")
|
2015-05-13 10:00:43 +00:00
|
|
|
parser.add_option("--extra-flags",
|
|
|
|
help="Additional flags to pass to the test executable",
|
|
|
|
default="")
|
2014-09-02 09:17:26 +00:00
|
|
|
parser.add_option("--json-test-results",
|
|
|
|
help="Path to a file for storing json results.")
|
2015-07-10 13:02:09 +00:00
|
|
|
parser.add_option("--json-test-results-no-patch",
|
|
|
|
help="Path to a file for storing json results from run "
|
|
|
|
"without patch.")
|
2014-09-02 09:17:26 +00:00
|
|
|
parser.add_option("--outdir", help="Base directory with compile output",
|
|
|
|
default="out")
|
2015-07-10 13:02:09 +00:00
|
|
|
parser.add_option("--outdir-no-patch",
|
|
|
|
help="Base directory with compile output without patch")
|
2014-09-02 09:17:26 +00:00
|
|
|
(options, args) = parser.parse_args(args)
|
|
|
|
|
|
|
|
if len(args) == 0: # pragma: no cover
|
|
|
|
parser.print_help()
|
|
|
|
return 1
|
|
|
|
|
|
|
|
if options.arch in ["auto", "native"]: # pragma: no cover
|
|
|
|
options.arch = ARCH_GUESS
|
|
|
|
|
|
|
|
if not options.arch in SUPPORTED_ARCHS: # pragma: no cover
|
|
|
|
print "Unknown architecture %s" % options.arch
|
|
|
|
return 1
|
|
|
|
|
2015-07-10 13:02:09 +00:00
|
|
|
if options.device and not options.android_build_tools: # pragma: no cover
|
2015-06-29 15:07:15 +00:00
|
|
|
print "Specifying a device requires Android build tools."
|
2014-11-28 18:39:41 +00:00
|
|
|
return 1
|
2014-09-02 09:17:26 +00:00
|
|
|
|
2015-07-10 13:02:09 +00:00
|
|
|
if (options.json_test_results_no_patch and
|
|
|
|
not options.outdir_no_patch): # pragma: no cover
|
|
|
|
print("For writing json test results without patch, an outdir without "
|
|
|
|
"patch must be specified.")
|
|
|
|
return 1
|
|
|
|
|
2014-12-02 14:23:51 +00:00
|
|
|
workspace = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
|
|
|
|
|
|
|
if options.buildbot:
|
2015-07-10 13:02:09 +00:00
|
|
|
build_config = "Release"
|
2014-12-02 14:23:51 +00:00
|
|
|
else:
|
2015-07-10 13:02:09 +00:00
|
|
|
build_config = "%s.release" % options.arch
|
|
|
|
|
|
|
|
options.shell_dir = os.path.join(workspace, options.outdir, build_config)
|
|
|
|
|
|
|
|
if options.outdir_no_patch:
|
|
|
|
options.shell_dir_no_patch = os.path.join(
|
|
|
|
workspace, options.outdir_no_patch, build_config)
|
|
|
|
else:
|
|
|
|
options.shell_dir_no_patch = None
|
2014-12-02 14:23:51 +00:00
|
|
|
|
2014-11-28 18:39:41 +00:00
|
|
|
platform = Platform.GetPlatform(options)
|
2014-09-02 09:17:26 +00:00
|
|
|
|
|
|
|
results = Results()
|
2015-07-10 13:02:09 +00:00
|
|
|
results_no_patch = Results()
|
2014-09-02 09:17:26 +00:00
|
|
|
for path in args:
|
|
|
|
path = os.path.abspath(path)
|
|
|
|
|
|
|
|
if not os.path.exists(path): # pragma: no cover
|
|
|
|
results.errors.append("Configuration file %s does not exist." % path)
|
|
|
|
continue
|
|
|
|
|
|
|
|
with open(path) as f:
|
|
|
|
suite = json.loads(f.read())
|
|
|
|
|
|
|
|
# If no name is given, default to the file name without .json.
|
|
|
|
suite.setdefault("name", os.path.splitext(os.path.basename(path))[0])
|
|
|
|
|
2014-12-03 15:34:53 +00:00
|
|
|
# Setup things common to one test suite.
|
|
|
|
platform.PreExecution()
|
|
|
|
|
|
|
|
# Build the graph/trace tree structure.
|
2015-07-08 12:31:29 +00:00
|
|
|
root = BuildGraphConfigs(suite, options.arch)
|
2014-12-03 15:34:53 +00:00
|
|
|
|
|
|
|
# Callback to be called on each node on traversal.
|
|
|
|
def NodeCB(node):
|
|
|
|
platform.PreTests(node, path)
|
|
|
|
|
|
|
|
# Traverse graph/trace tree and interate over all runnables.
|
|
|
|
for runnable in FlattenRunnables(root, NodeCB):
|
2014-09-02 09:17:26 +00:00
|
|
|
print ">>> Running suite: %s" % "/".join(runnable.graphs)
|
|
|
|
|
|
|
|
def Runner():
|
|
|
|
"""Output generator that reruns several times."""
|
|
|
|
for i in xrange(0, max(1, runnable.run_count)):
|
2014-09-15 13:00:32 +00:00
|
|
|
# TODO(machenbach): Allow timeout per arch like with run_count per
|
|
|
|
# arch.
|
2014-11-28 18:39:41 +00:00
|
|
|
yield platform.Run(runnable, i)
|
2014-09-02 09:17:26 +00:00
|
|
|
|
|
|
|
# Let runnable iterate over all runs and handle output.
|
2015-07-10 13:02:09 +00:00
|
|
|
result, result_no_patch = runnable.Run(
|
|
|
|
Runner, trybot=options.shell_dir_no_patch)
|
|
|
|
results += result
|
|
|
|
results_no_patch += result_no_patch
|
2014-12-03 15:34:53 +00:00
|
|
|
platform.PostExecution()
|
2014-12-02 14:23:51 +00:00
|
|
|
|
2014-09-02 09:17:26 +00:00
|
|
|
if options.json_test_results:
|
|
|
|
results.WriteToFile(options.json_test_results)
|
|
|
|
else: # pragma: no cover
|
|
|
|
print results
|
|
|
|
|
2015-07-10 13:02:09 +00:00
|
|
|
if options.json_test_results_no_patch:
|
|
|
|
results_no_patch.WriteToFile(options.json_test_results_no_patch)
|
|
|
|
else: # pragma: no cover
|
|
|
|
print results_no_patch
|
|
|
|
|
2014-09-02 09:17:26 +00:00
|
|
|
return min(1, len(results.errors))
|
|
|
|
|
|
|
|
if __name__ == "__main__": # pragma: no cover
|
|
|
|
sys.exit(Main(sys.argv[1:]))
|