[test] Implement results processor for perf runner.

This adds the possibility to specify a python script for post-processing stdout.

This also adds some system tests for testing the new feature.

NOTRY=true

Change-Id: I0383afb3e23513629508feeb639ed2dfce56b54a
Reviewed-on: https://chromium-review.googlesource.com/443449
Reviewed-by: Clemens Hammacher <clemensh@chromium.org>
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#43257}

tools/run_perf.py

@@ -42,12 +42,11 @@ A suite's results_regexp is expected to have one string place holder
 defaults.
 
 A suite's results_processor may point to an optional python script. If
-specified, it is called after running the tests like this (with a path
-relatve to the suite level's path):
-<results_processor file> <same flags as for d8> <suite level name> <output>
+specified, it is called after running the tests (with a path relative to the
+suite level's path). It is expected to read the measurement's output text
+on stdin and print the processed output to stdout.
 
-The <output> is a temporary file containing d8 output. The results_regexp will
-be applied to the output of this script.
+The results_regexp will be applied to the processed output.
 
 A suite without "tests" is considered a performance test itself.
 
@@ -238,6 +237,25 @@ def Unzip(iterable):
   return lambda: iter(left), lambda: iter(right)
 
 
+def RunResultsProcessor(results_processor, stdout, count):
+  # Dummy pass through for null-runs.
+  if stdout is None:
+    return None
+
+  # We assume the results processor is relative to the suite.
+  assert os.path.exists(results_processor)
+  p = subprocess.Popen(
+      [sys.executable, results_processor],
+      stdin=subprocess.PIPE,
+      stdout=subprocess.PIPE,
+      stderr=subprocess.PIPE,
+  )
+  result, _ = p.communicate(input=stdout)
+  print ">>> Processed stdout (#%d):" % count
+  print result
+  return result
+
+
 def AccumulateResults(
     graph_names, trace_configs, iter_output, trybot, no_patch, calc_total):
   """Iterates over the output of multiple benchmark reruns and accumulates
@@ -361,6 +379,7 @@ class DefaultSentinel(Node):
     self.flags = []
     self.test_flags = []
     self.resources = []
+    self.results_processor = None
     self.results_regexp = None
     self.stddev_regexp = None
     self.units = "score"
@@ -399,6 +418,8 @@ class GraphConfig(Node):
     self.timeout = suite.get("timeout_%s" % arch, self.timeout)
     self.units = suite.get("units", parent.units)
     self.total = suite.get("total", parent.total)
+    self.results_processor = suite.get(
+        "results_processor", parent.results_processor)
 
     # A regular expression for results. If the parent graph provides a
     # regexp and the current suite has none, a string place holder for the
@@ -445,6 +466,15 @@ class RunnableConfig(GraphConfig):
   def main(self):
     return self._suite.get("main", "")
 
+  def PostProcess(self, stdouts_iter):
+    if self.results_processor:
+      def it():
+        for i, stdout in enumerate(stdouts_iter()):
+          yield RunResultsProcessor(self.results_processor, stdout, i + 1)
+      return it
+    else:
+      return stdouts_iter
+
   def ChangeCWD(self, suite_path):
     """Changes the cwd to to path defined in the current graph.
 
@@ -462,6 +492,8 @@ class RunnableConfig(GraphConfig):
     # TODO(machenbach): This requires +.exe if run on windows.
     extra_flags = extra_flags or []
     cmd = [os.path.join(shell_dir, self.binary)]
+    if self.binary.endswith(".py"):
+      cmd = [sys.executable] + cmd
     if self.binary != 'd8' and '--prof' in extra_flags:
       print "Profiler supported only on a benchmark run with d8"
     return cmd + self.GetCommandFlags(extra_flags=extra_flags)
@@ -473,7 +505,7 @@ class RunnableConfig(GraphConfig):
         AccumulateResults(
             self.graphs,
             self._children,
-            iter_output=stdout_with_patch,
+            iter_output=self.PostProcess(stdout_with_patch),
             trybot=trybot,
             no_patch=False,
             calc_total=self.total,
@@ -481,7 +513,7 @@ class RunnableConfig(GraphConfig):
         AccumulateResults(
             self.graphs,
             self._children,
-            iter_output=stdout_no_patch,
+            iter_output=self.PostProcess(stdout_no_patch),
             trybot=trybot,
             no_patch=True,
             calc_total=self.total,
@@ -911,7 +943,6 @@ class CustomMachineConfiguration:
       raise Exception("Could not set CPU governor. Present value is %s"
                       % cur_value )
 
-# TODO: Implement results_processor.
 def Main(args):
   logging.getLogger().setLevel(logging.INFO)
   parser = optparse.OptionParser()

tools/unittests/run_perf_test.py

@@ -19,6 +19,10 @@ import unittest
 # Requires python-coverage and python-mock. Native python coverage
 # version >= 3.7.1 should be installed to get the best speed.
 
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+RUN_PERF = os.path.join(BASE_DIR, 'run_perf.py')
+TEST_DATA = os.path.join(BASE_DIR, 'unittests', 'testdata')
+
 TEST_WORKSPACE = path.join(tempfile.gettempdir(), "test-v8-run-perf")
 
 V8_JSON = {
@@ -473,3 +477,71 @@ class PerfTest(unittest.TestCase):
     l, r = run_perf.Unzip(Gen())
     self.assertEquals([1, 2, 3], list(l()))
     self.assertEquals([2, 3, 4], list(r()))
+
+  #############################################################################
+  ### System tests
+
+  def _RunPerf(self, mocked_d8, test_json):
+    output_json = path.join(TEST_WORKSPACE, "output.json")
+    args = [
+      sys.executable, RUN_PERF,
+      "--binary-override-path", os.path.join(TEST_DATA, mocked_d8),
+      "--json-test-results", output_json,
+      os.path.join(TEST_DATA, test_json),
+    ]
+    subprocess.check_output(args)
+    return self._LoadResults(output_json)
+
+  def testNormal(self):
+    results = self._RunPerf("d8_mocked1.py", "test1.json")
+    self.assertEquals([], results['errors'])
+    self.assertEquals([
+      {
+        'units': 'score',
+        'graphs': ['test1', 'Richards'],
+        'results': [u'1.2', u'1.2'],
+        'stddev': '',
+      },
+      {
+        'units': 'score',
+        'graphs': ['test1', 'DeltaBlue'],
+        'results': [u'2.1', u'2.1'],
+        'stddev': '',
+      },
+    ], results['traces'])
+
+  def testResultsProcessor(self):
+    results = self._RunPerf("d8_mocked2.py", "test2.json")
+    self.assertEquals([], results['errors'])
+    self.assertEquals([
+      {
+        'units': 'score',
+        'graphs': ['test2', 'Richards'],
+        'results': [u'1.2', u'1.2'],
+        'stddev': '',
+      },
+      {
+        'units': 'score',
+        'graphs': ['test2', 'DeltaBlue'],
+        'results': [u'2.1', u'2.1'],
+        'stddev': '',
+      },
+    ], results['traces'])
+
+  def testResultsProcessorNested(self):
+    results = self._RunPerf("d8_mocked2.py", "test3.json")
+    self.assertEquals([], results['errors'])
+    self.assertEquals([
+      {
+        'units': 'score',
+        'graphs': ['test3', 'Octane', 'Richards'],
+        'results': [u'1.2'],
+        'stddev': '',
+      },
+      {
+        'units': 'score',
+        'graphs': ['test3', 'Octane', 'DeltaBlue'],
+        'results': [u'2.1'],
+        'stddev': '',
+      },
+    ], results['traces'])

tools/unittests/testdata/d8_mocked1.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+# Copyright 2017 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+print 'Richards: 1.2'
+print 'DeltaBlue: 2.1'

tools/unittests/testdata/d8_mocked2.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# Copyright 2017 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+print 'Richards1: 1'
+print 'DeltaBlue1: 1'
+print 'Richards2: 0.2'
+print 'DeltaBlue2: 1.0'
+print 'DeltaBlue3: 0.1'

tools/unittests/testdata/results_processor.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+# Copyright 2017 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+Fake results processor for testing that just sums some things up.
+"""
+
+import fileinput
+import re
+
+richards = 0.0
+deltablue = 0.0
+
+for line in fileinput.input():
+  match = re.match(r'^Richards\d: (.*)$', line)
+  if match:
+    richards += float(match.group(1))
+  match = re.match(r'^DeltaBlue\d: (.*)$', line)
+  if match:
+    deltablue += float(match.group(1))
+
+print 'Richards: %f' % richards
+print 'DeltaBlue: %f' % deltablue

tools/unittests/testdata/test1.json

@@ -0,0 +1,11 @@
+{
+  "path": ["."],
+  "flags": [],
+  "main": "run.js",
+  "run_count": 2,
+  "results_regexp": "^%s: (.+)$",
+  "tests": [
+    {"name": "Richards"},
+    {"name": "DeltaBlue"}
+  ]
+}

tools/unittests/testdata/test2.json

@@ -0,0 +1,12 @@
+{
+  "path": ["."],
+  "flags": [],
+  "main": "run.js",
+  "run_count": 2,
+  "results_processor": "results_processor.py",
+  "results_regexp": "^%s: (.+)$",
+  "tests": [
+    {"name": "Richards"},
+    {"name": "DeltaBlue"}
+  ]
+}

tools/unittests/testdata/test3.json

@@ -0,0 +1,16 @@
+{
+  "path": ["."],
+  "flags": [],
+  "run_count": 1,
+  "results_processor": "results_processor.py",
+  "tests": [{
+    "path": ["."],
+    "name": "Octane",
+    "main": "run.js",
+    "results_regexp": "^%s: (.+)$",
+    "tests": [
+      {"name": "Richards"},
+      {"name": "DeltaBlue"}
+    ]
+  }]
+}