v8/tools/avg.py

#!/usr/bin/env python3
# Copyright 2018 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can
# be found in the LICENSE file.
"""
This script averages numbers output from another script. It is useful
to average over a benchmark that outputs one or more results of the form
  <key> <number> <unit>
key and unit are optional, but only one number per line is processed.

For example, if
  $ bch --allow-natives-syntax toNumber.js
outputs
  Number('undefined'):  155763
  (+'undefined'):  193050 Kps
  23736 Kps
then
  $ avg.py 10 bch --allow-natives-syntax toNumber.js
will output
  [10/10] (+'undefined')         : avg 192,240.40 stddev   6,486.24 (185,529.00 - 206,186.00)
  [10/10] Number('undefined')    : avg 156,990.10 stddev  16,327.56 (144,718.00 - 202,840.00) Kps
  [10/10] [default]              : avg  22,885.80 stddev   1,941.80 ( 17,584.00 -  24,266.00) Kps
"""

import argparse
import math
import re
import signal
import subprocess
import sys

PARSER = argparse.ArgumentParser(
    description="A script that averages numbers from another script's output",
    epilog="Example:\n\tavg.py 10 bash -c \"echo A: 100; echo B 120; sleep .1\""
)
PARSER.add_argument(
    'repetitions',
    type=int,
    help="number of times the command should be repeated")
PARSER.add_argument(
    'command',
    nargs=argparse.REMAINDER,
    help="command to run (no quotes needed)")
PARSER.add_argument(
    '--echo',
    '-e',
    action='store_true',
    default=False,
    help="set this flag to echo the command's output")

ARGS = vars(PARSER.parse_args())

if not ARGS['command']:
  print("No command provided.")
  exit(1)


class FieldWidth:

  def __init__(self, points=0, key=0, average=0, stddev=0, min_width=0, max_width=0):
    self.widths = dict(points=points, key=key, average=average, stddev=stddev,
                       min=min_width, max=max_width)

  def max_widths(self, other):
    self.widths = {k: max(v, other.widths[k]) for k, v in self.widths.items()}

  def __getattr__(self, key):
    return self.widths[key]


def fmtS(string, width=0):
  return "{0:<{1}}".format(string, width)


def fmtN(num, width=0):
  return "{0:>{1},.2f}".format(num, width)


def fmt(num):
  return "{0:>,.2f}".format(num)


def format_line(points, key, average, stddev, min_value, max_value,
                unit_string, widths):
  return "{:>{}};  {:<{}};  {:>{}};  {:>{}};  {:>{}};  {:>{}};  {}".format(
      points, widths.points,
      key, widths.key,
      average, widths.average,
      stddev, widths.stddev,
      min_value, widths.min,
      max_value, widths.max,
      unit_string)


def fmt_reps(msrmnt):
  rep_string = str(ARGS['repetitions'])
  return "[{0:>{1}}/{2}]".format(msrmnt.size(), len(rep_string), rep_string)


class Measurement:

  def __init__(self, key, unit):
    self.key = key
    self.unit = unit
    self.values = []
    self.average = 0
    self.count = 0
    self.M2 = 0
    self.min = float("inf")
    self.max = -float("inf")

  def addValue(self, value):
    try:
      num_value = float(value)
      self.values.append(num_value)
      self.min = min(self.min, num_value)
      self.max = max(self.max, num_value)
      self.count = self.count + 1
      delta = num_value - self.average
      self.average = self.average + delta / self.count
      delta2 = num_value - self.average
      self.M2 = self.M2 + delta * delta2
    except ValueError:
      print("Ignoring non-numeric value", value)

  def status(self, widths):
    return "{} {}: avg {} stddev {} ({} - {}) {}".format(
        fmt_reps(self),
        fmtS(self.key, widths.key), fmtN(self.average, widths.average),
        fmtN(self.stddev(), widths.stddev), fmtN(self.min, widths.min),
        fmtN(self.max, widths.max), fmtS(self.unit_string()))

  def result(self, widths):
    return format_line(self.size(), self.key, fmt(self.average),
                       fmt(self.stddev()), fmt(self.min),
                       fmt(self.max), self.unit_string(),
                       widths)

  def unit_string(self):
    if not self.unit:
      return ""
    return self.unit

  def variance(self):
    if self.count < 2:
      return float('NaN')
    return self.M2 / (self.count - 1)

  def stddev(self):
    return math.sqrt(self.variance())

  def size(self):
    return len(self.values)

  def widths(self):
    return FieldWidth(
        points=len("{}".format(self.size())) + 2,
        key=len(self.key),
        average=len(fmt(self.average)),
        stddev=len(fmt(self.stddev())),
        min_width=len(fmt(self.min)),
        max_width=len(fmt(self.max)))


def result_header(widths):
  return format_line("#/{}".format(ARGS['repetitions']),
                     "id", "avg", "stddev", "min", "max", "unit", widths)


class Measurements:

  def __init__(self):
    self.all = {}
    self.default_key = '[default]'
    self.max_widths = FieldWidth(
        points=len("{}".format(ARGS['repetitions'])) + 2,
        key=len("id"),
        average=len("avg"),
        stddev=len("stddev"),
        min_width=len("min"),
        max_width=len("max"))
    self.last_status_len = 0

  def record(self, key, value, unit):
    if not key:
      key = self.default_key
    if key not in self.all:
      self.all[key] = Measurement(key, unit)
    self.all[key].addValue(value)
    self.max_widths.max_widths(self.all[key].widths())

  def any(self):
    if self.all:
      return next(iter(self.all.values()))
    return None

  def print_results(self):
    print("{:<{}}".format("", self.last_status_len), end="\r")
    print(result_header(self.max_widths), sep=" ")
    for key in sorted(self.all):
      print(self.all[key].result(self.max_widths), sep=" ")

  def print_status(self):
    status = "No results found. Check format?"
    measurement = MEASUREMENTS.any()
    if measurement:
      status = measurement.status(MEASUREMENTS.max_widths)
    print("{:<{}}".format(status, self.last_status_len), end="\r")
    self.last_status_len = len(status)


MEASUREMENTS = Measurements()


def signal_handler(signum, frame):
  print("", end="\r")
  MEASUREMENTS.print_results()
  sys.exit(0)


signal.signal(signal.SIGINT, signal_handler)

SCORE_REGEX = (r'\A((console.timeEnd: )?'
               r'(?P<key>[^\s:,]+)[,:]?)?'
               r'(^\s*|\s+)'
               r'(?P<value>[0-9]+(.[0-9]+)?)'
               r'\ ?(?P<unit>[^\d\W]\w*)?[.\s]*\Z')

for x in range(0, ARGS['repetitions']):
  proc = subprocess.Popen(ARGS['command'], stdout=subprocess.PIPE)
  for line in proc.stdout:
    if ARGS['echo']:
      print(line.decode(), end="")
    for m in re.finditer(SCORE_REGEX, line.decode()):
      MEASUREMENTS.record(m.group('key'), m.group('value'), m.group('unit'))
  proc.wait()
  if proc.returncode != 0:
    print("Child exited with status %d" % proc.returncode)
    break

  MEASUREMENTS.print_status()

# Print final results
MEASUREMENTS.print_results()