skia2/bench/gen_bench_expectations.py
commit-bot@chromium.org 8cb46b963a Adds a blacklist of SKPs to exclude from bench expectations.
BUG=skia:2469
NOTRY=true
NOTREECHECKS=true
R=caryclark@google.com
TBR=caryclark@google.com

Author: bensong@google.com

Review URL: https://codereview.chromium.org/260053007

git-svn-id: http://skia.googlecode.com/svn/trunk@14422 2bbb7eff-a529-9590-31e7-b0007b416f81
2014-04-28 20:20:43 +00:00

124 lines
4.1 KiB
Python

#!/usr/bin/env python
# Copyright (c) 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
""" Generate bench_expectations file from a given set of bench data files. """
import argparse
import bench_util
import os
import re
import sys
# Parameters for calculating bench ranges.
RANGE_RATIO_UPPER = 1.2 # Ratio of range for upper bounds.
RANGE_RATIO_LOWER = 1.8 # Ratio of range for lower bounds.
ERR_RATIO = 0.08 # Further widens the range by the ratio of average value.
ERR_ABS = 0.5 # Adds an absolute error margin to cope with very small benches.
# List of bench configs to monitor. Ignore all other configs.
CONFIGS_TO_INCLUDE = ['simple_viewport_1000x1000',
'simple_viewport_1000x1000_gpu',
'simple_viewport_1000x1000_scalar_1.100000',
'simple_viewport_1000x1000_scalar_1.100000_gpu',
]
# List of flaky SKPs that should be excluded.
SKPS_TO_EXCLUDE = ['desk_chalkboard.skp',
]
def compute_ranges(benches):
"""Given a list of bench numbers, calculate the alert range.
Args:
benches: a list of float bench values.
Returns:
a list of float [lower_bound, upper_bound].
"""
minimum = min(benches)
maximum = max(benches)
diff = maximum - minimum
avg = sum(benches) / len(benches)
return [minimum - diff * RANGE_RATIO_LOWER - avg * ERR_RATIO - ERR_ABS,
maximum + diff * RANGE_RATIO_UPPER + avg * ERR_RATIO + ERR_ABS]
def create_expectations_dict(revision_data_points):
"""Convert list of bench data points into a dictionary of expectations data.
Args:
revision_data_points: a list of BenchDataPoint objects.
Returns:
a dictionary of this form:
keys = tuple of (config, bench) strings.
values = list of float [expected, lower_bound, upper_bound] for the key.
"""
bench_dict = {}
for point in revision_data_points:
if (point.time_type or # Not walltime which has time_type ''
not point.config in CONFIGS_TO_INCLUDE or
point.bench in SKPS_TO_EXCLUDE):
continue
key = (point.config, point.bench)
if key in bench_dict:
raise Exception('Duplicate bench entry: ' + str(key))
bench_dict[key] = [point.time] + compute_ranges(point.per_iter_time)
return bench_dict
def main():
"""Reads bench data points, then calculate and export expectations.
"""
parser = argparse.ArgumentParser()
parser.add_argument(
'-a', '--representation_alg', default='25th',
help='bench representation algorithm to use, see bench_util.py.')
parser.add_argument(
'-b', '--builder', required=True,
help='name of the builder whose bench ranges we are computing.')
parser.add_argument(
'-d', '--input_dir', required=True,
help='a directory containing bench data files.')
parser.add_argument(
'-o', '--output_file', required=True,
help='file path and name for storing the output bench expectations.')
parser.add_argument(
'-r', '--git_revision', required=True,
help='the git hash to indicate the revision of input data to use.')
args = parser.parse_args()
builder = args.builder
data_points = bench_util.parse_skp_bench_data(
args.input_dir, args.git_revision, args.representation_alg)
expectations_dict = create_expectations_dict(data_points)
out_lines = []
keys = expectations_dict.keys()
keys.sort()
for (config, bench) in keys:
(expected, lower_bound, upper_bound) = expectations_dict[(config, bench)]
out_lines.append('%(bench)s_%(config)s_,%(builder)s-%(representation)s,'
'%(expected)s,%(lower_bound)s,%(upper_bound)s' % {
'bench': bench,
'config': config,
'builder': builder,
'representation': args.representation_alg,
'expected': expected,
'lower_bound': lower_bound,
'upper_bound': upper_bound})
with open(args.output_file, 'w') as file_handle:
file_handle.write('\n'.join(out_lines))
if __name__ == "__main__":
main()