Add nanobench stats scripts to Skia repo.
These are the scripts I've been homegrowing for measuring perf impact. I think we found them useful today as a way of sifting through the noise. BUG=skia: Review URL: https://codereview.chromium.org/703713002
This commit is contained in:
parent
19cd0f1813
commit
7ba39cb9a6
24
bin/ac
Executable file
24
bin/ac
Executable file
@ -0,0 +1,24 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
BRANCH=$(git branch | grep \* | cut -d" " -f 2)
|
||||||
|
CLEAN=${CLEAN-clean}
|
||||||
|
SAMPLES=100
|
||||||
|
|
||||||
|
if [ $BRANCH == $CLEAN ]; then
|
||||||
|
echo "Comparing $BRANCH to itself."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
git checkout $CLEAN
|
||||||
|
./gyp_skia >/dev/null
|
||||||
|
platform_tools/android/bin/android_ninja -t Release nanobench
|
||||||
|
platform_tools/android/bin/android_run_skia -t Release nanobench $@ --skps /data/local/tmp/skps -i /data/local/tmp/resources --samples $SAMPLES -v > $CLEAN.log
|
||||||
|
|
||||||
|
git checkout $BRANCH
|
||||||
|
./gyp_skia >/dev/null
|
||||||
|
platform_tools/android/bin/android_ninja -t Release nanobench
|
||||||
|
platform_tools/android/bin/android_run_skia -t Release nanobench $@ --skps /data/local/tmp/skps -i /data/local/tmp/resources --samples $SAMPLES -v > $BRANCH.log
|
||||||
|
|
||||||
|
compare $CLEAN.log $BRANCH.log
|
24
bin/c
Executable file
24
bin/c
Executable file
@ -0,0 +1,24 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
BRANCH=$(git branch | grep \* | cut -d" " -f 2)
|
||||||
|
CLEAN=${CLEAN-clean}
|
||||||
|
SAMPLES=100
|
||||||
|
|
||||||
|
if [ $BRANCH == $CLEAN ]; then
|
||||||
|
echo "Comparing $BRANCH to itself."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
git checkout $CLEAN
|
||||||
|
./gyp_skia >/dev/null
|
||||||
|
ninja -C out/Release nanobench
|
||||||
|
out/Release/nanobench $@ --samples $SAMPLES -v 2> $CLEAN.log
|
||||||
|
|
||||||
|
git checkout $BRANCH
|
||||||
|
./gyp_skia >/dev/null
|
||||||
|
ninja -C out/Release nanobench
|
||||||
|
out/Release/nanobench $@ --samples $SAMPLES -v 2> $BRANCH.log
|
||||||
|
|
||||||
|
compare $CLEAN.log $BRANCH.log
|
39
bin/compare
Executable file
39
bin/compare
Executable file
@ -0,0 +1,39 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
|
||||||
|
import sys
|
||||||
|
from scipy.stats import mannwhitneyu
|
||||||
|
|
||||||
|
SIGNIFICANCE_THRESHOLD = 0.0001
|
||||||
|
|
||||||
|
a,b = {},{}
|
||||||
|
for (path, d) in [(sys.argv[1], a), (sys.argv[2], b)]:
|
||||||
|
for line in open(path):
|
||||||
|
try:
|
||||||
|
tokens = line.split()
|
||||||
|
samples = tokens[:-1]
|
||||||
|
label = tokens[-1]
|
||||||
|
d[label] = map(float, samples)
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
|
common = set(a.keys()).intersection(b.keys())
|
||||||
|
|
||||||
|
ps = []
|
||||||
|
for key in common:
|
||||||
|
_, p = mannwhitneyu(a[key], b[key]) # Non-parametric t-test. Doesn't assume normal dist.
|
||||||
|
am, bm = min(a[key]), min(b[key])
|
||||||
|
ps.append((bm/am, p, key, am, bm))
|
||||||
|
ps.sort(reverse=True)
|
||||||
|
|
||||||
|
def humanize(ns):
|
||||||
|
for threshold, suffix in [(1e9, 's'), (1e6, 'ms'), (1e3, 'us'), (1e0, 'ns')]:
|
||||||
|
if ns > threshold:
|
||||||
|
return "%.3g%s" % (ns/threshold, suffix)
|
||||||
|
|
||||||
|
maxlen = max(map(len, common))
|
||||||
|
|
||||||
|
# We print only signficant changes in benchmark timing distribution.
|
||||||
|
bonferroni = SIGNIFICANCE_THRESHOLD / len(ps) # Adjust for the fact we've run multiple tests.
|
||||||
|
for ratio, p, key, am, bm in ps:
|
||||||
|
if p < bonferroni:
|
||||||
|
print '%*s\t%6s -> %6s\t%.2gx' % (maxlen, key, humanize(am), humanize(bm), ratio)
|
Loading…
Reference in New Issue
Block a user