AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[canvaskit] Add benchmarks on SKPs

Browse Source 
For each skp in the corpus, we start a fresh instance of
Chromium (via puppeteer), draw the skp and measure that time.
This process is repeated a fixed amount of repetitions
and the median, the average, and the std deviation is reported
to perf (as well as the individual datapoints as an FYI).

Importantly (and something we'll need to change about
SkottieFrames), we measure the average time between frames
after unlocking the framerate. This ensures we account for
the time needed by the GPU to actually draw (flush() returns
after the GPU has all the instructions, but not necessarily
has been able to draw).

This implementation is very similar to the SkottieFrames
code; a notable deviation is the repetitions are handled
outside of the html, i.e. a new chrome window per run.

I explored using content_shell, but noticed that requires
building Chromium, which our infrastructure is not set up
to do well.

Change-Id: I14fdbdc951604d3fdf06e81a4be7e614d0e53c03
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/295079
Commit-Queue: Kevin Lubick <kjlubick@google.com>
Reviewed-by: Nathaniel Nifong <nifong@google.com>
Reviewed-by: Eric Boren <borenet@google.com>
This commit is contained in:
Kevin Lubick 2020-06-10 15:10:16 -04:00 committed by Skia Commit-Bot
parent e8d3ccadfe
commit 51891392d8
10 changed files with 1126 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
infra/bots/gen_tasks_logic/gen_tasks_logic.go
View File

@ -443,7 +443,7 @@ func (b *jobBuilder) deriveCompileTaskName() string {
"ReleaseAndAbandonGpuContext", "CCPR", "FSAA", "FAAA", "FDAA", "NativeFonts", "GDI",
"NoGPUThreads", "ProcDump", "DDL1", "DDL3", "T8888", "DDLTotal", "DDLRecord", "9x9",
"BonusConfigs", "SkottieTracing", "SkottieWASM", "GpuTess", "NonNVPR", "Mskp",
"Docker", "PDF", "SkVM", "Puppeteer", "SkottieFrames"}
"Docker", "PDF", "SkVM", "Puppeteer", "SkottieFrames", "RenderSKP"}
keep := make([]string, 0, len(ec))
for _, part := range ec {
if !In(part, ignore) {
@ -1401,41 +1401,64 @@ func (b *jobBuilder) puppeteer() {
compileTaskName := b.compile()
b.addTask(b.Name, func(b *taskBuilder) {
b.defaultSwarmDimensions()
b.isolate("perf_puppeteer.isolate")
b.cmd(
"./perf_puppeteer_skottie_frames",
"--project_id", "skia-swarming-bots",
"--git_hash", specs.PLACEHOLDER_REVISION,
"--task_id", specs.PLACEHOLDER_TASK_ID,
"--task_name", b.Name,
"--canvaskit_bin_path", "./build",
"--lotties_path", "./lotties_with_assets",
"--node_bin_path", "./node/node/bin",
"--benchmark_path", "./tools/perf-canvaskit-puppeteer",
"--output_path", OUTPUT_PERF,
"--os_trace", b.parts["os"],
"--model_trace", b.parts["model"],
"--cpu_or_gpu_trace", b.parts["cpu_or_gpu"],
"--cpu_or_gpu_value_trace", b.parts["cpu_or_gpu_value"],
"--alsologtostderr",
)
b.serviceAccount(b.cfg.ServiceAccountCompile)
// This CIPD package was made by hand with the following invocation:
// cipd create -name skia/internal/lotties_with_assets -in ./lotties/ -tag version:0
// cipd acl-edit skia/internal/lotties_with_assets -reader group:project-skia-external-task-accounts
// cipd acl-edit skia/internal/lotties_with_assets -reader user:pool-skia@chromium-swarm.iam.gserviceaccount.com
// Where lotties is a hand-selected set of lottie animations and (optionally) assets used in
// them (e.g. fonts, images).
b.cipd(&specs.CipdPackage{
Name: "skia/internal/lotties_with_assets",
Path: "lotties_with_assets",
Version: "version:0",
})
b.usesNode()
b.cipd(CIPD_PKG_LUCI_AUTH)
b.dep(b.buildTaskDrivers(), compileTaskName)
b.output(OUTPUT_PERF)
b.timeout(20 * time.Minute)
b.isolate("perf_puppeteer.isolate")
b.serviceAccount(b.cfg.ServiceAccountCompile)
if b.extraConfig("SkottieFrames") {
b.cmd(
"./perf_puppeteer_skottie_frames",
"--project_id", "skia-swarming-bots",
"--git_hash", specs.PLACEHOLDER_REVISION,
"--task_id", specs.PLACEHOLDER_TASK_ID,
"--task_name", b.Name,
"--canvaskit_bin_path", "./build",
"--lotties_path", "./lotties_with_assets",
"--node_bin_path", "./node/node/bin",
"--benchmark_path", "./tools/perf-canvaskit-puppeteer",
"--output_path", OUTPUT_PERF,
"--os_trace", b.parts["os"],
"--model_trace", b.parts["model"],
"--cpu_or_gpu_trace", b.parts["cpu_or_gpu"],
"--cpu_or_gpu_value_trace", b.parts["cpu_or_gpu_value"],
"--alsologtostderr",
)
// This CIPD package was made by hand with the following invocation:
// cipd create -name skia/internal/lotties_with_assets -in ./lotties/ -tag version:0
// cipd acl-edit skia/internal/lotties_with_assets -reader group:project-skia-external-task-accounts
// cipd acl-edit skia/internal/lotties_with_assets -reader user:pool-skia@chromium-swarm.iam.gserviceaccount.com
// Where lotties is a hand-selected set of lottie animations and (optionally) assets used in
// them (e.g. fonts, images).
b.cipd(&specs.CipdPackage{
Name: "skia/internal/lotties_with_assets",
Path: "lotties_with_assets",
Version: "version:0",
})
} else if b.extraConfig("RenderSKP") {
b.cmd(
"./perf_puppeteer_render_skps",
"--project_id", "skia-swarming-bots",
"--git_hash", specs.PLACEHOLDER_REVISION,
"--task_id", specs.PLACEHOLDER_TASK_ID,
"--task_name", b.Name,
"--canvaskit_bin_path", "./build",
"--skps_path", "./skp",
"--node_bin_path", "./node/node/bin",
"--benchmark_path", "./tools/perf-canvaskit-puppeteer",
"--output_path", OUTPUT_PERF,
"--os_trace", b.parts["os"],
"--model_trace", b.parts["model"],
"--cpu_or_gpu_trace", b.parts["cpu_or_gpu"],
"--cpu_or_gpu_value_trace", b.parts["cpu_or_gpu_value"],
"--alsologtostderr",
)
b.asset("skp")
}
})
// Upload results to Perf after.

2
infra/bots/jobs.json
View File

@ -230,6 +230,7 @@
"Perf-Debian10-EMCC-GCE-CPU-AVX2-wasm-Release-All-PathKit",
"Perf-Debian10-EMCC-GCE-CPU-AVX2-wasm-Release-All-SkottieWASM",
"Perf-Debian10-EMCC-GCE-CPU-AVX2-wasm-Release-All-Puppeteer_SkottieFrames",
"Perf-Debian10-EMCC-GCE-CPU-AVX2-wasm-Release-All-Puppeteer_RenderSKP",
"Perf-Debian10-EMCC-GCE-GPU-AVX2-wasm-Release-All-CanvasKit",
"Perf-Debian10-none-GCE-CPU-AVX2-x86_64-Release-All-LottieWeb",
"Perf-Mac10.13-Clang-MacBook10.1-GPU-IntelHD615-x86_64-Release-All",
@ -252,6 +253,7 @@
"Perf-Ubuntu18-Clang-Golo-GPU-QuadroP400-x86_64-Release-All-Vulkan",
"Perf-Ubuntu18-EMCC-Golo-GPU-QuadroP400-wasm-Release-All-SkottieWASM",
"Perf-Ubuntu18-EMCC-Golo-GPU-QuadroP400-wasm-Release-All-Puppeteer_SkottieFrames",
"Perf-Ubuntu18-EMCC-Golo-GPU-QuadroP400-wasm-Release-All-Puppeteer_RenderSKP",
"Perf-Ubuntu18-none-Golo-GPU-QuadroP400-x86_64-Release-All-LottieWeb",
"Perf-Win10-Clang-AlphaR2-GPU-RadeonR9M470X-x86_64-Release-All",
"Perf-Win10-Clang-AlphaR2-GPU-RadeonR9M470X-x86_64-Release-All-ANGLE",

349
infra/bots/task_drivers/perf_puppeteer_render_skps/perf_puppeteer_render_skps.go Normal file
View File

@ -0,0 +1,349 @@
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// This executable is meant to be a general way to gather perf data using puppeteer. The logic
// (e.g. what bench to run, how to process that particular output) is selected using the ExtraConfig
// part of the task name.
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"math"
"os"
"path/filepath"
"sort"
"strings"
"go.skia.org/infra/go/exec"
"go.skia.org/infra/go/skerr"
"go.skia.org/infra/go/sklog"
"go.skia.org/infra/task_driver/go/lib/os_steps"
"go.skia.org/infra/task_driver/go/td"
)
func main() {
var (
// Required properties for this task.
projectID = flag.String("project_id", "", "ID of the Google Cloud project.")
taskName = flag.String("task_name", "", "Name of the task.")
benchmarkPath = flag.String("benchmark_path", "", "Path to location of the benchmark files (e.g. //tools/perf-puppeteer).")
outputPath = flag.String("output_path", "", "Perf Output will be produced here")
gitHash = flag.String("git_hash", "", "Git hash this data corresponds to")
taskID = flag.String("task_id", "", "task id this data was generated on")
nodeBinPath = flag.String("node_bin_path", "", "Path to the node bin directory (should have npm also). This directory *must* be on the PATH when this executable is called, otherwise, the wrong node or npm version may be found (e.g. the one on the system), even if we are explicitly calling npm with the absolute path.")
// These flags feed into the perf trace keys associated with the output data.
osTrace = flag.String("os_trace", "", "OS this is running on.")
modelTrace = flag.String("model_trace", "", "Description of host machine.")
cpuOrGPUTrace = flag.String("cpu_or_gpu_trace", "", "If this is a CPU or GPU configuration.")
cpuOrGPUValueTrace = flag.String("cpu_or_gpu_value_trace", "", "The hardware of this CPU/GPU")
// Flags that may be required for certain configs
canvaskitBinPath = flag.String("canvaskit_bin_path", "", "The location of a canvaskit.js and canvaskit.wasm")
skpsPath = flag.String("skps_path", "", "Path to location of skps.")
// Debugging flags.
local = flag.Bool("local", false, "True if running locally (as opposed to on the bots)")
outputSteps = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.")
)
// Setup.
ctx := td.StartRun(projectID, taskID, taskName, outputSteps, local)
defer td.EndRun(ctx)
keys := map[string]string{
"os": *osTrace,
"model": *modelTrace,
perfKeyCpuOrGPU: *cpuOrGPUTrace,
"cpu_or_gpu_value": *cpuOrGPUValueTrace,
}
outputWithoutResults, err := makePerfObj(*gitHash, *taskID, os.Getenv("SWARMING_BOT_ID"), keys)
if err != nil {
td.Fatal(ctx, skerr.Wrap(err))
}
// Absolute paths work more consistently than relative paths.
nodeBinAbsPath := getAbsoluteOfRequiredFlag(ctx, *nodeBinPath, "node_bin_path")
benchmarkAbsPath := getAbsoluteOfRequiredFlag(ctx, *benchmarkPath, "benchmark_path")
canvaskitBinAbsPath := getAbsoluteOfRequiredFlag(ctx, *canvaskitBinPath, "canvaskit_bin_path")
skpsAbsPath := getAbsoluteOfRequiredFlag(ctx, *skpsPath, "skps_path")
outputAbsPath := getAbsoluteOfRequiredFlag(ctx, *outputPath, "output_path")
if err := setup(ctx, benchmarkAbsPath, nodeBinAbsPath); err != nil {
td.Fatal(ctx, skerr.Wrap(err))
}
if err := benchSKPs(ctx, outputWithoutResults, benchmarkAbsPath, canvaskitBinAbsPath, skpsAbsPath, nodeBinAbsPath); err != nil {
td.Fatal(ctx, skerr.Wrap(err))
}
// outputFile name should be unique between tasks, so as to avoid having duplicate name files
// uploaded to GCS.
outputFile := filepath.Join(outputAbsPath, fmt.Sprintf("perf-%s.json", *taskID))
if err := processSKPData(ctx, outputWithoutResults, benchmarkAbsPath, outputFile); err != nil {
td.Fatal(ctx, skerr.Wrap(err))
}
}
func getAbsoluteOfRequiredFlag(ctx context.Context, nonEmptyPath, flag string) string {
if nonEmptyPath == "" {
td.Fatalf(ctx, "--%s must be specified", flag)
}
absPath, err := filepath.Abs(nonEmptyPath)
if err != nil {
td.Fatal(ctx, skerr.Wrap(err))
}
return absPath
}
const perfKeyCpuOrGPU = "cpu_or_gpu"
func makePerfObj(gitHash, taskID, machineID string, keys map[string]string) (perfJSONFormat, error) {
rv := perfJSONFormat{}
if gitHash == "" {
return rv, skerr.Fmt("Must provide --git_hash")
}
if taskID == "" {
return rv, skerr.Fmt("Must provide --task_id")
}
rv.GitHash = gitHash
rv.SwarmingTaskID = taskID
rv.SwarmingMachineID = machineID
rv.Key = keys
rv.Key["arch"] = "wasm"
rv.Key["browser"] = "Chromium"
rv.Key["configuration"] = "Release"
rv.Key["extra_config"] = "RenderSKP"
rv.Key["binary"] = "CanvasKit"
rv.Results = map[string]map[string]perfResult{}
return rv, nil
}
func setup(ctx context.Context, benchmarkPath, nodeBinPath string) error {
ctx = td.StartStep(ctx, td.Props("setup").Infra())
defer td.EndStep(ctx)
if _, err := exec.RunCwd(ctx, benchmarkPath, filepath.Join(nodeBinPath, "npm"), "ci"); err != nil {
return td.FailStep(ctx, skerr.Wrap(err))
}
// This is very important to make sure chrome is not running because we want to run chrome
// with an unlocked framerate (see --disable-frame-rate-limit and --disable-gpu-vsync in
// perf-canvaskit-with-puppeteer.js) and that won't happen if there is already an existing
// chrome instance running when we try to run puppeteer. killall will return an error (e.g.
// a non-zero error code) if there isn't already a chrome instance running. We can safely
// ignore that error as we never expect there to be chrome running.
_, _ = exec.RunSimple(ctx, "killall chrome")
if err := os.MkdirAll(filepath.Join(benchmarkPath, "out"), 0777); err != nil {
return td.FailStep(ctx, skerr.Wrap(err))
}
return nil
}
// benchSKPs serves skps from a folder and runs the RenderSKPs benchmark on each of them
// individually. The benchmark is run N times to reduce the noise of the resulting data.
// The output for each will be a JSON file in $benchmarkPath/out/ corresponding to the skp name
// and the iteration it was.
func benchSKPs(ctx context.Context, perf perfJSONFormat, benchmarkPath, canvaskitBinPath, skpsPath, nodeBinPath string) error {
ctx = td.StartStep(ctx, td.Props("perf skps in "+skpsPath))
defer td.EndStep(ctx)
// We expect the skpsPath to a directory with skp files in it.
var skpFiles []string
err := td.Do(ctx, td.Props("locate skpfiles"), func(ctx context.Context) error {
return filepath.Walk(skpsPath, func(path string, info os.FileInfo, _ error) error {
if path == skpsPath {
return nil
}
if info.IsDir() {
return filepath.SkipDir
}
skpFiles = append(skpFiles, path)
return nil
})
})
if err != nil {
return td.FailStep(ctx, skerr.Wrap(err))
}
sklog.Infof("Identified %d skp files to benchmark", len(skpFiles))
for _, skp := range skpFiles {
name := filepath.Base(skp)
err = td.Do(ctx, td.Props(fmt.Sprintf("Benchmark %s", name)), func(ctx context.Context) error {
// See comment in setup about why we specify the absolute path for node.
args := []string{filepath.Join(nodeBinPath, "node"),
"perf-canvaskit-with-puppeteer",
"--bench_html", "render-skp.html",
"--canvaskit_js", filepath.Join(canvaskitBinPath, "canvaskit.js"),
"--canvaskit_wasm", filepath.Join(canvaskitBinPath, "canvaskit.wasm"),
"--input_skp", skp,
"--output", filepath.Join(benchmarkPath, "out", name+".json"),
}
if perf.Key[perfKeyCpuOrGPU] != "CPU" {
args = append(args, "--use_gpu")
}
_, err := exec.RunCwd(ctx, benchmarkPath, args...)
if err != nil {
return skerr.Wrap(err)
}
return nil
})
if err != nil {
return td.FailStep(ctx, skerr.Wrap(err))
}
}
return nil
}
// TODO(kjlubick,jcgregorio) Could this code directly refer to the struct in Perf?
type perfJSONFormat struct {
GitHash string `json:"gitHash"`
SwarmingTaskID string `json:"swarming_task_id"`
SwarmingMachineID string `json:"swarming_machine_id"`
Key map[string]string `json:"key"`
// Maps bench name -> "config" -> result key -> value
Results map[string]map[string]perfResult `json:"results"`
}
type perfResult map[string]float32
// processSKPData looks at the result of benchSKPs, computes summary data on
// those files and adds them as Results into the provided perf object. The perf object is then
// written in JSON format to outputPath.
func processSKPData(ctx context.Context, perf perfJSONFormat, benchmarkPath, outputFilePath string) error {
perfJSONPath := filepath.Join(benchmarkPath, "out")
ctx = td.StartStep(ctx, td.Props("process perf output "+perfJSONPath))
defer td.EndStep(ctx)
var jsonInputs []string
err := td.Do(ctx, td.Props("locate input JSON files"), func(ctx context.Context) error {
return filepath.Walk(perfJSONPath, func(path string, info os.FileInfo, _ error) error {
if strings.HasSuffix(path, ".json") {
jsonInputs = append(jsonInputs, path)
return nil
}
return nil
})
})
if err != nil {
return td.FailStep(ctx, skerr.Wrap(err))
}
sklog.Infof("Identified %d JSON inputs to process", len(jsonInputs))
for _, skp := range jsonInputs {
err = td.Do(ctx, td.Props("Process "+skp), func(ctx context.Context) error {
name := strings.TrimSuffix(filepath.Base(skp), ".json")
config := "software"
if perf.Key[perfKeyCpuOrGPU] != "CPU" {
config = "webgl2"
}
b, err := os_steps.ReadFile(ctx, skp)
if err != nil {
return skerr.Wrap(err)
}
metrics, err := parseSKPData(b)
if err != nil {
return skerr.Wrap(err)
}
perf.Results[name] = map[string]perfResult{
config: metrics,
}
return nil
})
if err != nil {
return td.FailStep(ctx, skerr.Wrap(err))
}
}
err = td.Do(ctx, td.Props("Writing perf JSON file to "+outputFilePath), func(ctx context.Context) error {
if err := os.MkdirAll(filepath.Dir(outputFilePath), 0777); err != nil {
return skerr.Wrap(err)
}
b, err := json.MarshalIndent(perf, "", " ")
if err != nil {
return skerr.Wrap(err)
}
if err = ioutil.WriteFile(outputFilePath, b, 0666); err != nil {
return skerr.Wrap(err)
}
return nil
})
if err != nil {
return td.FailStep(ctx, skerr.Wrap(err))
}
return nil
}
type skpPerfData struct {
WithoutFlushMS []float32 `json:"without_flush_ms"`
WithFlushMS []float32 `json:"with_flush_ms"`
TotalFrameMS []float32 `json:"total_frame_ms"`
SKPLoadMS float32 `json:"skp_load_ms"`
}
func parseSKPData(b []byte) (perfResult, error) {
var data skpPerfData
if err := json.Unmarshal(b, &data); err != nil {
return nil, skerr.Wrap(err)
}
avgWithoutFlushMS, medianWithoutFlushMS, stddevWithoutFlushMS := summarize(data.WithoutFlushMS)
avgWithFlushMS, medianWithFlushMS, stddevWithFlushMS := summarize(data.WithFlushMS)
avgTotalFrameMS, medianTotalFrameMS, stddevTotalFrameMS := summarize(data.TotalFrameMS)
return map[string]float32{
"avg_render_without_flush_ms": avgWithoutFlushMS,
"median_render_without_flush_ms": medianWithoutFlushMS,
"stddev_render_without_flush_ms": stddevWithoutFlushMS,
"avg_render_with_flush_ms": avgWithFlushMS,
"median_render_with_flush_ms": medianWithFlushMS,
"stddev_render_with_flush_ms": stddevWithFlushMS,
"avg_render_frame_ms": avgTotalFrameMS,
"median_render_frame_ms": medianTotalFrameMS,
"stddev_render_frame_ms": stddevTotalFrameMS,
"skp_load_ms": data.SKPLoadMS,
}, nil
}
func summarize(input []float32) (float32, float32, float32) {
// Make a copy of the data so we don't mutate the order of the original
sorted := make([]float32, len(input))
copy(sorted, input)
sort.Slice(sorted, func(i, j int) bool {
return sorted[i] < sorted[j]
})
avg := computeAverage(sorted)
variance := float32(0)
for i := 0; i < len(sorted); i++ {
variance += (sorted[i] - avg) * (sorted[i] - avg)
}
stddev := float32(math.Sqrt(float64(variance / float32(len(sorted)))))
medIdx := (len(sorted) * 50) / 100
return avg, sorted[medIdx], stddev
}
func computeAverage(d []float32) float32 {
avg := float32(0)
for i := 0; i < len(d); i++ {
avg += d[i]
}
avg /= float32(len(d))
return avg
}

248
infra/bots/task_drivers/perf_puppeteer_render_skps/perf_puppeteer_render_skps_test.go Normal file
View File

@ -0,0 +1,248 @@
// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package main
import (
"context"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.skia.org/infra/go/exec"
"go.skia.org/infra/go/testutils"
"go.skia.org/infra/task_driver/go/td"
)
func TestSetup_NPMInitializedChromeStoppedBenchmarkOutCreated(t *testing.T) {
benchmarkPath, err := ioutil.TempDir("", "benchmark")
require.NoError(t, err)
defer testutils.RemoveAll(t, benchmarkPath)
const fakeNodeBinPath = "/fake/path/to/node/bin"
res := td.RunTestSteps(t, false, func(ctx context.Context) error {
mock := exec.CommandCollector{}
ctx = td.WithExecRunFn(ctx, mock.Run)
err := setup(ctx, benchmarkPath, fakeNodeBinPath)
if err != nil {
assert.NoError(t, err)
return err
}
cmds := mock.Commands()
require.Len(t, cmds, 2)
cmd := cmds[0]
assert.Equal(t, "/fake/path/to/node/bin/npm", cmd.Name)
assert.Equal(t, []string{"ci"}, cmd.Args)
cmd = cmds[1]
assert.Equal(t, "killall", cmd.Name)
assert.Equal(t, []string{"chrome"}, cmd.Args)
return nil
})
require.Empty(t, res.Errors)
require.Empty(t, res.Exceptions)
fi, err := os.Stat(filepath.Join(benchmarkPath, "out"))
require.NoError(t, err)
assert.True(t, fi.IsDir())
}
func TestBenchSKPs_CPUHasNoUseGPUFlag(t *testing.T) {
skps, err := ioutil.TempDir("", "skps")
require.NoError(t, err)
defer testutils.RemoveAll(t, skps)
require.NoError(t, ioutil.WriteFile(filepath.Join(skps, "first_skp"), []byte("doesnt matter"), 0777))
const fakeNodeBinPath = "/fake/path/to/node/bin"
const fakeCanvasKitPath = "/fake/path/to/canvaskit"
const fakeBenchmarkPath = "/fake/path/to/perf-puppeteer"
perfObj := perfJSONFormat{
Key: map[string]string{
perfKeyCpuOrGPU: "CPU",
},
}
res := td.RunTestSteps(t, false, func(ctx context.Context) error {
mock := exec.CommandCollector{}
ctx = td.WithExecRunFn(ctx, mock.Run)
err := benchSKPs(ctx, perfObj, fakeBenchmarkPath, fakeCanvasKitPath, skps, fakeNodeBinPath)
if err != nil {
assert.NoError(t, err)
return err
}
require.Len(t, mock.Commands(), 1)
cmd := mock.Commands()[0]
assert.Equal(t, "/fake/path/to/node/bin/node", cmd.Name)
assert.Equal(t, []string{"perf-canvaskit-with-puppeteer",
"--bench_html", "render-skp.html",
"--canvaskit_js", "/fake/path/to/canvaskit/canvaskit.js",
"--canvaskit_wasm", "/fake/path/to/canvaskit/canvaskit.wasm",
"--input_skp", filepath.Join(skps, "first_skp"),
"--output", "/fake/path/to/perf-puppeteer/out/first_skp.json"}, cmd.Args)
return nil
})
require.Empty(t, res.Errors)
require.Empty(t, res.Exceptions)
}
func TestBenchSKPs_GPUHasFlag(t *testing.T) {
skps, err := ioutil.TempDir("", "skps")
require.NoError(t, err)
defer testutils.RemoveAll(t, skps)
require.NoError(t, ioutil.WriteFile(filepath.Join(skps, "first_skp"), []byte("doesnt matter"), 0777))
const fakeNodeBinPath = "/fake/path/to/node/bin"
const fakeCanvasKitPath = "/fake/path/to/canvaskit"
const fakeBenchmarkPath = "/fake/path/to/perf-puppeteer"
perfObj := perfJSONFormat{
Key: map[string]string{
perfKeyCpuOrGPU: "GPU",
},
}
res := td.RunTestSteps(t, false, func(ctx context.Context) error {
mock := exec.CommandCollector{}
ctx = td.WithExecRunFn(ctx, mock.Run)
err := benchSKPs(ctx, perfObj, fakeBenchmarkPath, fakeCanvasKitPath, skps, fakeNodeBinPath)
if err != nil {
assert.NoError(t, err)
return err
}
require.Len(t, mock.Commands(), 1)
cmd := mock.Commands()[0]
assert.Equal(t, "/fake/path/to/node/bin/node", cmd.Name)
assert.Equal(t, []string{"perf-canvaskit-with-puppeteer",
"--bench_html", "render-skp.html",
"--canvaskit_js", "/fake/path/to/canvaskit/canvaskit.js",
"--canvaskit_wasm", "/fake/path/to/canvaskit/canvaskit.wasm",
"--input_skp", filepath.Join(skps, "first_skp"),
"--output", "/fake/path/to/perf-puppeteer/out/first_skp.json",
"--use_gpu"}, cmd.Args)
return nil
})
require.Empty(t, res.Errors)
require.Empty(t, res.Exceptions)
}
func TestProcessSkottieFramesData_GPUTwoInputsGetSummarizedAndCombined(t *testing.T) {
input, err := ioutil.TempDir("", "inputs")
require.NoError(t, err)
defer testutils.RemoveAll(t, input)
err = writeFilesToDisk(filepath.Join(input, "out"), map[string]string{
"first_skp.json": firstSKP,
"second_skp.json": secondSKP,
})
require.NoError(t, err)
output, err := ioutil.TempDir("", "perf")
require.NoError(t, err)
defer testutils.RemoveAll(t, output)
// These are based off of realistic values.
keys := map[string]string{
"os": "Ubuntu18",
"model": "Golo",
perfKeyCpuOrGPU: "GPU",
"cpu_or_gpu_value": "QuadroP400",
}
perfObj, err := makePerfObj(someGitHash, someTaskID, someMachineID, keys)
require.NoError(t, err)
outputFile := filepath.Join(output, "perf-taskid1352.json")
res := td.RunTestSteps(t, false, func(ctx context.Context) error {
return processSKPData(ctx, perfObj, input, outputFile)
})
require.Empty(t, res.Errors)
require.Empty(t, res.Exceptions)
b, err := ioutil.ReadFile(outputFile)
require.NoError(t, err)
assert.Equal(t, `{
"gitHash": "032631e490db494128e0610a19adce4cab9706d1",
"swarming_task_id": "4bdd43ed7c906c11",
"swarming_machine_id": "skia-e-gce-203",
"key": {
"arch": "wasm",
"binary": "CanvasKit",
"browser": "Chromium",
"configuration": "Release",
"cpu_or_gpu": "GPU",
"cpu_or_gpu_value": "QuadroP400",
"extra_config": "RenderSKP",
"model": "Golo",
"os": "Ubuntu18"
},
"results": {
"first_skp": {
"webgl2": {
"avg_render_frame_ms": 150.065,
"avg_render_with_flush_ms": 133.91167,
"avg_render_without_flush_ms": 45.398335,
"median_render_frame_ms": 143.71,
"median_render_with_flush_ms": 125.185,
"median_render_without_flush_ms": 37.445,
"skp_load_ms": 1.715,
"stddev_render_frame_ms": 15.210527,
"stddev_render_with_flush_ms": 15.47429,
"stddev_render_without_flush_ms": 21.69691
}
},
"second_skp": {
"webgl2": {
"avg_render_frame_ms": 316.7317,
"avg_render_with_flush_ms": 233.91167,
"avg_render_without_flush_ms": 85.39834,
"median_render_frame_ms": 243.71,
"median_render_with_flush_ms": 225.185,
"median_render_without_flush_ms": 67.445,
"skp_load_ms": 3.715,
"stddev_render_frame_ms": 109.164635,
"stddev_render_with_flush_ms": 15.474287,
"stddev_render_without_flush_ms": 43.27188
}
}
}
}`, string(b))
}
func writeFilesToDisk(path string, fileNamesToContent map[string]string) error {
if err := os.MkdirAll(path, 0777); err != nil {
return err
}
for name, content := range fileNamesToContent {
if err := ioutil.WriteFile(filepath.Join(path, name), []byte(content), 0666); err != nil {
return err
}
}
return nil
}
const (
someGitHash = "032631e490db494128e0610a19adce4cab9706d1"
someTaskID = "4bdd43ed7c906c11"
someMachineID = "skia-e-gce-203"
)
const firstSKP = `{
"without_flush_ms": [23.71, 37.445, 75.04],
"with_flush_ms": [125.185, 120.895, 155.655],
"total_frame_ms": [143.71, 135.445, 171.04],
"skp_load_ms":1.715
}`
const secondSKP = `{
"without_flush_ms": [43.71, 67.445, 145.04],
"with_flush_ms": [225.185, 220.895, 255.655],
"total_frame_ms": [243.71, 235.445, 471.04],
"skp_load_ms":3.715
}`

1
infra/bots/task_drivers/perf_puppeteer_skottie_frames/perf_puppeteer_skottie_frames.go
View File

@ -74,7 +74,6 @@ func main() {
lottiesAbsPath := getAbsoluteOfRequiredFlag(ctx, *lottiesPath, "lotties_path")
outputAbsPath := getAbsoluteOfRequiredFlag(ctx, *outputPath, "output_path")
// Run the infra tests.
if err := setup(ctx, benchmarkAbsPath, nodeBinAbsPath); err != nil {
td.Fatal(ctx, skerr.Wrap(err))
}

11
infra/bots/task_drivers/perf_puppeteer_skottie_frames/perf_puppeteer_skottie_frames_test.go
View File

@ -7,7 +7,6 @@ package main
import (
"context"
"io/ioutil"
"math/rand"
"os"
"path/filepath"
"testing"
@ -15,12 +14,14 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.skia.org/infra/go/exec"
"go.skia.org/infra/go/testutils"
"go.skia.org/infra/task_driver/go/td"
)
func TestSetup_NPMInitializedBenchmarkOutCreated(t *testing.T) {
benchmarkPath, err := ioutil.TempDir("", "benchmark")
require.NoError(t, err)
defer testutils.RemoveAll(t, benchmarkPath)
const fakeNodeBinPath = "/fake/path/to/node/bin"
@ -49,6 +50,7 @@ func TestSetup_NPMInitializedBenchmarkOutCreated(t *testing.T) {
func TestBenchSkottieFrames_CPUHasNoUseGPUFlag(t *testing.T) {
lotties, err := ioutil.TempDir("", "lotties")
require.NoError(t, err)
defer testutils.RemoveAll(t, lotties)
require.NoError(t, os.MkdirAll(filepath.Join(lotties, "animation_1"), 0777))
@ -89,6 +91,7 @@ func TestBenchSkottieFrames_CPUHasNoUseGPUFlag(t *testing.T) {
func TestBenchSkottieFrames_GPUHasFlag(t *testing.T) {
lotties, err := ioutil.TempDir("", "lotties")
require.NoError(t, err)
defer testutils.RemoveAll(t, lotties)
require.NoError(t, os.MkdirAll(filepath.Join(lotties, "animation_1"), 0777))
@ -134,6 +137,7 @@ func TestBenchSkottieFrames_GPUHasFlag(t *testing.T) {
func TestProcessSkottieFramesData_CPUTwoInputsGetSummarizedAndCombined(t *testing.T) {
input, err := ioutil.TempDir("", "inputs")
require.NoError(t, err)
defer testutils.RemoveAll(t, input)
err = writeFilesToDisk(filepath.Join(input, "out"), map[string]string{
"first_animation.json": skottieFramesSampleOne,
"second_animation.json": skottieFramesSampleTwo,
@ -141,6 +145,7 @@ func TestProcessSkottieFramesData_CPUTwoInputsGetSummarizedAndCombined(t *testin
require.NoError(t, err)
output, err := ioutil.TempDir("", "perf")
require.NoError(t, err)
defer testutils.RemoveAll(t, output)
keys := map[string]string{
"os": "Debian10",
@ -225,6 +230,7 @@ func TestProcessSkottieFramesData_CPUTwoInputsGetSummarizedAndCombined(t *testin
func TestProcessSkottieFramesData_GPUTwoInputsGetSummarizedAndCombined(t *testing.T) {
input, err := ioutil.TempDir("", "inputs")
require.NoError(t, err)
defer testutils.RemoveAll(t, input)
err = writeFilesToDisk(filepath.Join(input, "out"), map[string]string{
"first_animation.json": skottieFramesSampleOne,
"second_animation.json": skottieFramesSampleTwo,
@ -232,6 +238,7 @@ func TestProcessSkottieFramesData_GPUTwoInputsGetSummarizedAndCombined(t *testin
require.NoError(t, err)
output, err := ioutil.TempDir("", "perf")
require.NoError(t, err)
defer testutils.RemoveAll(t, output)
// These are based off of realistic values.
keys := map[string]string{
@ -250,8 +257,6 @@ func TestProcessSkottieFramesData_GPUTwoInputsGetSummarizedAndCombined(t *testin
})
require.Empty(t, res.Errors)
// Re-seed the RNG, so we can get the filename the code wrote to.
rand.Seed(0)
b, err := ioutil.ReadFile(outputFile)
require.NoError(t, err)

285
infra/bots/tasks.json
View File

@ -1157,6 +1157,11 @@
"Upload-Perf-Debian10-EMCC-GCE-CPU-AVX2-wasm-Release-All-PathKit"
]
},
"Perf-Debian10-EMCC-GCE-CPU-AVX2-wasm-Release-All-Puppeteer_RenderSKP": {
"tasks": [
"Upload-Perf-Debian10-EMCC-GCE-CPU-AVX2-wasm-Release-All-Puppeteer_RenderSKP"
]
},
"Perf-Debian10-EMCC-GCE-CPU-AVX2-wasm-Release-All-Puppeteer_SkottieFrames": {
"tasks": [
"Upload-Perf-Debian10-EMCC-GCE-CPU-AVX2-wasm-Release-All-Puppeteer_SkottieFrames"
@ -1272,6 +1277,11 @@
"Upload-Perf-Ubuntu18-Clang-Golo-GPU-QuadroP400-x86_64-Release-All-Vulkan"
]
},
"Perf-Ubuntu18-EMCC-Golo-GPU-QuadroP400-wasm-Release-All-Puppeteer_RenderSKP": {
"tasks": [
"Upload-Perf-Ubuntu18-EMCC-Golo-GPU-QuadroP400-wasm-Release-All-Puppeteer_RenderSKP"
]
},
"Perf-Ubuntu18-EMCC-Golo-GPU-QuadroP400-wasm-Release-All-Puppeteer_SkottieFrames": {
"tasks": [
"Upload-Perf-Ubuntu18-EMCC-Golo-GPU-QuadroP400-wasm-Release-All-Puppeteer_SkottieFrames"
@ -20727,6 +20737,77 @@
"perf"
]
},
"Perf-Debian10-EMCC-GCE-CPU-AVX2-wasm-Release-All-Puppeteer_RenderSKP": {
"cipd_packages": [
{
"name": "infra/tools/luci-auth/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:00ed4f248e3011ef391ff0ca76197e7ffa84eb3c"
},
{
"name": "skia/bots/node",
"path": "node",
"version": "version:3"
},
{
"name": "skia/bots/skp",
"path": "skp",
"version": "version:244"
}
],
"command": [
"./perf_puppeteer_render_skps",
"--project_id",
"skia-swarming-bots",
"--git_hash",
"<(REVISION)",
"--task_id",
"<(TASK_ID)",
"--task_name",
"Perf-Debian10-EMCC-GCE-CPU-AVX2-wasm-Release-All-Puppeteer_RenderSKP",
"--canvaskit_bin_path",
"./build",
"--skps_path",
"./skp",
"--node_bin_path",
"./node/node/bin",
"--benchmark_path",
"./tools/perf-canvaskit-puppeteer",
"--output_path",
"perf",
"--os_trace",
"Debian10",
"--model_trace",
"GCE",
"--cpu_or_gpu_trace",
"CPU",
"--cpu_or_gpu_value_trace",
"AVX2",
"--alsologtostderr"
],
"dependencies": [
"Build-Debian10-EMCC-wasm-Release-CanvasKit_CPU",
"Housekeeper-PerCommit-BuildTaskDrivers"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
"machine_type:n1-standard-16",
"os:Debian-10.3",
"pool:Skia"
],
"env_prefixes": {
"PATH": [
"node/node/bin"
]
},
"execution_timeout_ns": 1200000000000,
"io_timeout_ns": 1200000000000,
"isolate": "perf_puppeteer.isolate",
"outputs": [
"perf"
],
"service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
},
"Perf-Debian10-EMCC-GCE-CPU-AVX2-wasm-Release-All-Puppeteer_SkottieFrames": {
"cipd_packages": [
{
@ -22468,6 +22549,76 @@
"perf"
]
},
"Perf-Ubuntu18-EMCC-Golo-GPU-QuadroP400-wasm-Release-All-Puppeteer_RenderSKP": {
"cipd_packages": [
{
"name": "infra/tools/luci-auth/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:00ed4f248e3011ef391ff0ca76197e7ffa84eb3c"
},
{
"name": "skia/bots/node",
"path": "node",
"version": "version:3"
},
{
"name": "skia/bots/skp",
"path": "skp",
"version": "version:244"
}
],
"command": [
"./perf_puppeteer_render_skps",
"--project_id",
"skia-swarming-bots",
"--git_hash",
"<(REVISION)",
"--task_id",
"<(TASK_ID)",
"--task_name",
"Perf-Ubuntu18-EMCC-Golo-GPU-QuadroP400-wasm-Release-All-Puppeteer_RenderSKP",
"--canvaskit_bin_path",
"./build",
"--skps_path",
"./skp",
"--node_bin_path",
"./node/node/bin",
"--benchmark_path",
"./tools/perf-canvaskit-puppeteer",
"--output_path",
"perf",
"--os_trace",
"Ubuntu18",
"--model_trace",
"Golo",
"--cpu_or_gpu_trace",
"GPU",
"--cpu_or_gpu_value_trace",
"QuadroP400",
"--alsologtostderr"
],
"dependencies": [
"Build-Debian10-EMCC-wasm-Release-CanvasKit",
"Housekeeper-PerCommit-BuildTaskDrivers"
],
"dimensions": [
"gpu:10de:1cb3-430.14",
"os:Ubuntu-18.04",
"pool:Skia"
],
"env_prefixes": {
"PATH": [
"node/node/bin"
]
},
"execution_timeout_ns": 1200000000000,
"io_timeout_ns": 1200000000000,
"isolate": "perf_puppeteer.isolate",
"outputs": [
"perf"
],
"service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
},
"Perf-Ubuntu18-EMCC-Golo-GPU-QuadroP400-wasm-Release-All-Puppeteer_SkottieFrames": {
"cipd_packages": [
{
@ -56756,6 +56907,73 @@
"max_attempts": 2,
"service_account": "skia-external-nano-uploader@skia-swarming-bots.iam.gserviceaccount.com"
},
"Upload-Perf-Debian10-EMCC-GCE-CPU-AVX2-wasm-Release-All-Puppeteer_RenderSKP": {
"caches": [
{
"name": "vpython",
"path": "cache/vpython"
}
],
"cipd_packages": [
{
"name": "infra/gsutil",
"path": "cipd_bin_packages",
"version": "version:4.46"
},
{
"name": "infra/tools/luci-auth/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:00ed4f248e3011ef391ff0ca76197e7ffa84eb3c"
},
{
"name": "infra/tools/luci/kitchen/${platform}",
"path": ".",
"version": "git_revision:00ed4f248e3011ef391ff0ca76197e7ffa84eb3c"
},
{
"name": "infra/tools/luci/vpython/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:00ed4f248e3011ef391ff0ca76197e7ffa84eb3c"
}
],
"command": [
"cipd_bin_packages/vpython${EXECUTABLE_SUFFIX}",
"-u",
"skia/infra/bots/run_recipe.py",
"${ISOLATED_OUTDIR}",
"upload_nano_results",
"{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildbucket_build_id\":\"<(BUILDBUCKET_BUILD_ID)\",\"buildername\":\"Perf-Debian10-EMCC-GCE-CPU-AVX2-wasm-Release-All-Puppeteer_RenderSKP\",\"gs_bucket\":\"skia-perf\",\"patch_issue\":\"<(ISSUE_INT)\",\"patch_ref\":\"<(PATCH_REF)\",\"patch_repo\":\"<(PATCH_REPO)\",\"patch_set\":\"<(PATCHSET_INT)\",\"patch_storage\":\"<(PATCH_STORAGE)\",\"repository\":\"<(REPO)\",\"revision\":\"<(REVISION)\",\"swarm_out_dir\":\"output_ignored\",\"task_id\":\"<(TASK_ID)\"}",
"skia"
],
"dependencies": [
"Housekeeper-PerCommit-BundleRecipes",
"Perf-Debian10-EMCC-GCE-CPU-AVX2-wasm-Release-All-Puppeteer_RenderSKP"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
"gpu:none",
"machine_type:n1-highmem-2",
"os:Debian-10.3",
"pool:Skia"
],
"env_prefixes": {
"PATH": [
"cipd_bin_packages",
"cipd_bin_packages/bin"
],
"VPYTHON_VIRTUALENV_ROOT": [
"cache/vpython"
]
},
"execution_timeout_ns": 3600000000000,
"extra_tags": {
"log_location": "logdog://logs.chromium.org/skia/${SWARMING_TASK_ID}/+/annotations"
},
"io_timeout_ns": 3600000000000,
"isolate": "swarm_recipe.isolate",
"max_attempts": 2,
"service_account": "skia-external-nano-uploader@skia-swarming-bots.iam.gserviceaccount.com"
},
"Upload-Perf-Debian10-EMCC-GCE-CPU-AVX2-wasm-Release-All-Puppeteer_SkottieFrames": {
"caches": [
{
@ -58029,6 +58247,73 @@
"max_attempts": 2,
"service_account": "skia-external-nano-uploader@skia-swarming-bots.iam.gserviceaccount.com"
},
"Upload-Perf-Ubuntu18-EMCC-Golo-GPU-QuadroP400-wasm-Release-All-Puppeteer_RenderSKP": {
"caches": [
{
"name": "vpython",
"path": "cache/vpython"
}
],
"cipd_packages": [
{
"name": "infra/gsutil",
"path": "cipd_bin_packages",
"version": "version:4.46"
},
{
"name": "infra/tools/luci-auth/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:00ed4f248e3011ef391ff0ca76197e7ffa84eb3c"
},
{
"name": "infra/tools/luci/kitchen/${platform}",
"path": ".",
"version": "git_revision:00ed4f248e3011ef391ff0ca76197e7ffa84eb3c"
},
{
"name": "infra/tools/luci/vpython/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:00ed4f248e3011ef391ff0ca76197e7ffa84eb3c"
}
],
"command": [
"cipd_bin_packages/vpython${EXECUTABLE_SUFFIX}",
"-u",
"skia/infra/bots/run_recipe.py",
"${ISOLATED_OUTDIR}",
"upload_nano_results",
"{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildbucket_build_id\":\"<(BUILDBUCKET_BUILD_ID)\",\"buildername\":\"Perf-Ubuntu18-EMCC-Golo-GPU-QuadroP400-wasm-Release-All-Puppeteer_RenderSKP\",\"gs_bucket\":\"skia-perf\",\"patch_issue\":\"<(ISSUE_INT)\",\"patch_ref\":\"<(PATCH_REF)\",\"patch_repo\":\"<(PATCH_REPO)\",\"patch_set\":\"<(PATCHSET_INT)\",\"patch_storage\":\"<(PATCH_STORAGE)\",\"repository\":\"<(REPO)\",\"revision\":\"<(REVISION)\",\"swarm_out_dir\":\"output_ignored\",\"task_id\":\"<(TASK_ID)\"}",
"skia"
],
"dependencies": [
"Housekeeper-PerCommit-BundleRecipes",
"Perf-Ubuntu18-EMCC-Golo-GPU-QuadroP400-wasm-Release-All-Puppeteer_RenderSKP"
],
"dimensions": [
"cpu:x86-64-Haswell_GCE",
"gpu:none",
"machine_type:n1-highmem-2",
"os:Debian-10.3",
"pool:Skia"
],
"env_prefixes": {
"PATH": [
"cipd_bin_packages",
"cipd_bin_packages/bin"
],
"VPYTHON_VIRTUALENV_ROOT": [
"cache/vpython"
]
},
"execution_timeout_ns": 3600000000000,
"extra_tags": {
"log_location": "logdog://logs.chromium.org/skia/${SWARMING_TASK_ID}/+/annotations"
},
"io_timeout_ns": 3600000000000,
"isolate": "swarm_recipe.isolate",
"max_attempts": 2,
"service_account": "skia-external-nano-uploader@skia-swarming-bots.iam.gserviceaccount.com"
},
"Upload-Perf-Ubuntu18-EMCC-Golo-GPU-QuadroP400-wasm-Release-All-Puppeteer_SkottieFrames": {
"caches": [
{

8
tools/perf-canvaskit-puppeteer/Makefile
View File

@ -17,4 +17,10 @@ test_path_transform_with_snap:
--canvaskit_wasm ../../out/canvaskit_wasm/canvaskit.wasm --use_gpu \
--assets path_translate_assets \
--bench_html path-transform.html \
--query_params translate opacity snap
--query_params translate opacity snap
skp_with_local:
node perf-canvaskit-with-puppeteer.js --canvaskit_js ../../out/canvaskit_wasm/canvaskit.js \
--canvaskit_wasm ../../out/canvaskit_wasm/canvaskit.wasm --use_gpu \
--input_skp ${HOME}/skps/desk_nytimes.skp \
--bench_html render-skp.html

16
tools/perf-canvaskit-puppeteer/perf-canvaskit-with-puppeteer.js
View File

@ -32,6 +32,11 @@ const opts = [
typeLabel: '{underline file}',
description: 'The Lottie JSON file to process.'
},
{
name: 'input_skp',
typeLabel: '{underline file}',
description: 'The SKP file to process.'
},
{
name: 'assets',
typeLabel: '{underline file}',
@ -137,6 +142,12 @@ if (options.input_lottie) {
const lottieJSON = fs.readFileSync(options.input_lottie, 'utf8');
app.get('/static/lottie.json', (req, res) => res.send(lottieJSON));
}
if (options.input_skp) {
const skpBytes = fs.readFileSync(options.input_skp, 'binary');
app.get('/static/test.skp', (req, res) => {
res.send(new Buffer(skpBytes, 'binary'));
});
}
if (options.assets) {
app.use('/static/assets/', express.static(options.assets));
console.log('assets served from', options.assets);
@ -167,6 +178,11 @@ async function driveBrowser() {
'--no-sandbox',
'--disable-setuid-sandbox',
'--window-size=' + viewPort.width + ',' + viewPort.height,
// The following two params allow Chrome to run at an unlimited fps. Note, if there is
// already a chrome instance running, these arguments will have NO EFFECT, as the existing
// Chrome instance will be used instead of puppeteer spinning up a new one.
'--disable-frame-rate-limit',
'--disable-gpu-vsync',
];
if (options.use_gpu) {
browser_args.push('--ignore-gpu-blacklist');

157
tools/perf-canvaskit-puppeteer/render-skp.html Normal file
View File

@ -0,0 +1,157 @@
<!-- This benchmark aims to accurately measure the time it takes for CanvasKit to render
an SKP from our test corpus. It is very careful to measure the time between frames. This form
of measurement makes sure we are capturing the GPU draw time. CanvasKit.flush() returns after
it has sent all the instructions to the GPU, but we don't know the GPU is done until the next
frame is requested. Thus, we need to keep track of the time between frames in order to
accurately calculate draw time. Keeping track of the drawPicture and drawPicture+flush is still
useful for telling us how much time we are spending in WASM land and if our drawing is CPU
bound or GPU bound. If total_frame_ms is close to with_flush_ms, we are CPU bound; if
total_frame_ms >> with_flush_ms, we are GPU bound.
-->
<!DOCTYPE html>
<html>
<head>
<title>CanvasKit SKP Perf</title>
<meta charset="utf-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<script src="/static/canvaskit.js" type="text/javascript" charset="utf-8"></script>
<style type="text/css" media="screen">
body {
margin: 0;
padding: 0;
}
</style>
</head>
<body>
<main>
<button id="start_bench">Start Benchmark</button>
<br>
<canvas id=anim width=1000 height=1000 style="height: 1000px; width: 1000px;"></canvas>
</main>
<script type="text/javascript" charset="utf-8">
const WIDTH = 1000;
const HEIGHT = 1000;
// Run this number of frames before starting to measure things. This allows us to make sure
// the noise from the first few renders (e.g shader compilation, caches) is removed from the
// data we capture.
const WARM_UP_FRAMES = 10;
const MAX_FRAMES = 201; // This should be sufficient to have low noise.
const SKP_PATH = '/static/test.skp';
(function() {
const loadKit = CanvasKitInit({
locateFile: (file) => '/static/' + file,
});
const loadSKP = fetch(SKP_PATH).then((resp) => {
return resp.arrayBuffer();
});
Promise.all([loadKit, loadSKP]).then((values) => {
const [CanvasKit, skpBytes] = values;
const loadStart = performance.now();
const skp = CanvasKit.MakeSkPicture(skpBytes);
const loadTime = performance.now() - loadStart;
console.log('loaded skp', skp, loadTime);
if (!skp) {
window._error = 'could not read skp';
return;
}
const surface = getSurface(CanvasKit);
if (!surface) {
console.error('Could not make surface', window._error);
return;
}
const canvas = surface.getCanvas();
document.getElementById('start_bench').addEventListener('click', () => {
const clearColor = CanvasKit.WHITE;
const totalFrame = new Float32Array(MAX_FRAMES);
const withFlush = new Float32Array(MAX_FRAMES);
const withoutFlush = new Float32Array(MAX_FRAMES);
let warmUp = true;
let idx = 0;
let previousFrame;
function drawFrame() {
const start = performance.now();
canvas.clear(clearColor);
canvas.drawPicture(skp);
const afterDraw = performance.now();
surface.flush();
const end = performance.now();
if (warmUp) {
idx++;
if (idx >= WARM_UP_FRAMES) {
idx = -1;
warmUp = false;
}
window.requestAnimationFrame(drawFrame);
return;
}
if (idx >= 0) {
// Fill out total time the previous frame took to draw.
totalFrame[idx] = start - previousFrame;
}
previousFrame = start;
idx++;
// If we have maxed out the frames we are measuring or have completed the animation,
// we stop benchmarking.
if (idx >= withFlush.length) {
window._perfData = {
total_frame_ms: Array.from(totalFrame).slice(0, idx),
with_flush_ms: Array.from(withFlush).slice(0, idx),
without_flush_ms: Array.from(withoutFlush).slice(0, idx),
skp_load_ms: loadTime,
};
window._perfDone = true;
return;
}
// We can fill out this frame's intermediate steps.
withFlush[idx] = end - start;
withoutFlush[idx] = afterDraw - start;
window.requestAnimationFrame(drawFrame);
}
window.requestAnimationFrame(drawFrame);
});
console.log('Perf is ready');
window._perfReady = true;
});
}
)();
// TODO(kjlubick) make this configurable to return a WEBGL 1 or WEBGL 2 surface.
function getSurface(CanvasKit) {
let surface;
if (window.location.hash.indexOf('gpu') !== -1) {
surface = CanvasKit.MakeWebGLCanvasSurface('anim');
if (!surface) {
window._error = 'Could not make GPU surface';
return null;
}
let c = document.getElementById('anim');
// If CanvasKit was unable to instantiate a WebGL context, it will fallback
// to CPU and add a ck-replaced class to the canvas element.
if (c.classList.contains('ck-replaced')) {
window._error = 'fell back to CPU';
return null;
}
} else {
surface = CanvasKit.MakeSWCanvasSurface('anim');
if (!surface) {
window._error = 'Could not make CPU surface';
return null;
}
}
return surface;
}
</script>
</body>
</html>