From bd719361961a5b70e85c8a358e197a42b3ffcf1c Mon Sep 17 00:00:00 2001 From: Kevin Lubick Date: Fri, 25 Jun 2021 11:04:11 -0400 Subject: [PATCH] [canvaskit] Document perf-canvaskit-puppeteer We also catch any errors that happen during running the test or flushing. This allows them to show up in the CI logs. For skottie-frames in particular, we stop clearing the canvas before each render, as that was masking an error (and not how we do things on skottie.skia.org). Change-Id: I83936a35b6c314da76a0a64e15deaabd156e71f2 Reviewed-on: https://skia-review.googlesource.com/c/skia/+/421923 Reviewed-by: Kevin Lubick --- tools/perf-canvaskit-puppeteer/Makefile | 52 +++++++---------- tools/perf-canvaskit-puppeteer/README.md | 56 +++++++++++++++++++ tools/perf-canvaskit-puppeteer/benchmark.js | 17 ++++-- .../skottie-frames.html | 4 +- 4 files changed, 88 insertions(+), 41 deletions(-) create mode 100644 tools/perf-canvaskit-puppeteer/README.md diff --git a/tools/perf-canvaskit-puppeteer/Makefile b/tools/perf-canvaskit-puppeteer/Makefile index 383b287111..d6cdcb1cab 100644 --- a/tools/perf-canvaskit-puppeteer/Makefile +++ b/tools/perf-canvaskit-puppeteer/Makefile @@ -1,37 +1,5 @@ -# This shows an example invocation presuming the user has built canvaskit locally. -frames_lego: - node perf-canvaskit-with-puppeteer.js --canvaskit_js ../../out/canvaskit_wasm/canvaskit.js \ - --canvaskit_wasm ../../out/canvaskit_wasm/canvaskit.wasm --use_gpu \ - --input_lottie ../../modules/canvaskit/perf/assets/lego_loader.json \ - --bench_html skottie-frames.html +# All of the targets here assume a user has run `make release` in //modules/canvaskit -frames_cpu_lego: - node perf-canvaskit-with-puppeteer.js --canvaskit_js ../../out/canvaskit_wasm/canvaskit.js \ - --canvaskit_wasm ../../out/canvaskit_wasm/canvaskit.wasm \ - --input_lottie ../../modules/canvaskit/perf/assets/lego_loader.json \ - --bench_html skottie-frames.html - -test_path_transform_without_snap: - node perf-canvaskit-with-puppeteer.js --canvaskit_js --canvaskit_js ../../out/canvaskit_wasm/canvaskit.js \ - --canvaskit_wasm ../../out/canvaskit_wasm/canvaskit.wasm --use_gpu \ - --assets path_translate_assets \ - --bench_html path-transform.html \ - --query_params translate opacity - -test_path_transform_with_snap: - node perf-canvaskit-with-puppeteer.js --canvaskit_js --canvaskit_js ../../out/canvaskit_wasm/canvaskit.js \ - --canvaskit_wasm ../../out/canvaskit_wasm/canvaskit.wasm --use_gpu \ - --assets path_translate_assets \ - --bench_html path-transform.html \ - --query_params translate opacity snap - -skp_with_local: - node perf-canvaskit-with-puppeteer.js --canvaskit_js ../../out/canvaskit_wasm/canvaskit.js \ - --canvaskit_wasm ../../out/canvaskit_wasm/canvaskit.wasm --use_gpu \ - --input_skp ${HOME}/skps/desk_nytimes.skp \ - --bench_html render-skp.html - -# runs all the tests in canvas_perf.js perf_js: node perf-canvaskit-with-puppeteer.js --canvaskit_js ../../out/canvaskit_wasm/canvaskit.js \ --canvaskit_wasm ../../out/canvaskit_wasm/canvaskit.wasm --use_gpu \ @@ -39,5 +7,23 @@ perf_js: --bench_html canvas_perf.html \ --timeout 600 # seconds +frames: + node perf-canvaskit-with-puppeteer.js --canvaskit_js ../../out/canvaskit_wasm/canvaskit.js \ + --canvaskit_wasm ../../out/canvaskit_wasm/canvaskit.wasm --use_gpu \ + --input_lottie ~/Downloads/lottie-samples/lottiefiles.com\ -\ Octopus.json \ + --bench_html skottie-frames.html + +frames_cpu: + node perf-canvaskit-with-puppeteer.js --canvaskit_js ../../out/canvaskit_wasm/canvaskit.js \ + --canvaskit_wasm ../../out/canvaskit_wasm/canvaskit.wasm \ + --input_lottie ~/Downloads/lottie-samples/404.json \ + --bench_html skottie-frames.html + +skp: + node perf-canvaskit-with-puppeteer.js --canvaskit_js ../../out/canvaskit_wasm/canvaskit.js \ + --canvaskit_wasm ../../out/canvaskit_wasm/canvaskit.wasm --use_gpu \ + --input_skp ~/Downloads/skps/desk_nytimes.skp \ + --bench_html render-skp.html + skps_release_and_simd: ./perf_all_skps.sh --release --simd --summary \ No newline at end of file diff --git a/tools/perf-canvaskit-puppeteer/README.md b/tools/perf-canvaskit-puppeteer/README.md new file mode 100644 index 0000000000..8461284024 --- /dev/null +++ b/tools/perf-canvaskit-puppeteer/README.md @@ -0,0 +1,56 @@ +Measuring the performance of CanvasKit using Puppeteer and Chrome. + +## Initial setup + +Run `npm ci` to install the dependencies need to run the tests. In //modules/canvaskit, run +`make release` to build the canvaskit that will be used. With modifications to the Makefile, +other builds (e.g. `make profile`) can be used as well. + +If needed, one can download the lottie-samples and/or skp assets from CIPD using the sk tool: +``` +sk asset download lottie-samples ~/Downloads/lottie-samples +sk asset download skps ~/Downloads/skps +``` + +The actual location that these assets can be downloaded to is not important - the Makefile assumes +them to be in Downloads, but that can be modified by the local user. + +## Basic Performance Tests +We have a harness for running benchmarks. Benchmark code snippets can be added to `canvas_perf.js`. +The harness itself is the `canvas_perf.html` and `benchmark.js`. It will run the "test" portion of +the code on multiple frames and gather data. + +To run the benchmarks, run `make perf_js`. By default, this will use the most recent release build +of canvaskit done locally. If you want to only run one or a few, modify the +`canvas_perf.js` file by changing the relevent `tests.push` to `onlytests.push` and then run +`make perf_js`. + +On the CI, the results from these tests are uploaded to Perf. For example: + +We include metrics such as the 90th, 95th, and 99th percentile frame, average frame time, median +frame time, and standard deviation. There are three types of measurements: without_flush_ms is +the measurement of the test() function; with_flush_ms is the measurement of test() and the +subsequent flush() call; total_frame_ms is the frame-to-frame time. Frame-to-frame is important to +measure because it accounts for any work the GPU needs to do, even after CanvasKit flushes. + +## Skottie Frames Performance +There is a harness that gathers data about rendering 600 frames of a skottie animation, cycling +through it in a similar fashion to how it would be displayed to a user (e.g. as it is on +skottie.skia.org). + +To test it locally with a specific skottie animation, feel free to modify the Makefile to adjust the +`input_lottie` argument and then run `make frames`. The harness itself is `skottie-frames.html` and +`benchmark.js`. + +On the CI, the results from these tests are uploaded to Perf. For example: + +We include metrics such as the first 5 frame times, average frame times, 90th, 95th and 99th +percentile frame time. + +## SKP Performance +There is a harness that repeatedly will draw an SKP and measure various metrics. This is handled +by `skottie-frames.html` and `benchmark.js`. As before, feel free to modify the Makefile (the +`input_skp` argument) and run `make skp`. + +On the CI, the results from these tests are uploaded to Perf. For example: + \ No newline at end of file diff --git a/tools/perf-canvaskit-puppeteer/benchmark.js b/tools/perf-canvaskit-puppeteer/benchmark.js index fccc4b2e56..a4e6c26267 100644 --- a/tools/perf-canvaskit-puppeteer/benchmark.js +++ b/tools/perf-canvaskit-puppeteer/benchmark.js @@ -46,11 +46,18 @@ function startTimingFrames(drawFn, surface, warmupFrames, maxFrames, timeoutMill let previousFrame; function drawFrame() { - const start = performance.now(); - drawFn(); - const afterDraw = performance.now(); - surface.flush(); - const end = performance.now(); + let start, afterDraw, end; + try { + start = performance.now(); + drawFn(); + afterDraw = performance.now(); + surface.flush(); + end = performance.now(); + } catch (e) { + console.error(e); + window._error = e.stack || e.toString(); + return; + } if (warmUp) { idx++; diff --git a/tools/perf-canvaskit-puppeteer/skottie-frames.html b/tools/perf-canvaskit-puppeteer/skottie-frames.html index 52973d9ce0..2b657c2884 100644 --- a/tools/perf-canvaskit-puppeteer/skottie-frames.html +++ b/tools/perf-canvaskit-puppeteer/skottie-frames.html @@ -28,7 +28,7 @@ would display the animation (e.g. using clock time to determine where to seek, n