[canvaskit] Document perf-canvaskit-puppeteer

We also catch any errors that happen during running the
test or flushing. This allows them to show up in the CI logs.

For skottie-frames in particular, we stop clearing the canvas
before each render, as that was masking an error (and not how
we do things on skottie.skia.org).

Change-Id: I83936a35b6c314da76a0a64e15deaabd156e71f2
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/421923
Reviewed-by: Kevin Lubick <kjlubick@google.com>

tools/perf-canvaskit-puppeteer/Makefile

@@ -1,37 +1,5 @@
-# This shows an example invocation presuming the user has built canvaskit locally.
+# All of the targets here assume a user has run `make release` in //modules/canvaskit
-frames_lego:
-	node perf-canvaskit-with-puppeteer.js --canvaskit_js ../../out/canvaskit_wasm/canvaskit.js \
-	--canvaskit_wasm ../../out/canvaskit_wasm/canvaskit.wasm --use_gpu \
-	--input_lottie ../../modules/canvaskit/perf/assets/lego_loader.json \
-	--bench_html skottie-frames.html
 
-frames_cpu_lego:
-	node perf-canvaskit-with-puppeteer.js --canvaskit_js ../../out/canvaskit_wasm/canvaskit.js \
-	--canvaskit_wasm ../../out/canvaskit_wasm/canvaskit.wasm \
-	--input_lottie ../../modules/canvaskit/perf/assets/lego_loader.json \
-	--bench_html skottie-frames.html
-
-test_path_transform_without_snap:
-	node perf-canvaskit-with-puppeteer.js --canvaskit_js --canvaskit_js ../../out/canvaskit_wasm/canvaskit.js \
-	--canvaskit_wasm ../../out/canvaskit_wasm/canvaskit.wasm --use_gpu \
-	--assets path_translate_assets \
-	--bench_html path-transform.html \
-	--query_params translate opacity
-
-test_path_transform_with_snap:
-	node perf-canvaskit-with-puppeteer.js --canvaskit_js --canvaskit_js ../../out/canvaskit_wasm/canvaskit.js \
-	--canvaskit_wasm ../../out/canvaskit_wasm/canvaskit.wasm --use_gpu \
-	--assets path_translate_assets \
-	--bench_html path-transform.html \
-	--query_params translate opacity snap
-
-skp_with_local:
-	node perf-canvaskit-with-puppeteer.js --canvaskit_js ../../out/canvaskit_wasm/canvaskit.js \
-	--canvaskit_wasm ../../out/canvaskit_wasm/canvaskit.wasm --use_gpu \
-	--input_skp ${HOME}/skps/desk_nytimes.skp \
-	--bench_html render-skp.html
-
-# runs all the tests in canvas_perf.js
 perf_js:
 	node perf-canvaskit-with-puppeteer.js --canvaskit_js ../../out/canvaskit_wasm/canvaskit.js \
 	--canvaskit_wasm ../../out/canvaskit_wasm/canvaskit.wasm --use_gpu \
@@ -39,5 +7,23 @@ perf_js:
 	--bench_html canvas_perf.html \
 	--timeout 600 # seconds
 
+frames:
+	node perf-canvaskit-with-puppeteer.js --canvaskit_js ../../out/canvaskit_wasm/canvaskit.js \
+	--canvaskit_wasm ../../out/canvaskit_wasm/canvaskit.wasm --use_gpu \
+	--input_lottie ~/Downloads/lottie-samples/lottiefiles.com\ -\ Octopus.json \
+	--bench_html skottie-frames.html
+
+frames_cpu:
+	node perf-canvaskit-with-puppeteer.js --canvaskit_js ../../out/canvaskit_wasm/canvaskit.js \
+	--canvaskit_wasm ../../out/canvaskit_wasm/canvaskit.wasm \
+	--input_lottie ~/Downloads/lottie-samples/404.json  \
+	--bench_html skottie-frames.html
+
+skp:
+	node perf-canvaskit-with-puppeteer.js --canvaskit_js ../../out/canvaskit_wasm/canvaskit.js \
+	--canvaskit_wasm ../../out/canvaskit_wasm/canvaskit.wasm --use_gpu \
+	--input_skp ~/Downloads/skps/desk_nytimes.skp \
+	--bench_html render-skp.html
+
 skps_release_and_simd:
 	./perf_all_skps.sh --release --simd --summary

tools/perf-canvaskit-puppeteer/README.md

@@ -0,0 +1,56 @@
+Measuring the performance of CanvasKit using Puppeteer and Chrome.
+
+## Initial setup
+
+Run `npm ci` to install the dependencies need to run the tests. In //modules/canvaskit, run
+`make release` to build the canvaskit that will be used. With modifications to the Makefile,
+other builds (e.g. `make profile`) can be used as well.
+
+If needed, one can download the lottie-samples and/or skp assets from CIPD using the sk tool:
+```
+sk asset download lottie-samples ~/Downloads/lottie-samples
+sk asset download skps ~/Downloads/skps
+```
+
+The actual location that these assets can be downloaded to is not important - the Makefile assumes
+them to be in Downloads, but that can be modified by the local user.
+
+## Basic Performance Tests
+We have a harness for running benchmarks. Benchmark code snippets can be added to `canvas_perf.js`.
+The harness itself is the `canvas_perf.html` and `benchmark.js`. It will run the "test" portion of
+the code on multiple frames and gather data.
+
+To run the benchmarks, run `make perf_js`. By default, this will use the most recent release build
+of canvaskit done locally. If you want to only run one or a few, modify the
+`canvas_perf.js` file by changing the relevent `tests.push` to `onlytests.push` and then run
+`make perf_js`.
+
+On the CI, the results from these tests are uploaded to Perf. For example:
+<https://perf.skia.org/e/?queries=test%3Dcanvas_drawOval>
+We include metrics such as the 90th, 95th, and 99th percentile frame, average frame time, median
+frame time, and standard deviation. There are three types of measurements: without_flush_ms is
+the measurement of the test() function; with_flush_ms is the measurement of test() and the
+subsequent flush() call; total_frame_ms is the frame-to-frame time. Frame-to-frame is important to
+measure because it accounts for any work the GPU needs to do, even after CanvasKit flushes.
+
+## Skottie Frames Performance
+There is a harness that gathers data about rendering 600 frames of a skottie animation, cycling
+through it in a similar fashion to how it would be displayed to a user (e.g. as it is on
+skottie.skia.org).
+
+To test it locally with a specific skottie animation, feel free to modify the Makefile to adjust the
+`input_lottie` argument and then run `make frames`. The harness itself is `skottie-frames.html` and
+`benchmark.js`.
+
+On the CI, the results from these tests are uploaded to Perf. For example:
+<https://perf.skia.org/e/?queries=test%3Dlego_loader>
+We include metrics such as the first 5 frame times, average frame times, 90th, 95th and 99th
+percentile frame time.
+
+## SKP Performance
+There is a harness that repeatedly will draw an SKP and measure various metrics. This is handled
+by `skottie-frames.html` and `benchmark.js`. As before, feel free to modify the Makefile (the
+`input_skp` argument) and run `make skp`.
+
+On the CI, the results from these tests are uploaded to Perf. For example:
+<https://perf.skia.org/e/?queries=binary%3DCanvasKit%26test%3Ddesk_chalkboard.skp>

tools/perf-canvaskit-puppeteer/benchmark.js

@@ -46,11 +46,18 @@ function startTimingFrames(drawFn, surface, warmupFrames, maxFrames, timeoutMill
     let previousFrame;
 
     function drawFrame() {
-      const start = performance.now();
+      let start, afterDraw, end;
-      drawFn();
+      try {
-      const afterDraw = performance.now();
+        start = performance.now();
-      surface.flush();
+        drawFn();
-      const end = performance.now();
+        afterDraw = performance.now();
+        surface.flush();
+        end = performance.now();
+      } catch (e) {
+        console.error(e);
+        window._error = e.stack || e.toString();
+        return;
+      }
 
       if (warmUp) {
         idx++;

tools/perf-canvaskit-puppeteer/skottie-frames.html

@@ -28,7 +28,7 @@ would display the animation (e.g. using clock time to determine where to seek, n
   <script type="text/javascript" charset="utf-8">
     const WIDTH  = 1000;
     const HEIGHT = 1000;
-    const WARM_UP_FRAMES = 0; // No warmup, so that the jank of initial frames gets's measured.
+    const WARM_UP_FRAMES = 0; // No warmup, so that the jank of initial frames gets measured.
     // We sample MAX_FRAMES or until MAX_SAMPLE_SECONDS has elapsed.
     const MAX_FRAMES = 600; // ~10s at 60fps
     const MAX_SAMPLE_MS = 45 * 1000; // in case something takes a while, stop after 30 seconds.
@@ -86,7 +86,6 @@ would display the animation (e.g. using clock time to determine where to seek, n
         const canvas = surface.getCanvas();
 
         document.getElementById('start_bench').addEventListener('click', async () => {
-          const clearColor = CanvasKit.WHITE;
           const startTime = Date.now();
           const damageRect = Float32Array.of(0, 0, 0, 0);
 
@@ -95,7 +94,6 @@ would display the animation (e.g. using clock time to determine where to seek, n
             const damage = animation.seek(seek, damageRect);
 
             if (damage[2] > damage[0] && damage[3] > damage[1]) {
-              canvas.clear(clearColor);
               animation.render(canvas, bounds);
             }
           }