Add Puppeteer perf for release and experimental_simd builds of CanvasKit against SKPs

Adds the command:
  make skps_release_and_SIMD
for perfing builds against a set of SKPs in ~/skps for release and
simd builds of CanvasKit. Also outputs a summary of the perf results
in a table format.

See the document "SIMD CanvasKit Build Performance Testing"
for more details:
https://docs.google.com/document/d/114kdSGPMnOSQCZ7pFgd3MGMn5mIW562RMoXVmD13e0M/edit#

Bug: skia:10453
Change-Id: I311629a1420301dda41f7ec57ce1403b05fd949b
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/301982
Reviewed-by: Elliot Evans <elliotevans@google.com>
Reviewed-by: Kevin Lubick <kjlubick@google.com>

modules/canvaskit/README.md

@@ -94,6 +94,15 @@ gold results are reported to gold.skia.org
 
 Coverage is not measured while running tests this way.
 
+# Inspecting output WASM
+
+The `wasm2wat` tool from [the WebAssembly Binary Toolkit](https://github.com/WebAssembly/wabt)
+can be used to produce a human-readable text version of a `.wasm` file.
+
+The output of `wasm2wat --version` should be `1.0.13 (1.0.17)`. This version has been checked to
+work with the tools in `wasm_tools/SIMD/`. These tools programmatically inspect the `.wasm` output
+of a CanvasKit build to detect the presence of [wasm SIMD](https://github.com/WebAssembly/simd)
+operations.
 
 # Infrastructure Playbook
 

modules/canvaskit/wasm_tools/SIMD/build_simd_test.sh

@@ -4,14 +4,18 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+# This script takes a path to a .cpp file, compiles the file to wasm using emscripten, outputs
+# textual representations of all wasm SIMD operations present in the compiled .wasm, and starts
+# a static file server so that the running .wasm can be manually inspect in a browser.
+#
+# Example usage: ./build_simd_test.sh simd_float_capabilities.cpp
+
 # Requires that emscripten and wasm2wat are added to your PATH.
 # Requires, and is verified to work with
-# - wasm2wat 1.0.13 (1.0.17)
+# - The output of `wasm2wat --version` should be `1.0.13 (1.0.17)`
 #   - install from here: https://github.com/WebAssembly/wabt
 # - emscripten 1.39.16
 # - Chrome Canary 86.0.4186.0 with chrome://flags#enable-webassembly-simd enabled
-#
-# Example usage: ./build_simd_test.sh simd_float_test.cpp
 
 # build the file specified as the first argument with SIMD enabled.
 em++ $1 -I ../../../../ -msimd128 -Os -s WASM=1 -o output/simd_test.html
@@ -27,5 +31,6 @@ echo "The following WASM SIMD operations were used in the compiled code:"
 grep -f wasm_simd_types.txt output/simd_test.wat
 
 # Serve the compiled WASM so output can be manually inspected for correctness.
-echo "Go check out http://localhost:8000/output/simd_test.html"
+echo "Go check out http://localhost:8000/output/simd_test.html in Chrome Canary 86.0.4186.0 \
+or later and enable the chrome://flags#enable-webassembly-simd flag!"
 python ../../serve.py

modules/canvaskit/wasm_tools/SIMD/simd_test.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Copyright 2020 Google LLC
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This script takes a path to a .wasm file as input and outputs textual representations of all wasm
+# SIMD operations present in the .wasm file.
+#
+# Example usage: ./simd_test.sh simd_float_capabilities.wasm
+
+# Requires that emscripten and wasm2wat are added to your PATH.
+# Requires, and is verified to work with
+# - The output of `wasm2wat --version` should be `1.0.13 (1.0.17)`
+#   - install from here: https://github.com/WebAssembly/wabt
+
+
+wasm2wat --enable-simd $1 > output/simd_test.wat
+
+# The following lines output all SIMD operations produced in the output WASM.
+# Useful for checking that SIMD instructions are actually being used.
+# e.g. for the following C++ code:
+#  auto vec1 = skvx::Vec<2, double>({11.f, -22.f}) + skvx::Vec<2, double>({13.f, -1.f});
+# it is expected that the f64x2.add operation is present in the output WASM.
+echo "The following WASM SIMD operations were used in the compiled code:"
+grep -f wasm_simd_types.txt output/simd_test.wat

tools/perf-canvaskit-puppeteer/Makefile

@@ -23,4 +23,7 @@ skp_with_local:
 	node perf-canvaskit-with-puppeteer.js --canvaskit_js ../../out/canvaskit_wasm/canvaskit.js \
 	--canvaskit_wasm ../../out/canvaskit_wasm/canvaskit.wasm --use_gpu \
 	--input_skp ${HOME}/skps/desk_nytimes.skp \
-	--bench_html render-skp.html
+	--bench_html render-skp.html
+
+skps_release_and_simd:
+	./perf_all_skps.sh --release --simd --summary

tools/perf-canvaskit-puppeteer/benchmark.js

@@ -78,7 +78,7 @@ function startTimingFrames(drawFn, surface, warmupFrames, maxFrames, timeoutMill
     // We can fill out this frame's intermediate steps.
     withFlush[idx] = end - start;
     withoutFlush[idx] = afterDraw - start;
-    
+
     if (timeoutMillis && ((beginTest + timeoutMillis) < performance.now())) {
       console.log('test aborted due to timeout');
       return;

tools/perf-canvaskit-puppeteer/perf-canvaskit-with-puppeteer.js

@@ -47,6 +47,16 @@ const opts = [
     typeLabel: '{underline file}',
     description: 'The perf file to write. Defaults to perf.json',
   },
+  {
+    name: 'chromium_executable_path',
+    typeLabel: '{underline file}',
+    description: 'The chromium executable to be used by puppeteer to run tests',
+  },
+  {
+    name: 'merge_output_as',
+    typeLabel: String,
+    description: 'Overwrites a json property in an existing output file.',
+  },
   {
     name: 'use_gpu',
     description: 'Whether we should run in non-headless mode with GPU.',
@@ -58,6 +68,11 @@ const opts = [
       'measured and returned in the output JSON. Example: "blink,cc,gpu"',
     type: String,
   },
+  {
+    name: 'enable_simd',
+    description: 'enable execution of wasm SIMD operations in chromium',
+    type: Boolean
+  },
   {
     name: 'port',
     description: 'The port number to use, defaults to 8081.',
@@ -187,6 +202,9 @@ async function driveBrowser() {
       '--disable-frame-rate-limit',
       '--disable-gpu-vsync',
   ];
+  if (options.enable_simd) {
+    browser_args.push('--enable-features=WebAssemblySimd');
+  }
   if (options.use_gpu) {
     browser_args.push('--ignore-gpu-blacklist');
     browser_args.push('--ignore-gpu-blocklist');
@@ -194,7 +212,11 @@ async function driveBrowser() {
   }
   console.log("Running with headless: " + headless + " args: " + browser_args);
   try {
-    browser = await puppeteer.launch({headless: headless, args: browser_args});
+    browser = await puppeteer.launch({
+      headless: headless,
+      args: browser_args,
+      executablePath: options.chromium_executable_path
+    });
     page = await browser.newPage();
     await page.setViewport(viewPort);
   } catch (e) {
@@ -252,7 +274,19 @@ async function driveBrowser() {
     } else {
       const perfResults = await page.evaluate('window._perfData');
       console.debug('Perf results: ', perfResults);
-      fs.writeFileSync(options.output, JSON.stringify(perfResults));
+
+      if (options.merge_output_as) {
+        const existing_output_file_contents = fs.readFileSync(options.output, 'utf8');
+        let existing_dataset = {};
+        try {
+          existing_dataset = JSON.parse(existing_output_file_contents);
+        } catch (e) {}
+
+        existing_dataset[options.merge_output_as] = perfResults;
+        fs.writeFileSync(options.output, JSON.stringify(existing_dataset));
+      } else {
+        fs.writeFileSync(options.output, JSON.stringify(perfResults));
+      }
     }
 
   } catch(e) {

tools/perf-canvaskit-puppeteer/perf_all_skps.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+# Copyright 2020 Google LLC
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This script measures frametimes for CanvasKit rendering all skps in ~/skps, using puppeteer. It
+# can optionally output a human-readable summary of the collected measurements.
+# See the document "SIMD CanvasKit Build Performance Testing" for results and context:
+# https://docs.google.com/document/d/114kdSGPMnOSQCZ7pFgd3MGMn5mIW562RMoXVmD13e0M/edit?ts=5f0eedf6#
+#
+# arguments:
+# --release     perfs the release build of CanvasKit and outputs data to release_out.json
+# --simd        perfs the experimental_simd build of CanvasKit outputs data to simd_out.json
+# --summary     outputs results from the perfs in a human readable table format.
+#
+# example usage: ./perf_all_skps.sh --release --simd --summary
+
+for f in $HOME/skps/*.skp;
+do
+    if [[ "$*" == *"--release"* ]]
+    then
+        echo $f
+        node perf-canvaskit-with-puppeteer.js \
+            --canvaskit_js ../../out/canvaskit_wasm/canvaskit.js \
+            --canvaskit_wasm ../../out/canvaskit_wasm/canvaskit.wasm --use_gpu \
+            --input_skp $f \
+            --bench_html render-skp.html \
+            --chromium_executable_path "/applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary" \
+            --output release_out.json \
+            --merge_output_as `basename $f`
+    fi
+    if [[ "$*" == *"--simd"* ]]
+    then
+        node perf-canvaskit-with-puppeteer.js \
+            --canvaskit_js ../../out/canvaskit_wasm_experimental_simd/canvaskit.js \
+            --canvaskit_wasm ../../out/canvaskit_wasm_experimental_simd/canvaskit.wasm --use_gpu \
+            --input_skp $f \
+            --bench_html render-skp.html \
+            --chromium_executable_path "/applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary" \
+            --enable_simd \
+            --output simd_out.json \
+            --merge_output_as `basename $f`
+    fi
+done
+if [[ "$*" == *"--summary"* ]]
+then
+    node skp_data_prep
+fi

tools/perf-canvaskit-puppeteer/skp_data_prep.js

@@ -0,0 +1,110 @@
+/**
+ * Command line application to process the output of
+ *   make skps_release_and_SIMD
+ * and present some statistical results in a human-readable table format.
+ */
+
+const fs = require('fs');
+
+// These files are the output of `make skps_release_and_SIMD`
+const SIMD_DATA = JSON.parse(fs.readFileSync('simd_out.json', 'utf8'));
+const RELEASE_DATA = JSON.parse(fs.readFileSync('release_out.json', 'utf8'));
+
+let skp_names = new Set();
+for (const key of Object.keys(SIMD_DATA)) {
+    skp_names.add(key)
+}
+for (const key of Object.keys(RELEASE_DATA)) {
+    skp_names.add(key)
+}
+
+let simd_frame_average_accumulator = 0;
+let simd_frame_median_accumulator = 0;
+let release_frame_average_accumulator = 0;
+let release_frame_median_accumulator = 0;
+const comparisonData = [];
+
+for (const skp_name of skp_names) {
+    if (SIMD_DATA[skp_name] && RELEASE_DATA[skp_name]) {
+        // note: frames are frametimes, measured in ms
+        const simd_frames = SIMD_DATA[skp_name].total_frame_ms;
+        const simd_frames_average = averageFromArray(simd_frames);
+        const simd_frames_median = medianFromArray(simd_frames);
+        simd_frame_average_accumulator += simd_frames_average;
+        simd_frame_median_accumulator += simd_frames_median;
+
+        const release_frames = RELEASE_DATA[skp_name].total_frame_ms;
+        const release_frames_average = averageFromArray(release_frames);
+        const release_frames_median = medianFromArray(release_frames);
+        release_frame_average_accumulator += release_frames_average;
+        release_frame_median_accumulator += release_frames_median;
+
+        comparisonData.push({
+            skp_name: skp_name,
+            frames_average_difference: release_frames_average - simd_frames_average,
+            frames_median_difference: release_frames_median - simd_frames_median,
+            simd_frames_median: simd_frames_median,
+            simd_frames_average: simd_frames_average,
+            release_frames_average: release_frames_average,
+            release_frames_median: release_frames_median
+        });
+    }
+}
+const simd_average_frame = simd_frame_average_accumulator / comparisonData.length;
+const simd_skps_median_frame = simd_frame_median_accumulator / comparisonData.length;
+const release_average_frame = release_frame_average_accumulator / comparisonData.length;
+const release_median_frame = release_frame_median_accumulator / comparisonData.length;
+
+console.log('\nAverages across all SKP files');
+console.table({
+    'Average frame time average': {
+        'release CanvasKit build (ms)': release_average_frame.toFixed(2),
+        'experimental_simd CanvasKit build (ms)': simd_average_frame.toFixed(2),
+        'difference (ms)': (release_average_frame - simd_average_frame).toFixed(2)
+    },
+    'Median frame time average': {
+        'release CanvasKit build (ms)': release_median_frame.toFixed(2),
+        'experimental_simd CanvasKit build (ms)': simd_skps_median_frame.toFixed(2),
+        'difference (ms)': (release_median_frame - simd_skps_median_frame).toFixed(2)
+    }
+});
+
+const frameTimeMedianDifferenceSorted =
+    comparisonData.sort(
+        ({frames_median_difference: m1}, {frames_median_difference: m2}) => m2 - m1
+    );
+
+console.log('\nBest 3 Individual SKP frame time median differences in favor of the SIMD build');
+console.table(
+    frameTimeMedianDifferenceSorted
+    .map(tableDataFromComparisonDataObject)
+    .slice(0,3)
+);
+
+console.log('\nWorst 3 Individual SKP frame time median differences NOT in favor of the SIMD build');
+console.table(
+    frameTimeMedianDifferenceSorted
+    .map(tableDataFromComparisonDataObject)
+    .reverse().slice(0,3)
+);
+
+function averageFromArray(array) {
+    return array.reduce((a, b) => a+b, 0) / array.length;
+}
+function medianFromArray(array) {
+    return array.sort((a,b) => a-b)[Math.floor(array.length/2)];
+}
+
+function tableDataFromComparisonDataObject({
+    skp_name,
+    frames_median_difference,
+    simd_frames_median,
+    release_frames_median
+}) {
+    return {
+        '.SKP name': skp_name,
+        'release CanvasKit build (ms)': release_frames_median.toFixed(2),
+        'experimental_simd CanvasKit build (ms)': simd_frames_median.toFixed(2),
+        'difference (ms)': frames_median_difference.toFixed(2)
+    };
+}