SkQP: Cloud Scripts

- tools/skqp/generate_gn_args.sh - single script to generate
    configuration used to build SkQP.
  - tools/skqp/make_model.sh - script to execute make_gmkb.go and put
    the results in the correct place.
  - tools/skqp/{up,down}load_directory_contents.sh - scripts to move
    large assets into the cloud and retrieve them.
  - tools/skqp/gm_runner.cpp - treat missing text files as empty.
  - tools/skqp/make_apk.sh - make use of generate_gn_args.sh.
  - tools/skqp/make_{model,known_tests}.sh - better error handling,
    and also stop calling `git add`.

Change-Id: Ib1803b4d68e63945f2c47f8eb6cb96375d24e3be
Reviewed-on: https://skia-review.googlesource.com/98842
Reviewed-by: Hal Canary <halcanary@google.com>
Commit-Queue: Hal Canary <halcanary@google.com>

platform_tools/android/apps/skqp/src/main/assets/.gitignore

@@ -0,0 +1,3 @@
+/gmkb
+/skqp
+/files.txt

tools/skqp/README.md

@@ -20,14 +20,8 @@ How To Use SkQP on your Android device:
 
         arch='arm64'  # Also valid: 'arm', 'x68', 'x64'
         android_ndk="${HOME}/android-ndk"  # Or wherever you installed the NDK.
-        mkdir -p out/${arch}-rel
+
-        cat > out/${arch}-rel/args.gn << EOF
+        tools/skqp/generate_gn_args.sh out/${arch}-rel "$android_ndk" $arch
-            ndk = "$android_ndk"
-            ndk_api = 26
-            target_cpu = "$arch"
-            skia_embed_resources = true
-            is_debug = false
-        EOF
         tools/git-sync-deps
         bin/gn gen out/${arch}-rel
         ninja -C out/${arch}-rel skqp_lib
@@ -37,9 +31,7 @@ How To Use SkQP on your Android device:
 
 5.  Generate the validation model data:
 
-        go get -u go.skia.org/infra/golden/go/search
+        tools/skqp/make_model.sh ~/Downloads/meta.json
-        go run tools/skqp/make_gmkb.go ~/Downloads/meta.json \
-            platform_tools/android/apps/skqp/src/main/assets/gmkb
 
 Run as an executable
 --------------------

tools/skqp/download_directory_contents.sh

@@ -0,0 +1,39 @@
+#! /bin/sh
+
+# Copyright 2018 Google Inc.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -e
+
+cd "$(dirname "$0")/../../platform_tools/android/apps/skqp/src/main/assets"
+
+checksum() {
+    [ -f "$1" ] && { md5sum < "$1" | head -c 32; }
+}
+
+download() {
+    if ! [ $1 = "$(checksum "$2")" ]; then
+        mkdir -p "$(dirname "$2")"
+        curl -s -o "$2" "https://storage.googleapis.com/skia-skqp-assets/$1"
+    fi
+}
+
+download $(cat files.checksum) files.txt
+
+COUNT=$(wc -l < files.txt)
+INDEX=1
+SHARD_COUNT=32
+
+cat files.txt | while IFS= read -r LINE; do
+    MD5=$(echo $LINE | awk -F\; '{print $1}')
+    FILENAME=$(echo $LINE | awk -F\; '{print $2}')
+    download $MD5 "$FILENAME" &
+    if [ $(($INDEX % $SHARD_COUNT)) = 0 ]; then
+        wait
+        printf '\r %d / %d ' "$INDEX" "$COUNT"
+    fi
+    INDEX=$(($INDEX + 1))
+done
+printf '\rdone                \n'
+

tools/skqp/generate_gn_args.sh

@@ -0,0 +1,30 @@
+#! /bin/sh
+
+# Copyright 2018 Google Inc.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+BUILD="$1"
+ANDROID_NDK="$2"
+ARCH="$3"
+
+if [ $# -ne 3 ] || [ -z "$BUILD" ] || ! [ -d "$ANDROID_NDK" ] || [ -z "$ARCH" ]; then
+    printf 'Usage:\n  %s TARGET_BUILD_DIR ANDROID_NDK_DIR ARCHITECTURE\n\n' "$0" >&2
+    printf 'ARCHITECTURE should be "arm" "arm64" "x86" or "x64"\n\n' >&2
+    exit 1
+fi
+
+ANDROID_NDK="$(cd "$ANDROID_NDK"; pwd)"
+
+mkdir -p "$BUILD"
+
+cat > "$BUILD/args.gn" << EOF
+ndk = "$ANDROID_NDK"
+ndk_api = 26
+target_cpu = "$ARCH"
+skia_embed_resources = true
+is_debug = false
+skia_enable_pdf = false
+EOF
+
+

tools/skqp/gm_runner.cpp

@@ -37,9 +37,8 @@ static bool in_set(const char* s, const SkTHashSet<SkString>& set) {
 
 static void readlist(skqp::AssetManager* mgr, const char* path, SkTHashSet<SkString>* dst) {
     auto asset = mgr->open(path);
-    SkASSERT_RELEASE(asset);
     if (!asset || asset->getLength() == 0) {
-        return;
+        return;  // missing file same as empty file.
     }
     std::vector<char> buffer(asset->getLength() + 1);
     asset->read(buffer.data(), buffer.size());

tools/skqp/make_apk.sh

@@ -31,26 +31,19 @@ ARCH=${SKQP_ARCH:-arm}
 
 cd "$(dirname "$0")/../.."
 
-mkdir -p out/skqp-${ARCH}
+BUILD=out/skqp-${ARCH}
 
-cat > out/skqp-${ARCH}/args.gn << EOF
+tools/skqp/generate_gn_args.sh $BUILD "$ANDROID_NDK" $ARCH
-  ndk = "$ANDROID_NDK"
-  ndk_api = 26
-  target_cpu = "${ARCH}"
-  skia_embed_resources = true
-  is_debug = false
-  skia_enable_pdf = false
-EOF
 
 GIT_SYNC_DEPS_QUIET=Y tools/git-sync-deps
 
-bin/gn gen out/skqp-${ARCH}
+bin/gn gen $BUILD
 
-rm -rf out/skqp-${ARCH}/gen
+rm -rf $BUILD/gen
 
-platform_tools/android/bin/android_build_app -C out/skqp-${ARCH} skqp
+platform_tools/android/bin/android_build_app -C $BUILD skqp
 
 set +x
 
-printf '\n\nAPK built: "%s/skqp.apk"\n\n' "$(pwd)/out/skqp-${ARCH}"
+printf '\n\nAPK built: "%s/skqp.apk"\n\n' "$(pwd)/$BUILD"
 

tools/skqp/make_known_tests.sh

@@ -6,14 +6,21 @@
 
 set -e -x
 
-tools/git-sync-deps
+cd "$(dirname "$0")/../.."
-bin/bn gen out/default
+
-ninja -C out/default list_gms list_gpu_unit_tests
+BUILD=out/default
-out/default/list_gms > \
+
-    platform_tools/android/apps/skqp/src/main/assets/skqp/KnownGMs.txt
+python tools/git-sync-deps
-out/default/list_gpu_unit_tests > \
+
-    platform_tools/android/apps/skqp/src/main/assets/skqp/KnownGpuUnitTests.txt
+bin/gn gen $BUILD
-git add \
+
-    platform_tools/android/apps/skqp/src/main/assets/skqp/KnownGMs.txt \
+ninja -C $BUILD list_gms list_gpu_unit_tests
-    platform_tools/android/apps/skqp/src/main/assets/skqp/KnownGpuUnitTests.txt
+
+DIR=platform_tools/android/apps/skqp/src/main/assets/skqp
+
+mkdir -p $DIR
+
+$BUILD/list_gms > $DIR/KnownGMs.txt
+
+$BUILD/list_gpu_unit_tests > $DIR/KnownGpuUnitTests.txt
 

tools/skqp/make_model.sh

@@ -1,15 +1,20 @@
 #! /bin/sh
+
 # Copyright 2018 Google Inc.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+if ! [ -f "$1" ]; then
+    printf 'Usage:\n  %s META_JSON_FILE_PATH\n\n' "$0" >&2
+    exit 1
+fi
+
 set -e -x
 
-test -f "$1"
+SKIA="$(dirname "$0")/../.."
 
 go get -u go.skia.org/infra/golden/go/search
 
-go run tools/skqp/make_gmkb.go "$1" platform_tools/android/apps/skqp/src/main/assets/gmkb
+go run "${SKIA}/tools/skqp/make_gmkb.go" "$1" \
-
+    "${SKIA}/platform_tools/android/apps/skqp/src/main/assets/gmkb"
-git add platform_tools/android/apps/skqp/src/main/assets/gmkb
 

tools/skqp/upload_directory_contents.sh

@@ -0,0 +1,45 @@
+#! /bin/sh
+
+# Copyright 2018 Google Inc.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -e
+
+EXTANT="$(mktemp "${TMPDIR:-/tmp}/extant.XXXXXXXXXX")"
+gsutil ls gs://skia-skqp-assets/ | sed 's|^gs://skia-skqp-assets/||'  > "$EXTANT"
+
+upload() {
+    MD5=$(md5sum < "$1" | head -c 32)
+    if ! grep -q "$MD5" "$EXTANT"; then
+        URL="gs://skia-skqp-assets/$MD5"
+        gsutil cp "$1" "$URL" > /dev/null 2>&1
+    fi
+    echo $MD5
+}
+
+cd "$(dirname "$0")/../../platform_tools/android/apps/skqp/src/main/assets"
+
+rm -f files.checksum
+
+FILES="$(mktemp "${TMPDIR:-/tmp}/files.XXXXXXXXXX")"
+
+: > "$FILES"
+
+COUNT=$(find * -type f | wc -l)
+INDEX=1
+find * -type f | sort | while IFS= read -r FILENAME; do
+    printf '\r %d / %d   ' "$INDEX" "$COUNT"
+    INDEX=$(( $INDEX + 1))
+    if ! [ -f "$FILENAME" ]; then
+        echo error [${FILENAME}] >&2;
+        exit 1;
+    fi
+    case "$FILENAME" in *\;*) echo bad filename: $FILENAME >&2; exit 1;; esac
+    MD5=$(upload "$FILENAME")
+    printf '%s;%s\n' "$MD5" "$FILENAME" >> "$FILES"
+done
+
+upload "$FILES" > files.checksum
+
+rm "$FILES" "$EXTANT"