From ce243ba1099127faf6367ae66e32b9e83c8ecc76 Mon Sep 17 00:00:00 2001
From: Hal Canary <halcanary@google.com>
Date: Tue, 30 Jan 2018 16:08:17 -0500
Subject: [PATCH] SkQP: fix upload/download scripts

Change-Id: If2b2ec786630e1ecea159e0804547a79478138f5
Reviewed-on: https://skia-review.googlesource.com/101844
Reviewed-by: Hal Canary <halcanary@google.com>
Commit-Queue: Hal Canary <halcanary@google.com>
---
 tools/skqp/download_model | 11 +++++++++--
 tools/skqp/upload_model   | 22 ++++++++++++++++------
 2 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/tools/skqp/download_model b/tools/skqp/download_model
index de1aeeddc6..90a46256f6 100755
--- a/tools/skqp/download_model
+++ b/tools/skqp/download_model
@@ -9,6 +9,7 @@ import os
 import shutil
 import sys
 import threading
+import tempfile
 import urllib2
 
 def checksum(path):
@@ -36,16 +37,22 @@ def download(md5, path):
         with open(path, 'wb') as o:
             shutil.copyfileobj(urllib2.urlopen(url), o)
 
+def tmp(prefix):
+    fd, path = tempfile.mkstemp(prefix=prefix)
+    os.close(fd)
+    return path
+
 def main():
     os.chdir(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
                          'platform_tools', 'android', 'apps', 'skqp', 'src',
                          'main', 'assets'))
     sys.stderr.write('\n    0 / ????   ')
+    file_list_file = tmp('files_')
     with open('files.checksum', 'r') as f:
         md5 = f.read().strip()
         assert(len(md5) == 32)
-        download(md5, 'files.txt')
-    with open('files.txt', 'r') as f:
+        download(md5, file_list_file)
+    with open(file_list_file, 'r') as f:
         records = []
         for line in f:
             md5, path = line.strip().split(';', 1)
diff --git a/tools/skqp/upload_model b/tools/skqp/upload_model
index c7f294e69b..cd9554c671 100755
--- a/tools/skqp/upload_model
+++ b/tools/skqp/upload_model
@@ -6,19 +6,25 @@
 
 set -e
 
+BASE_DIR='platform_tools/android/apps/skqp/src/main/assets'
+PATH_LIST='gmkb skqp'
+BUCKET=skia-skqp-assets
+
 EXTANT="$(mktemp "${TMPDIR:-/tmp}/extant.XXXXXXXXXX")"
-gsutil ls gs://skia-skqp-assets/ | sed 's|^gs://skia-skqp-assets/||'  > "$EXTANT"
+gsutil ls gs://$BUCKET/ | sed "s|^gs://$BUCKET/||"  > "$EXTANT"
 
 upload() {
     MD5=$(md5sum < "$1" | head -c 32)
     if ! grep -q "$MD5" "$EXTANT"; then
-        URL="gs://skia-skqp-assets/$MD5"
+        URL="gs://${BUCKET}/$MD5"
         gsutil cp "$1" "$URL" > /dev/null 2>&1 &
     fi
     echo $MD5
 }
 
-cd "$(dirname "$0")/../../platform_tools/android/apps/skqp/src/main/assets"
+size() { gsutil du -s gs://$BUCKET | awk '{print $1}'; }
+
+cd "$(dirname "$0")/../../$BASE_DIR"
 
 rm -f files.checksum
 
@@ -26,11 +32,12 @@ FILES="$(mktemp "${TMPDIR:-/tmp}/files.XXXXXXXXXX")"
 
 : > "$FILES"
 
-COUNT=$(find * -type f | wc -l)
+COUNT=$(find $PATH_LIST -type f | wc -l)
 INDEX=1
 SHARD_COUNT=32
 
-find * -type f | sort | while IFS= read -r FILENAME; do
+SIZE=$(size)
+find $PATH_LIST -type f | sort | while IFS= read -r FILENAME; do
     printf '\r %d / %d   ' "$INDEX" "$COUNT"
     if ! [ -f "$FILENAME" ]; then
         echo error [${FILENAME}] >&2;
@@ -49,4 +56,7 @@ printf '\rdone          \n'
 upload "$FILES" > files.checksum
 wait
 
-rm "$FILES" "$EXTANT"
+D=$(( $(size) - $SIZE ))
+printf 'Added %d bytes to %s, %d%%\n' $D $BUCKET $(( $D * 100 / $SIZE ))
+
+rm "$EXTANT"