AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

SkQP: fix upload/download scripts

Browse Source 
Change-Id: If2b2ec786630e1ecea159e0804547a79478138f5
Reviewed-on: https://skia-review.googlesource.com/101844
Reviewed-by: Hal Canary <halcanary@google.com>
Commit-Queue: Hal Canary <halcanary@google.com>
This commit is contained in:
Hal Canary 2018-01-30 16:08:17 -05:00 committed by Skia Commit-Bot
parent 57b0430e19
commit ce243ba109
2 changed files with 25 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
tools/skqp/download_model
View File

@ -9,6 +9,7 @@ import os
import shutil
import sys
import threading
import tempfile
import urllib2
def checksum(path):
@ -36,16 +37,22 @@ def download(md5, path):
with open(path, 'wb') as o:
shutil.copyfileobj(urllib2.urlopen(url), o)
def tmp(prefix):
fd, path = tempfile.mkstemp(prefix=prefix)
os.close(fd)
return path
def main():
os.chdir(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
'platform_tools', 'android', 'apps', 'skqp', 'src',
'main', 'assets'))
sys.stderr.write('\n 0 / ???? ')
file_list_file = tmp('files_')
with open('files.checksum', 'r') as f:
md5 = f.read().strip()
assert(len(md5) == 32)
download(md5, 'files.txt')
with open('files.txt', 'r') as f:
download(md5, file_list_file)
with open(file_list_file, 'r') as f:
records = []
for line in f:
md5, path = line.strip().split(';', 1)

22
tools/skqp/upload_model
View File

@ -6,19 +6,25 @@
set -e
BASE_DIR='platform_tools/android/apps/skqp/src/main/assets'
PATH_LIST='gmkb skqp'
BUCKET=skia-skqp-assets
EXTANT="$(mktemp "${TMPDIR:-/tmp}/extant.XXXXXXXXXX")"
gsutil ls gs://skia-skqp-assets/ | sed 's|^gs://skia-skqp-assets/||' > "$EXTANT"
gsutil ls gs://$BUCKET/ | sed "s|^gs://$BUCKET/||" > "$EXTANT"
upload() {
MD5=$(md5sum < "$1" | head -c 32)
if ! grep -q "$MD5" "$EXTANT"; then
URL="gs://skia-skqp-assets/$MD5"
URL="gs://${BUCKET}/$MD5"
gsutil cp "$1" "$URL" > /dev/null 2>&1 &
fi
echo $MD5
}
cd "$(dirname "$0")/../../platform_tools/android/apps/skqp/src/main/assets"
size() { gsutil du -s gs://$BUCKET | awk '{print $1}'; }
cd "$(dirname "$0")/../../$BASE_DIR"
rm -f files.checksum
@ -26,11 +32,12 @@ FILES="$(mktemp "${TMPDIR:-/tmp}/files.XXXXXXXXXX")"
: > "$FILES"
COUNT=$(find * -type f | wc -l)
COUNT=$(find $PATH_LIST -type f | wc -l)
INDEX=1
SHARD_COUNT=32
find * -type f | sort | while IFS= read -r FILENAME; do
SIZE=$(size)
find $PATH_LIST -type f | sort | while IFS= read -r FILENAME; do
printf '\r %d / %d ' "$INDEX" "$COUNT"
if ! [ -f "$FILENAME" ]; then
echo error [${FILENAME}] >&2;
@ -49,4 +56,7 @@ printf '\rdone \n'
upload "$FILES" > files.checksum
wait
rm "$FILES" "$EXTANT"
D=$(( $(size) - $SIZE ))
printf 'Added %d bytes to %s, %d%%\n' $D $BUCKET $(( $D * 100 / $SIZE ))
rm "$EXTANT"