From 19e80c16a5bac6969bd39ceaa2f63d2e4e664a11 Mon Sep 17 00:00:00 2001 From: "epoger@google.com" Date: Thu, 6 Jun 2013 15:11:11 +0000 Subject: [PATCH] rebaseline.py self-tests: split test inputs and outputs into completely separate directories Review URL: https://codereview.chromium.org/16557002 git-svn-id: http://skia.googlecode.com/svn/trunk@9459 2bbb7eff-a529-9590-31e7-b0007b416f81 --- tools/tests/rebaseline.sh | 45 +++++++++++++------ .../actual-results.json | 0 .../base-macmini/actual-results.json | 0 .../actual-results.json | 0 .../all/output-expected/command_line | 0 .../all/output-expected/return_value | 0 .../{ => output}/all/output-expected/stdout | 0 .../subset/output-expected/command_line | 0 .../subset/output-expected/return_value | 0 .../subset/output-expected/stdout | 0 .../using-json1/output-expected/command_line | 1 + .../using-json1}/output-expected/return_value | 0 .../using-json1}/output-expected/stdout | 4 +- .../using-json/output-expected/command_line | 1 - tools/tests/run.sh | 9 ++-- 15 files changed, 40 insertions(+), 20 deletions(-) rename tools/tests/rebaseline/{using-json => input/json1}/base-android-galaxy-nexus/Test-Android-GalaxyNexus-SGX540-Arm7-Debug/base-android-galaxy-nexus/actual-results.json (100%) rename tools/tests/rebaseline/{using-json => input/json1}/base-macmini/Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release/base-macmini/actual-results.json (100%) rename tools/tests/rebaseline/{using-json => input/json1}/base-shuttle-win7-intel-float/Test-Win7-ShuttleA-HD2000-x86-Release/base-shuttle-win7-intel-float/actual-results.json (100%) rename tools/tests/rebaseline/{ => output}/all/output-expected/command_line (100%) rename tools/tests/rebaseline/{ => output}/all/output-expected/return_value (100%) rename tools/tests/rebaseline/{ => output}/all/output-expected/stdout (100%) rename tools/tests/rebaseline/{ => output}/subset/output-expected/command_line (100%) rename tools/tests/rebaseline/{ => output}/subset/output-expected/return_value (100%) rename tools/tests/rebaseline/{ => output}/subset/output-expected/stdout (100%) create mode 100644 tools/tests/rebaseline/output/using-json1/output-expected/command_line rename tools/tests/rebaseline/{using-json => output/using-json1}/output-expected/return_value (100%) rename tools/tests/rebaseline/{using-json => output/using-json1}/output-expected/stdout (85%) delete mode 100644 tools/tests/rebaseline/using-json/output-expected/command_line diff --git a/tools/tests/rebaseline.sh b/tools/tests/rebaseline.sh index 74e846243d..38f19499ed 100755 --- a/tools/tests/rebaseline.sh +++ b/tools/tests/rebaseline.sh @@ -13,16 +13,22 @@ # See https://code.google.com/p/skia/issues/detail?id=677 # ('make tools/tests/run.sh work cross-platform') +# Replace expected output with actual output, within subdir $1. function replace_expected_with_actual { + if [ $# != 1 ]; then + echo "replace_expected_with_actual requires exactly 1 parameter, got $#" + exit 1 + fi + # Delete all the expected output files - EXPECTED_FILES=$(find $WHICHTOOL/*/output-expected -type f | grep -v /\.svn/) + EXPECTED_FILES=$(find $1/*/output-expected -type f | grep -v /\.svn/) for EXPECTED_FILE in $EXPECTED_FILES; do rm $EXPECTED_FILE done # Copy all the actual output files into the "expected" directories, # creating new subdirs as we go. - ACTUAL_FILES=$(find $WHICHTOOL/*/output-actual -type f | grep -v /\.svn/) + ACTUAL_FILES=$(find $1/*/output-actual -type f | grep -v /\.svn/) for ACTUAL_FILE in $ACTUAL_FILES; do EXPECTED_FILE=${ACTUAL_FILE//actual/expected} mkdir -p $(dirname $EXPECTED_FILE) @@ -30,26 +36,39 @@ function replace_expected_with_actual { done } +# Add all new files to SVN control, within subdir $1. function svn_add_new_files { + if [ $# != 1 ]; then + echo "svn_add_new_files requires exactly 1 parameter, got $#" + exit 1 + fi + # Delete all the "actual" directories, so we can svn-add any new "expected" # directories without adding the "actual" ones. - rm -rf $WHICHTOOL/*/output-actual $WHICHTOOL/*/raw-bench-data - FILES=$(svn stat $WHICHTOOL/* | grep ^\? | awk '{print $2}') + rm -rf $1/*/output-actual $1/*/raw-bench-data + FILES=$(svn stat $1/* | grep ^\? | awk '{print $2}') for FILE in $FILES; do svn add $FILE done - FILES=$(svn stat $WHICHTOOL/*/output-expected | grep ^\? | awk '{print $2}') + FILES=$(svn stat $1/*/output-expected | grep ^\? | awk '{print $2}') for FILE in $FILES; do svn add $FILE done } +# For any files that have been removed from subdir $1, remove them from +# SVN control. function svn_delete_old_files { - FILES=$(svn stat $WHICHTOOL/*/output-expected | grep ^\! | awk '{print $2}') + if [ $# != 1 ]; then + echo "svn_delete_old_files requires exactly 1 parameter, got $#" + exit 1 + fi + + FILES=$(svn stat $1/*/output-expected | grep ^\! | awk '{print $2}') for FILE in $FILES; do svn rm $FILE done - FILES=$(svn stat $WHICHTOOL/* | grep ^\! | awk '{print $2}') + FILES=$(svn stat $1/* | grep ^\! | awk '{print $2}') for FILE in $FILES; do svn rm $FILE done @@ -61,17 +80,17 @@ cd $(dirname $0) ./run.sh SELFTEST_RESULT=$? -TOOLS="skdiff benchgraphs rebaseline" +SUBDIRS="skdiff benchgraphs rebaseline/output" echo if [ "$SELFTEST_RESULT" != "0" ]; then - for WHICHTOOL in $TOOLS; do - replace_expected_with_actual + for SUBDIR in $SUBDIRS; do + replace_expected_with_actual $SUBDIR done echo "Self-tests still failing, you should probably run this again..." else - for WHICHTOOL in $TOOLS; do - svn_add_new_files - svn_delete_old_files + for SUBDIR in $SUBDIRS; do + svn_add_new_files $SUBDIR + svn_delete_old_files $SUBDIR done echo "Self-tests succeeded this time, you should be done!" fi diff --git a/tools/tests/rebaseline/using-json/base-android-galaxy-nexus/Test-Android-GalaxyNexus-SGX540-Arm7-Debug/base-android-galaxy-nexus/actual-results.json b/tools/tests/rebaseline/input/json1/base-android-galaxy-nexus/Test-Android-GalaxyNexus-SGX540-Arm7-Debug/base-android-galaxy-nexus/actual-results.json similarity index 100% rename from tools/tests/rebaseline/using-json/base-android-galaxy-nexus/Test-Android-GalaxyNexus-SGX540-Arm7-Debug/base-android-galaxy-nexus/actual-results.json rename to tools/tests/rebaseline/input/json1/base-android-galaxy-nexus/Test-Android-GalaxyNexus-SGX540-Arm7-Debug/base-android-galaxy-nexus/actual-results.json diff --git a/tools/tests/rebaseline/using-json/base-macmini/Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release/base-macmini/actual-results.json b/tools/tests/rebaseline/input/json1/base-macmini/Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release/base-macmini/actual-results.json similarity index 100% rename from tools/tests/rebaseline/using-json/base-macmini/Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release/base-macmini/actual-results.json rename to tools/tests/rebaseline/input/json1/base-macmini/Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release/base-macmini/actual-results.json diff --git a/tools/tests/rebaseline/using-json/base-shuttle-win7-intel-float/Test-Win7-ShuttleA-HD2000-x86-Release/base-shuttle-win7-intel-float/actual-results.json b/tools/tests/rebaseline/input/json1/base-shuttle-win7-intel-float/Test-Win7-ShuttleA-HD2000-x86-Release/base-shuttle-win7-intel-float/actual-results.json similarity index 100% rename from tools/tests/rebaseline/using-json/base-shuttle-win7-intel-float/Test-Win7-ShuttleA-HD2000-x86-Release/base-shuttle-win7-intel-float/actual-results.json rename to tools/tests/rebaseline/input/json1/base-shuttle-win7-intel-float/Test-Win7-ShuttleA-HD2000-x86-Release/base-shuttle-win7-intel-float/actual-results.json diff --git a/tools/tests/rebaseline/all/output-expected/command_line b/tools/tests/rebaseline/output/all/output-expected/command_line similarity index 100% rename from tools/tests/rebaseline/all/output-expected/command_line rename to tools/tests/rebaseline/output/all/output-expected/command_line diff --git a/tools/tests/rebaseline/all/output-expected/return_value b/tools/tests/rebaseline/output/all/output-expected/return_value similarity index 100% rename from tools/tests/rebaseline/all/output-expected/return_value rename to tools/tests/rebaseline/output/all/output-expected/return_value diff --git a/tools/tests/rebaseline/all/output-expected/stdout b/tools/tests/rebaseline/output/all/output-expected/stdout similarity index 100% rename from tools/tests/rebaseline/all/output-expected/stdout rename to tools/tests/rebaseline/output/all/output-expected/stdout diff --git a/tools/tests/rebaseline/subset/output-expected/command_line b/tools/tests/rebaseline/output/subset/output-expected/command_line similarity index 100% rename from tools/tests/rebaseline/subset/output-expected/command_line rename to tools/tests/rebaseline/output/subset/output-expected/command_line diff --git a/tools/tests/rebaseline/subset/output-expected/return_value b/tools/tests/rebaseline/output/subset/output-expected/return_value similarity index 100% rename from tools/tests/rebaseline/subset/output-expected/return_value rename to tools/tests/rebaseline/output/subset/output-expected/return_value diff --git a/tools/tests/rebaseline/subset/output-expected/stdout b/tools/tests/rebaseline/output/subset/output-expected/stdout similarity index 100% rename from tools/tests/rebaseline/subset/output-expected/stdout rename to tools/tests/rebaseline/output/subset/output-expected/stdout diff --git a/tools/tests/rebaseline/output/using-json1/output-expected/command_line b/tools/tests/rebaseline/output/using-json1/output-expected/command_line new file mode 100644 index 0000000000..d905e61e61 --- /dev/null +++ b/tools/tests/rebaseline/output/using-json1/output-expected/command_line @@ -0,0 +1 @@ +python tools/rebaseline.py --dry_run --json_base_url file:tools/tests/rebaseline/input/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float diff --git a/tools/tests/rebaseline/using-json/output-expected/return_value b/tools/tests/rebaseline/output/using-json1/output-expected/return_value similarity index 100% rename from tools/tests/rebaseline/using-json/output-expected/return_value rename to tools/tests/rebaseline/output/using-json1/output-expected/return_value diff --git a/tools/tests/rebaseline/using-json/output-expected/stdout b/tools/tests/rebaseline/output/using-json1/output-expected/stdout similarity index 85% rename from tools/tests/rebaseline/using-json/output-expected/stdout rename to tools/tests/rebaseline/output/using-json1/output-expected/stdout index 58814e0bbb..c87bbed191 100644 --- a/tools/tests/rebaseline/using-json/output-expected/stdout +++ b/tools/tests/rebaseline/output/using-json1/output-expected/stdout @@ -1,4 +1,4 @@ -# Getting files to rebaseline from JSON summary URL file:tools/tests/rebaseline/using-json/base-android-galaxy-nexus/Test-Android-GalaxyNexus-SGX540-Arm7-Debug/base-android-galaxy-nexus/actual-results.json ... +# Getting files to rebaseline from JSON summary URL file:tools/tests/rebaseline/input/json1/base-android-galaxy-nexus/Test-Android-GalaxyNexus-SGX540-Arm7-Debug/base-android-galaxy-nexus/actual-results.json ... # ... found files_to_rebaseline [u'imageblur_565.png', u'imageblur_8888.png', u'shadertext3_8888.png'] # imageblur_565.png curl --fail --silent http://skia-autogen.googlecode.com/svn/gm-actual/base-android-galaxy-nexus/Test-Android-GalaxyNexus-SGX540-Arm7-Debug/base-android-galaxy-nexus/imageblur_565.png --output base-android-galaxy-nexus/.temp-imageblur_565.png @@ -15,5 +15,5 @@ curl --fail --silent http://skia-autogen.googlecode.com/svn/gm-actual/base-andro mv base-android-galaxy-nexus/.temp-shadertext3_8888.png base-android-galaxy-nexus/shadertext3_8888.png svn add --quiet base-android-galaxy-nexus/shadertext3_8888.png svn propset --quiet svn:mime-type image/png base-android-galaxy-nexus/shadertext3_8888.png -# Getting files to rebaseline from JSON summary URL file:tools/tests/rebaseline/using-json/base-shuttle-win7-intel-float/Test-Win7-ShuttleA-HD2000-x86-Release/base-shuttle-win7-intel-float/actual-results.json ... +# Getting files to rebaseline from JSON summary URL file:tools/tests/rebaseline/input/json1/base-shuttle-win7-intel-float/Test-Win7-ShuttleA-HD2000-x86-Release/base-shuttle-win7-intel-float/actual-results.json ... # ... found files_to_rebaseline [] diff --git a/tools/tests/rebaseline/using-json/output-expected/command_line b/tools/tests/rebaseline/using-json/output-expected/command_line deleted file mode 100644 index c96194641f..0000000000 --- a/tools/tests/rebaseline/using-json/output-expected/command_line +++ /dev/null @@ -1 +0,0 @@ -python tools/rebaseline.py --dry_run --json_base_url file:tools/tests/rebaseline/using-json --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float diff --git a/tools/tests/run.sh b/tools/tests/run.sh index af1a35a629..940f59732f 100755 --- a/tools/tests/run.sh +++ b/tools/tests/run.sh @@ -194,10 +194,11 @@ benchgraph_test $PLATFORM # Test rebaseline.py ... # -REBASELINE_TESTDIR=tools/tests/rebaseline -rebaseline_test "--tests test1 test2 --configs 565 8888 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_TESTDIR/subset" -rebaseline_test "--tests test1 test2" "$REBASELINE_TESTDIR/all" -rebaseline_test "--json_base_url file:$REBASELINE_TESTDIR/using-json --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_TESTDIR/using-json" +REBASELINE_INPUT=tools/tests/rebaseline/input +REBASELINE_OUTPUT=tools/tests/rebaseline/output +rebaseline_test "--tests test1 test2 --configs 565 8888 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/subset" +rebaseline_test "--tests test1 test2" "$REBASELINE_OUTPUT/all" +rebaseline_test "--json_base_url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/using-json1" echo "All tests passed."