tools/tests/run.sh: add self-test for bench_graph_svg.py

Review URL: https://codereview.appspot.com/7319044

git-svn-id: http://skia.googlecode.com/svn/trunk@7735 2bbb7eff-a529-9590-31e7-b0007b416f81

tools/tests/benchgraphs/Skia_Shuttle_Ubuntu12_ATI5770_Float_Bench_32/output-expected/command_line

@@ -0,0 +1 @@
+python bench/bench_graph_svg.py -d tools/tests/benchgraphs/Skia_Shuttle_Ubuntu12_ATI5770_Float_Bench_32/raw-bench-data -r -150 -f -150 -x 1024 -y 768 -l Title -m 25th -o tools/tests/benchgraphs/Skia_Shuttle_Ubuntu12_ATI5770_Float_Bench_32/output-actual/graph.xhtml

tools/tests/benchgraphs/Skia_Shuttle_Ubuntu12_ATI5770_Float_Bench_32/output-expected/return_value

@@ -0,0 +1 @@
+0

tools/tests/benchgraphs/Skia_Shuttle_Ubuntu12_ATI5770_Float_Bench_32/tarballs/README.sh

@@ -0,0 +1,38 @@
+# This directory contains tarballs of a subset of real performance
+# data from our production bots.
+#
+# How I created these tarballs:
+
+PLATFORM=Skia_Shuttle_Ubuntu12_ATI5770_Float_Bench_32
+TEMPDIR=$(mktemp -d)
+
+
+# DOWNLOAD SELECTED RAW DATA FROM GOOGLE STORAGE
+
+REVS="7671 7679 7686"
+for REV in $REVS; do
+
+  FILES="bench_r${REV}_data_skp_device_bitmap_multi_2_mode_tile_256_256_timeIndividualTiles bench_r${REV}_data_skp_device_bitmap_multi_3_mode_tile_256_256_timeIndividualTiles bench_r${REV}_data_skp_device_bitmap_multi_4_mode_tile_256_256_timeIndividualTiles"
+
+  for FILE in $FILES; do
+    URL=http://chromium-skia-gm.commondatastorage.googleapis.com/playback/perfdata/${PLATFORM}/data/${FILE}
+    curl $URL --output $TEMPDIR/$FILE
+  done
+
+done
+
+
+# TAR UP THE RAW DATA
+
+TARBALL_DIR="$PWD/tools/tests/benchgraphs/$PLATFORM/tarballs"
+mkdir -p $TARBALL_DIR
+pushd $TEMPDIR
+for REV in $REVS; do
+  tar --create --gzip --file ${TARBALL_DIR}/r${REV}.tgz bench_r${REV}_*
+done 
+popd
+
+
+# CLEAN UP
+
+rm -rf $TEMPDIR

tools/tests/rebaseline.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 
 # Rebaseline the skdiff/*/output-expected/ subdirectories used by the skdiff
-# self-tests.
+# self-tests, and similar for benchgraphs/*/output-expected.
+#
 # Use with caution: are you sure the new results are actually correct?
 #
 # YOU MUST RE-RUN THIS UNTIL THE SELF-TESTS SUCCEED!
@@ -14,14 +15,14 @@
 
 function replace_expected_with_actual {
   # Delete all the expected output files
-  EXPECTED_FILES=$(find skdiff/*/output-expected -type f | grep -v /\.svn/)
+  EXPECTED_FILES=$(find $WHICHTOOL/*/output-expected -type f | grep -v /\.svn/)
   for EXPECTED_FILE in $EXPECTED_FILES; do
     rm $EXPECTED_FILE
   done
 
   # Copy all the actual output files into the "expected" directories,
   # creating new subdirs as we go.
-  ACTUAL_FILES=$(find skdiff/*/output-actual -type f | grep -v /\.svn/)
+  ACTUAL_FILES=$(find $WHICHTOOL/*/output-actual -type f | grep -v /\.svn/)
   for ACTUAL_FILE in $ACTUAL_FILES; do
     EXPECTED_FILE=${ACTUAL_FILE//actual/expected}
     mkdir -p $(dirname $EXPECTED_FILE)
@@ -32,23 +33,23 @@ function replace_expected_with_actual {
 function svn_add_new_files {
   # Delete all the "actual" directories, so we can svn-add any new "expected"
   # directories without adding the "actual" ones.
-  rm -rf skdiff/*/output-actual
+  rm -rf $WHICHTOOL/*/output-actual $WHICHTOOL/*/raw-bench-data
-  FILES=$(svn stat skdiff/* | grep ^\? | awk '{print $2}')
+  FILES=$(svn stat $WHICHTOOL/* | grep ^\? | awk '{print $2}')
   for FILE in $FILES; do
     svn add $FILE
   done
-  FILES=$(svn stat skdiff/*/output-expected | grep ^\? | awk '{print $2}')
+  FILES=$(svn stat $WHICHTOOL/*/output-expected | grep ^\? | awk '{print $2}')
   for FILE in $FILES; do
     svn add $FILE
   done
 }
 
 function svn_delete_old_files {
-  FILES=$(svn stat skdiff/*/output-expected | grep ^\! | awk '{print $2}')
+  FILES=$(svn stat $WHICHTOOL/*/output-expected | grep ^\! | awk '{print $2}')
   for FILE in $FILES; do
     svn rm $FILE
   done
-  FILES=$(svn stat skdiff/* | grep ^\! | awk '{print $2}')
+  FILES=$(svn stat $WHICHTOOL/* | grep ^\! | awk '{print $2}')
   for FILE in $FILES; do
     svn rm $FILE
   done
@@ -62,9 +63,16 @@ cd $(dirname $0)
 SELFTEST_RESULT=$?
 echo
 if [ "$SELFTEST_RESULT" != "0" ]; then
+  WHICHTOOL=skdiff
+  replace_expected_with_actual
+  WHICHTOOL=benchgraphs
   replace_expected_with_actual
   echo "Self-tests still failing, you should probably run this again..."
 else
+  WHICHTOOL=skdiff
+  svn_add_new_files
+  svn_delete_old_files
+  WHICHTOOL=benchgraphs
   svn_add_new_files
   svn_delete_old_files
   echo "Self-tests succeeded this time, you should be done!"

tools/tests/run.sh

@@ -2,8 +2,6 @@
 
 # Tests for our tools.
 #
-# TODO: for now, it only tests skdiff
-#
 # TODO: currently, this only passes on Linux (which is the platform that
 # the housekeeper bot runs on, e.g.
 # http://70.32.156.51:10117/builders/Skia_PerCommit_House_Keeping/builds/1415/steps/RunToolSelfTests/logs/stdio )
@@ -77,4 +75,47 @@ skdiff_test "--nodiffs --match identical-bits $SKDIFF_TESTDIR/baseDir $SKDIFF_TE
 # Run skdiff over just the files that have identical bits or identical pixels.
 skdiff_test "--nodiffs --match identical-bits --match identical-pixels $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/identical-bits-or-pixels"
 
+
+# Run bench_graph_svg.py across the data from platform $1,
+# writing its output to output-actual and comparing those results against
+# output-expected.
+function benchgraph_test {
+  if [ $# != 1 ]; then
+    echo "benchgraph_test requires exactly 1 parameter, got $#"
+    exit 1
+  fi
+  PLATFORM="$1"
+
+  PLATFORM_DIR="tools/tests/benchgraphs/$PLATFORM"
+  TARBALL_DIR="$PLATFORM_DIR/tarballs"
+  RAW_BENCH_DATA_DIR="$PLATFORM_DIR/raw-bench-data"
+  ACTUAL_OUTPUT_DIR="$PLATFORM_DIR/output-actual"
+  EXPECTED_OUTPUT_DIR="$PLATFORM_DIR/output-expected"
+
+  # First, unpack raw bench data from tarballs.
+  # (The raw bench data files are large, so this saves space in our SVN repo.)
+  rm -rf $RAW_BENCH_DATA_DIR
+  mkdir -p $RAW_BENCH_DATA_DIR
+  for TARBALL in $TARBALL_DIR/*.tgz ; do
+    tar --extract --gunzip --directory $RAW_BENCH_DATA_DIR --file $TARBALL
+  done
+
+  # Now that we have the input files we need, run bench_graph_svg.py .
+  rm -rf $ACTUAL_OUTPUT_DIR
+  mkdir -p $ACTUAL_OUTPUT_DIR
+  COMMAND="python bench/bench_graph_svg.py -d $RAW_BENCH_DATA_DIR -r -150 -f -150 -x 1024 -y 768 -l Title -m 25th -o $ACTUAL_OUTPUT_DIR/graph.xhtml"
+  echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line
+  START_TIMESTAMP=$(date +%s)
+  $COMMAND &>$ACTUAL_OUTPUT_DIR/stdout
+  echo $? >$ACTUAL_OUTPUT_DIR/return_value
+  END_TIMESTAMP=$(date +%s)
+
+  SECONDS_RUN=$(expr $END_TIMESTAMP - $START_TIMESTAMP)
+  echo "bench_graph_svg.py for $PLATFORM took $SECONDS_RUN seconds to complete"
+
+  compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR
+}
+
+benchgraph_test Skia_Shuttle_Ubuntu12_ATI5770_Float_Bench_32
+
 echo "All tests passed."