diff --git a/gm/gm_expectations.cpp b/gm/gm_expectations.cpp index 2f921201fa..ff6fc2c70f 100644 --- a/gm/gm_expectations.cpp +++ b/gm/gm_expectations.cpp @@ -109,6 +109,20 @@ namespace skiagm { return jsonTypeValuePair; } + SkString GmResultDigest::getHashType() const { + // TODO(epoger): The current implementation assumes that the + // result digest is always of type kJsonKey_Hashtype_Bitmap_64bitMD5 + return SkString(kJsonKey_Hashtype_Bitmap_64bitMD5); + } + + SkString GmResultDigest::getDigestValue() const { + // TODO(epoger): The current implementation assumes that the + // result digest is always of type kJsonKey_Hashtype_Bitmap_64bitMD5 + SkString retval; + retval.appendU64(fHashDigest); + return retval; + } + // Expectations class... diff --git a/gm/gm_expectations.h b/gm/gm_expectations.h index 55122d4ab8..ec6899794a 100644 --- a/gm/gm_expectations.h +++ b/gm/gm_expectations.h @@ -76,11 +76,32 @@ namespace skiagm { */ Json::Value asJsonTypeValuePair() const; + /** + * Returns the hashtype, such as "bitmap-64bitMD5", as an SkString. + */ + SkString getHashType() const; + + /** + * Returns the hash digest value, such as "12345", as an SkString. + */ + SkString getDigestValue() const; + private: bool fIsValid; // always check this first--if it's false, other fields are meaningless uint64_t fHashDigest; }; + /** + * Encapsulates an SkBitmap and its GmResultDigest, guaranteed to keep them in sync. + */ + class BitmapAndDigest { + public: + BitmapAndDigest(const SkBitmap &bitmap) : fBitmap(bitmap), fDigest(bitmap) {} + + const SkBitmap fBitmap; + const GmResultDigest fDigest; + }; + /** * Test expectations (allowed image results, etc.) */ diff --git a/gm/gmmain.cpp b/gm/gmmain.cpp index 443a26bfbc..42b7bdaefa 100644 --- a/gm/gmmain.cpp +++ b/gm/gmmain.cpp @@ -191,7 +191,8 @@ const static ErrorCombination kDefaultIgnorableErrorTypes = ErrorCombination() class GMMain { public: - GMMain() : fUseFileHierarchy(false), fIgnorableErrorTypes(kDefaultIgnorableErrorTypes), + GMMain() : fUseFileHierarchy(false), fWriteChecksumBasedFilenames(false), + fIgnorableErrorTypes(kDefaultIgnorableErrorTypes), fMismatchPath(NULL), fTestsRun(0), fRenderModesEncountered(1) {} /** @@ -228,6 +229,30 @@ public: return SkOSPath::SkPathJoin(path, filename.c_str()); } + /** + * Assemble filename suitable for writing out an SkBitmap. + */ + SkString make_bitmap_filename(const char *path, + const char *shortName, + const char *configName, + const char *renderModeDescriptor, + const GmResultDigest &bitmapDigest) { + if (fWriteChecksumBasedFilenames) { + SkString filename; + filename.append(bitmapDigest.getHashType()); + filename.appendUnichar('_'); + filename.append(shortName); + filename.appendUnichar('_'); + filename.append(bitmapDigest.getDigestValue()); + filename.appendUnichar('.'); + filename.append(kPNG_FileExtension); + return SkOSPath::SkPathJoin(path, filename.c_str()); + } else { + return make_filename(path, shortName, configName, renderModeDescriptor, + kPNG_FileExtension); + } + } + /* since PNG insists on unpremultiplying our alpha, we take no precision chances and force all pixels to be 100% opaque, otherwise on compare we may not get a perfect match. @@ -637,7 +662,8 @@ public: ErrorCombination write_reference_image(const ConfigData& gRec, const char writePath [], const char renderModeDescriptor [], - const char *shortName, SkBitmap& bitmap, + const char *shortName, + const BitmapAndDigest& bitmapAndDigest, SkDynamicMemoryWStream* document) { SkString path; bool success = false; @@ -645,9 +671,9 @@ public: gRec.fBackend == kGPU_Backend || (gRec.fBackend == kPDF_Backend && CAN_IMAGE_PDF)) { - path = make_filename(writePath, shortName, gRec.fName, renderModeDescriptor, - kPNG_FileExtension); - success = write_bitmap(path, bitmap); + path = make_bitmap_filename(writePath, shortName, gRec.fName, renderModeDescriptor, + bitmapAndDigest.fDigest); + success = write_bitmap(path, bitmapAndDigest.fBitmap); } if (kPDF_Backend == gRec.fBackend) { path = make_filename(writePath, shortName, gRec.fName, renderModeDescriptor, @@ -743,7 +769,7 @@ public: * actual bitmap will be written out to a file within fMismatchPath. * * @param expectations what expectations to compare actualBitmap against - * @param actualBitmap the image we actually generated + * @param actualBitmapAndDigest the SkBitmap we actually generated, and its GmResultDigest * @param shortName name of test, e.g. "selftest1" * @param configName name of config, e.g. "8888" * @param renderModeDescriptor e.g., "-rtree", "-deferred" @@ -755,12 +781,11 @@ public: * See https://codereview.chromium.org/13650002/ ) */ ErrorCombination compare_to_expectations(Expectations expectations, - const SkBitmap& actualBitmap, + const BitmapAndDigest& actualBitmapAndDigest, const char *shortName, const char *configName, const char *renderModeDescriptor, bool addToJsonSummary) { ErrorCombination errors; - GmResultDigest actualResultDigest(actualBitmap); SkString shortNamePlusConfig = make_shortname_plus_config(shortName, configName); SkString completeNameString(shortNamePlusConfig); completeNameString.append(renderModeDescriptor); @@ -770,7 +795,7 @@ public: if (expectations.empty()) { errors.add(kMissingExpectations_ErrorType); - } else if (!expectations.match(actualResultDigest)) { + } else if (!expectations.match(actualBitmapAndDigest.fDigest)) { addToJsonSummary = true; // The error mode we record depends on whether this was running // in a non-standard renderMode. @@ -783,23 +808,24 @@ public: // Write out the "actuals" for any mismatches, if we have // been directed to do so. if (fMismatchPath) { - SkString path = - make_filename(fMismatchPath, shortName, configName, renderModeDescriptor, - kPNG_FileExtension); - write_bitmap(path, actualBitmap); + SkString path = make_bitmap_filename(fMismatchPath, shortName, configName, + renderModeDescriptor, + actualBitmapAndDigest.fDigest); + write_bitmap(path, actualBitmapAndDigest.fBitmap); } // If we have access to a single expected bitmap, log more // detail about the mismatch. const SkBitmap *expectedBitmapPtr = expectations.asBitmap(); if (NULL != expectedBitmapPtr) { - report_bitmap_diffs(*expectedBitmapPtr, actualBitmap, completeName); + report_bitmap_diffs(*expectedBitmapPtr, actualBitmapAndDigest.fBitmap, + completeName); } } RecordTestResults(errors, shortNamePlusConfig, renderModeDescriptor); if (addToJsonSummary) { - add_actual_results_to_json_summary(completeName, actualResultDigest, errors, + add_actual_results_to_json_summary(completeName, actualBitmapAndDigest.fDigest, errors, expectations.ignoreFailure()); add_expected_results_to_json_summary(completeName, expectations); } @@ -874,6 +900,7 @@ public: GM* gm, const ConfigData& gRec, const char writePath[], SkBitmap& actualBitmap, SkDynamicMemoryWStream* pdf) { + BitmapAndDigest actualBitmapAndDigest(actualBitmap); SkString shortNamePlusConfig = make_shortname_plus_config(gm->shortName(), gRec.fName); SkString nameWithExtension(shortNamePlusConfig); nameWithExtension.append("."); @@ -896,13 +923,13 @@ public: * See comments above complete_bitmap() for more detail. */ Expectations expectations = expectationsSource->get(nameWithExtension.c_str()); - errors.add(compare_to_expectations(expectations, actualBitmap, + errors.add(compare_to_expectations(expectations, actualBitmapAndDigest, gm->shortName(), gRec.fName, "", true)); } else { // If we are running without expectations, we still want to // record the actual results. - GmResultDigest actualResultDigest(actualBitmap); - add_actual_results_to_json_summary(nameWithExtension.c_str(), actualResultDigest, + add_actual_results_to_json_summary(nameWithExtension.c_str(), + actualBitmapAndDigest.fDigest, ErrorCombination(kMissingExpectations_ErrorType), false); RecordTestResults(ErrorCombination(kMissingExpectations_ErrorType), @@ -915,7 +942,7 @@ public: // renderModes of all tests! That would be a lot of files. if (writePath && (gRec.fFlags & kWrite_ConfigFlag)) { errors.add(write_reference_image(gRec, writePath, "", gm->shortName(), - actualBitmap, pdf)); + actualBitmapAndDigest, pdf)); } return errors; @@ -936,7 +963,8 @@ public: SkASSERT(referenceBitmap); Expectations expectations(*referenceBitmap); - return compare_to_expectations(expectations, actualBitmap, shortName, + BitmapAndDigest actualBitmapAndDigest(actualBitmap); + return compare_to_expectations(expectations, actualBitmapAndDigest, shortName, configName, renderModeDescriptor, false); } @@ -1146,7 +1174,7 @@ public: // They are public for now, to allow easier setting by tool_main(). // - bool fUseFileHierarchy; + bool fUseFileHierarchy, fWriteChecksumBasedFilenames; ErrorCombination fIgnorableErrorTypes; const char* fMismatchPath; @@ -1306,9 +1334,11 @@ DEFINE_bool(tiledPipe, false, "Exercise tiled SkGPipe replay."); DEFINE_bool(tileGrid, true, "Exercise the tile grid variant of SkPicture."); DEFINE_string(tileGridReplayScales, "", "Space separated list of floating-point scale " "factors to be used for tileGrid playback testing. Default value: 1.0"); -DEFINE_string(writeJsonSummaryPath, "", "Write a JSON-formatted result summary to this file."); DEFINE_bool2(verbose, v, false, "Give more detail (e.g. list all GMs run, more info about " "each test)."); +DEFINE_bool(writeChecksumBasedFilenames, false, "When writing out actual images, use checksum-" + "based filenames, as rebaseline.py will use when downloading them from Google Storage"); +DEFINE_string(writeJsonSummaryPath, "", "Write a JSON-formatted result summary to this file."); DEFINE_string2(writePath, w, "", "Write rendered images into this directory."); DEFINE_string2(writePicturePath, p, "", "Write .skp files into this directory."); DEFINE_int32(pdfJpegQuality, -1, "Encodes images in JPEG at quality level N, " @@ -1765,6 +1795,7 @@ int tool_main(int argc, char** argv) { SkCommandLineFlags::Parse(argc, argv); gmmain.fUseFileHierarchy = FLAGS_hierarchy; + gmmain.fWriteChecksumBasedFilenames = FLAGS_writeChecksumBasedFilenames; if (FLAGS_mismatchPath.count() == 1) { gmmain.fMismatchPath = FLAGS_mismatchPath[0]; } diff --git a/gm/tests/outputs/checksum-based-filenames/output-expected/command_line b/gm/tests/outputs/checksum-based-filenames/output-expected/command_line new file mode 100644 index 0000000000..3e4ddb944f --- /dev/null +++ b/gm/tests/outputs/checksum-based-filenames/output-expected/command_line @@ -0,0 +1 @@ +out/Debug/gm --verbose --writeChecksumBasedFilenames --match selftest1 --config 8888 565 -r gm/tests/inputs/json/different-pixels-no-hierarchy.json --writeJsonSummaryPath gm/tests/outputs/checksum-based-filenames/output-actual/json-summary.txt --writePath gm/tests/outputs/checksum-based-filenames/output-actual/writePath --mismatchPath gm/tests/outputs/checksum-based-filenames/output-actual/mismatchPath diff --git a/gm/tests/outputs/checksum-based-filenames/output-expected/json-summary.txt b/gm/tests/outputs/checksum-based-filenames/output-expected/json-summary.txt new file mode 100644 index 0000000000..b3e2a8150f --- /dev/null +++ b/gm/tests/outputs/checksum-based-filenames/output-expected/json-summary.txt @@ -0,0 +1,25 @@ +{ + "actual-results" : { + "failed" : { + "selftest1_565.png" : [ "bitmap-64bitMD5", 12927999507540085554 ], + "selftest1_8888.png" : [ "bitmap-64bitMD5", 1209453360120438698 ] + }, + "failure-ignored" : null, + "no-comparison" : null, + "succeeded" : null + }, + "expected-results" : { + "selftest1_565.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 8863920166200910451 ] + ], + "ignore-failure" : false + }, + "selftest1_8888.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 13451349865803053525 ] + ], + "ignore-failure" : false + } + } +} diff --git a/gm/tests/outputs/checksum-based-filenames/output-expected/mismatchPath/bitmap-64bitMD5_selftest1_1209453360120438698.png b/gm/tests/outputs/checksum-based-filenames/output-expected/mismatchPath/bitmap-64bitMD5_selftest1_1209453360120438698.png new file mode 100644 index 0000000000..bd5820f02e --- /dev/null +++ b/gm/tests/outputs/checksum-based-filenames/output-expected/mismatchPath/bitmap-64bitMD5_selftest1_1209453360120438698.png @@ -0,0 +1 @@ +[contents of gm/tests/outputs/checksum-based-filenames/output-actual/mismatchPath/bitmap-64bitMD5_selftest1_1209453360120438698.png] diff --git a/gm/tests/outputs/checksum-based-filenames/output-expected/mismatchPath/bitmap-64bitMD5_selftest1_12927999507540085554.png b/gm/tests/outputs/checksum-based-filenames/output-expected/mismatchPath/bitmap-64bitMD5_selftest1_12927999507540085554.png new file mode 100644 index 0000000000..b1ab39e5bc --- /dev/null +++ b/gm/tests/outputs/checksum-based-filenames/output-expected/mismatchPath/bitmap-64bitMD5_selftest1_12927999507540085554.png @@ -0,0 +1 @@ +[contents of gm/tests/outputs/checksum-based-filenames/output-actual/mismatchPath/bitmap-64bitMD5_selftest1_12927999507540085554.png] diff --git a/gm/tests/outputs/checksum-based-filenames/output-expected/return_value b/gm/tests/outputs/checksum-based-filenames/output-expected/return_value new file mode 100644 index 0000000000..ace9d03621 --- /dev/null +++ b/gm/tests/outputs/checksum-based-filenames/output-expected/return_value @@ -0,0 +1 @@ +255 diff --git a/gm/tests/outputs/checksum-based-filenames/output-expected/stderr b/gm/tests/outputs/checksum-based-filenames/output-expected/stderr new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gm/tests/outputs/checksum-based-filenames/output-expected/stdout b/gm/tests/outputs/checksum-based-filenames/output-expected/stdout new file mode 100644 index 0000000000..656d6958b1 --- /dev/null +++ b/gm/tests/outputs/checksum-based-filenames/output-expected/stdout @@ -0,0 +1,17 @@ +GM: These configs will be run: 8888 565 +GM: reading expectations from JSON summary file gm/tests/inputs/json/different-pixels-no-hierarchy.json +GM: writing to gm/tests/outputs/checksum-based-filenames/output-actual/writePath +GM: writing mismatches to gm/tests/outputs/checksum-based-filenames/output-actual/mismatchPath +GM: drawing... selftest1 [300 200] +GM: Ran 1 GMs +GM: ... over 2 configs ["8888", "565"] +GM: ... and 7 modes ["pipe", "pipe cross-process", "pipe cross-process, shared address", "replay", "rtree", "serialize", "tilegrid"] +GM: ... so there should be a total of 9 tests. +GM: Ran 9 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0 +GM: [*] 0 NoGpuContext: +GM: [ ] 0 IntentionallySkipped: +GM: [*] 0 RenderModeMismatch: +GM: [*] 2 ExpectationsMismatch: selftest1_8888 selftest1_565 +GM: [ ] 0 MissingExpectations: +GM: [*] 0 WritingReferenceImage: +GM: (results marked with [*] will cause nonzero return value) diff --git a/gm/tests/outputs/checksum-based-filenames/output-expected/writePath/bitmap-64bitMD5_selftest1_1209453360120438698.png b/gm/tests/outputs/checksum-based-filenames/output-expected/writePath/bitmap-64bitMD5_selftest1_1209453360120438698.png new file mode 100644 index 0000000000..e13581c69f --- /dev/null +++ b/gm/tests/outputs/checksum-based-filenames/output-expected/writePath/bitmap-64bitMD5_selftest1_1209453360120438698.png @@ -0,0 +1 @@ +[contents of gm/tests/outputs/checksum-based-filenames/output-actual/writePath/bitmap-64bitMD5_selftest1_1209453360120438698.png] diff --git a/gm/tests/outputs/checksum-based-filenames/output-expected/writePath/bitmap-64bitMD5_selftest1_12927999507540085554.png b/gm/tests/outputs/checksum-based-filenames/output-expected/writePath/bitmap-64bitMD5_selftest1_12927999507540085554.png new file mode 100644 index 0000000000..0a719f7670 --- /dev/null +++ b/gm/tests/outputs/checksum-based-filenames/output-expected/writePath/bitmap-64bitMD5_selftest1_12927999507540085554.png @@ -0,0 +1 @@ +[contents of gm/tests/outputs/checksum-based-filenames/output-actual/writePath/bitmap-64bitMD5_selftest1_12927999507540085554.png] diff --git a/gm/tests/run.sh b/gm/tests/run.sh index 809c7d038d..ab930a1c1b 100755 --- a/gm/tests/run.sh +++ b/gm/tests/run.sh @@ -216,6 +216,10 @@ gm_test "--ignoreErrorTypes ExpectationsMismatch NoGpuContext --verbose --hierar # Test non-hierarchical mode. gm_test "--verbose --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels-no-hierarchy.json" "$GM_OUTPUTS/no-hierarchy" +# Try writing out actual images using checksum-based filenames, like we do when +# uploading to Google Storage. +gm_test "--verbose --writeChecksumBasedFilenames --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels-no-hierarchy.json" "$GM_OUTPUTS/checksum-based-filenames" + # Exercise display_json_results.py PASSING_CASES="compared-against-identical-bytes-json compared-against-identical-pixels-json" FAILING_CASES="compared-against-different-pixels-json"