AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

skdiff: add --failonmismatches and --listfilename options, plus cleanup

Browse Source 
These changes are needed in order to switch the buildbots from using "gm -r" to "skdiff" to compare gm results, and should be generally good for humans too.
Review URL: https://codereview.appspot.com/6392054

git-svn-id: http://skia.googlecode.com/svn/trunk@4579 2bbb7eff-a529-9590-31e7-b0007b416f81
This commit is contained in:
epoger@google.com 2012-07-12 18:16:02 +00:00
parent a6c9e0e02b
commit 46a45963fa
11 changed files with 149 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

182
tools/skdiff_main.cpp
View File

@ -42,18 +42,42 @@
// Result of comparison for each pair of files. // Result of comparison for each pair of files.
// Listed from "better" to "worse", for sorting of results. // Listed from "better" to "worse", for sorting of results.
enum Result { enum Result {
kEqualBits, // both files in the pair contain exactly the same bits kEqualBits,
kEqualPixels, // not bitwise equal, but their pixels are exactly the same kEqualPixels,
kDifferentPixels,// both are images we can parse, but with different pixels kDifferentPixels,
kDifferentSizes, // both are images we can parse, but of different sizes kDifferentSizes,
kDifferentOther, // files have different bits but are not parsable images kDifferentOther,
kComparisonMissing,// missing from comparisonDir kComparisonMissing,
kBaseMissing, // missing from baseDir kBaseMissing,
kUnknown, // not compared yet kUnknown,
// //
kNumResultTypes // NOT A VALID VALUE--used to set up arrays. Must be last. kNumResultTypes // NOT A VALID VALUE--used to set up arrays. Must be last.
}; };
// Returns a text description of the given Result type.
const char *getResultDescription(Result result) {
switch (result) {
case kEqualBits:
return "contain exactly the same bits";
case kEqualPixels:
return "contain the same pixel values, but not the same bits";
case kDifferentPixels:
return "have identical dimensions but some differing pixels";
case kDifferentSizes:
return "have differing dimensions";
case kDifferentOther:
return "contain different bits and are not parsable images";
case kBaseMissing:
return "missing from baseDir";
case kComparisonMissing:
return "missing from comparisonDir";
case kUnknown:
return "not compared yet";
default:
return NULL;
}
}
struct DiffRecord { struct DiffRecord {
DiffRecord (const SkString filename, DiffRecord (const SkString filename,
const SkString basePath, const SkString basePath,
@ -138,31 +162,32 @@ struct DiffSummary {
FileArray fResultsOfType[kNumResultTypes]; FileArray fResultsOfType[kNumResultTypes];
// Print the contents of this FileArray, if any, to stdout. // Print a line about the contents of this FileArray to stdout; if the FileArray is empty,
// (If the FileArray is empty, print nothing.) // print nothing.
void printContents(const FileArray& fileArray, const char* headerText) { void printContents(const FileArray& fileArray, const char* headerText, bool listFilenames) {
int n = fileArray.count(); int n = fileArray.count();
if (n > 0) { if (n > 0) {
printf("%s:\n", headerText); printf(" %d file pairs %s", n, headerText);
if (listFilenames) {
printf(": ");
for (int i = 0; i < n; ++i) { for (int i = 0; i < n; ++i) {
printf("\t%s\n", fileArray[i]->c_str()); printf("%s ", fileArray[i]->c_str());
} }
} }
printf("\n");
}
} }
void print () { void print(bool listFilenames) {
printContents(fResultsOfType[kBaseMissing], printf("compared %d file pairs:\n", fNumMatches + fNumMismatches);
"Missing in baseDir"); for (int resultInt = 0; resultInt < kNumResultTypes; resultInt++) {
printContents(fResultsOfType[kComparisonMissing], Result result = static_cast<Result>(resultInt);
"Missing in comparisonDir"); printContents(fResultsOfType[result], getResultDescription(result), listFilenames);
printf("%d of %d images matched.\n", fNumMatches, }
fNumMatches + fNumMismatches);
if (fNumMismatches > 0) { if (fNumMismatches > 0) {
printf("Maximum pixel intensity mismatch %d\n", fMaxMismatchV); printf("Maximum pixel intensity mismatch %d\n", fMaxMismatchV);
printf("Largest area mismatch was %.2f%% of pixels\n", printf("Largest area mismatch was %.2f%% of pixels\n",fMaxMismatchPercent);
fMaxMismatchPercent);
} }
} }
void add (DiffRecord* drp) { void add (DiffRecord* drp) {
@ -831,9 +856,9 @@ static void print_table_header (SkFILEWStream* stream,
stream->writeText("every different pixel shown in white"); stream->writeText("every different pixel shown in white");
stream->writeText("</th>\n<th>"); stream->writeText("</th>\n<th>");
stream->writeText("color difference at each pixel"); stream->writeText("color difference at each pixel");
stream->writeText("</th>\n<th>"); stream->writeText("</th>\n<th>baseDir: ");
stream->writeText(baseDir.c_str()); stream->writeText(baseDir.c_str());
stream->writeText("</th>\n<th>"); stream->writeText("</th>\n<th>comparisonDir: ");
stream->writeText(comparisonDir.c_str()); stream->writeText(comparisonDir.c_str());
stream->writeText("</th>\n"); stream->writeText("</th>\n");
stream->writeText("</tr>\n"); stream->writeText("</tr>\n");
@ -1052,35 +1077,40 @@ static void usage (char * argv0) {
"Usage: \n" "Usage: \n"
" %s <baseDir> <comparisonDir> [outputDir] \n" " %s <baseDir> <comparisonDir> [outputDir] \n"
, argv0, argv0); , argv0, argv0);
SkDebugf("\n"
"Arguments: \n"
" --nodiffs: don't write out image diffs or index.html, just generate \n"
" report on stdout \n"
" --threshold <n>: only report differences > n (per color channel) [default 0]\n"
" --match: compare files whose filenames contain this substring; if \n"
" unspecified, compare ALL files. \n"
" this flag may be repeated to add more matching substrings. \n"
" --nomatch: regardless of --match, DO NOT compare files whose filenames \n"
" contain this substring. \n"
" this flag may be repeated to add more forbidden substrings. \n"
" --sortbymismatch: sort by average color channel mismatch\n");
SkDebugf( SkDebugf(
" --sortbymaxmismatch: sort by worst color channel mismatch;\n" "\nArguments:"
" break ties with -sortbymismatch\n" "\n --help: display this info"
" [default sort is by fraction of pixels mismatching]\n"); "\n --failonmismatches: exit with nonzero return code (number of mismatching"
SkDebugf( "\n image pairs) if any pairs differ by more than threshold;"
" --weighted: sort by # pixels different weighted by color difference\n"); "\n otherwise, only return nonzero if the tool itself fails"
SkDebugf( "\n --listfilenames: list all filenames for each result type in stdout"
" --noprintdirs: do not print the directories used."); "\n --match: compare files whose filenames contain this substring; if"
SkDebugf( "\n unspecified, compare ALL files."
" baseDir: directory to read baseline images from.\n"); "\n this flag may be repeated to add more matching substrings."
SkDebugf( "\n --nodiffs: don't write out image diffs or index.html, just generate"
" comparisonDir: directory to read comparison images from\n"); "\n report on stdout"
SkDebugf( "\n --nomatch: regardless of --match, DO NOT compare files whose filenames"
" outputDir: directory to write difference images and index.html to; \n" "\n contain this substring."
" defaults to comparisonDir \n"); "\n this flag may be repeated to add more forbidden substrings."
"\n --noprintdirs: do not print the directories used."
"\n --sortbymaxmismatch: sort by worst color channel mismatch;"
"\n break ties with -sortbymismatch"
"\n --sortbymismatch: sort by average color channel mismatch"
"\n --threshold <n>: only report differences > n (per color channel) [default 0]"
"\n --weighted: sort by # pixels different weighted by color difference"
"\n"
"\n baseDir: directory to read baseline images from."
"\n comparisonDir: directory to read comparison images from"
"\n outputDir: directory to write difference images and index.html to;"
"\n defaults to comparisonDir"
"\n"
"\nIf no sort is specified, it will sort by fraction of pixels mismatching."
"\n");
} }
const int NO_ERROR = 0;
const int GENERIC_ERROR = -1;
int main (int argc, char ** argv) { int main (int argc, char ** argv) {
DiffMetricProc diffProc = compute_diff_pmcolor; DiffMetricProc diffProc = compute_diff_pmcolor;
int (*sortProc)(const void*, const void*) = compare<CompareDiffMetrics>; int (*sortProc)(const void*, const void*) = compare<CompareDiffMetrics>;
@ -1094,7 +1124,9 @@ int main (int argc, char ** argv) {
StringArray matchSubstrings; StringArray matchSubstrings;
StringArray nomatchSubstrings; StringArray nomatchSubstrings;
bool failOnMismatches = false;
bool generateDiffs = true; bool generateDiffs = true;
bool listFilenames = false;
bool printDirs = true; bool printDirs = true;
RecordArray differences; RecordArray differences;
@ -1103,40 +1135,48 @@ int main (int argc, char ** argv) {
int i; int i;
int numUnflaggedArguments = 0; int numUnflaggedArguments = 0;
for (i = 1; i < argc; i++) { for (i = 1; i < argc; i++) {
if (!strcmp(argv[i], "--help")) { if (!strcmp(argv[i], "--failonmismatches")) {
usage(argv[0]); failOnMismatches = true;
return 0;
}
if (!strcmp(argv[i], "--nodiffs")) {
generateDiffs = false;
continue; continue;
} }
if (!strcmp(argv[i], "--threshold")) { if (!strcmp(argv[i], "--help")) {
colorThreshold = atoi(argv[++i]); usage(argv[0]);
return NO_ERROR;
}
if (!strcmp(argv[i], "--listfilenames")) {
listFilenames = true;
continue; continue;
} }
if (!strcmp(argv[i], "--match")) { if (!strcmp(argv[i], "--match")) {
matchSubstrings.push(new SkString(argv[++i])); matchSubstrings.push(new SkString(argv[++i]));
continue; continue;
} }
if (!strcmp(argv[i], "--nodiffs")) {
generateDiffs = false;
continue;
}
if (!strcmp(argv[i], "--nomatch")) { if (!strcmp(argv[i], "--nomatch")) {
nomatchSubstrings.push(new SkString(argv[++i])); nomatchSubstrings.push(new SkString(argv[++i]));
continue; continue;
} }
if (!strcmp(argv[i], "--sortbymismatch")) { if (!strcmp(argv[i], "--noprintdirs")) {
sortProc = compare<CompareDiffMeanMismatches>; printDirs = false;
continue; continue;
} }
if (!strcmp(argv[i], "--sortbymaxmismatch")) { if (!strcmp(argv[i], "--sortbymaxmismatch")) {
sortProc = compare<CompareDiffMaxMismatches>; sortProc = compare<CompareDiffMaxMismatches>;
continue; continue;
} }
if (!strcmp(argv[i], "--weighted")) { if (!strcmp(argv[i], "--sortbymismatch")) {
sortProc = compare<CompareDiffWeighted>; sortProc = compare<CompareDiffMeanMismatches>;
continue; continue;
} }
if (!strcmp(argv[i], "--noprintdirs")) { if (!strcmp(argv[i], "--threshold")) {
printDirs = false; colorThreshold = atoi(argv[++i]);
continue;
}
if (!strcmp(argv[i], "--weighted")) {
sortProc = compare<CompareDiffWeighted>;
continue; continue;
} }
if (argv[i][0] != '-') { if (argv[i][0] != '-') {
@ -1153,20 +1193,20 @@ int main (int argc, char ** argv) {
default: default:
SkDebugf("extra unflagged argument <%s>\n", argv[i]); SkDebugf("extra unflagged argument <%s>\n", argv[i]);
usage(argv[0]); usage(argv[0]);
return 0; return GENERIC_ERROR;
} }
} }
SkDebugf("Unrecognized argument <%s>\n", argv[i]); SkDebugf("Unrecognized argument <%s>\n", argv[i]);
usage(argv[0]); usage(argv[0]);
return 0; return GENERIC_ERROR;
} }
if (numUnflaggedArguments == 2) { if (numUnflaggedArguments == 2) {
outputDir = comparisonDir; outputDir = comparisonDir;
} else if (numUnflaggedArguments != 3) { } else if (numUnflaggedArguments != 3) {
usage(argv[0]); usage(argv[0]);
return 0; return GENERIC_ERROR;
} }
if (!baseDir.endsWith(PATH_DIV_STR)) { if (!baseDir.endsWith(PATH_DIV_STR)) {
@ -1206,7 +1246,7 @@ int main (int argc, char ** argv) {
create_diff_images(diffProc, colorThreshold, &differences, create_diff_images(diffProc, colorThreshold, &differences,
baseDir, comparisonDir, outputDir, baseDir, comparisonDir, outputDir,
matchSubstrings, nomatchSubstrings, &summary); matchSubstrings, nomatchSubstrings, &summary);
summary.print(); summary.print(listFilenames);
if (differences.count()) { if (differences.count()) {
qsort(differences.begin(), differences.count(), qsort(differences.begin(), differences.count(),
@ -1224,5 +1264,9 @@ int main (int argc, char ** argv) {
matchSubstrings.deleteAll(); matchSubstrings.deleteAll();
nomatchSubstrings.deleteAll(); nomatchSubstrings.deleteAll();
if (failOnMismatches) {
return summary.fNumMismatches; return summary.fNumMismatches;
} else {
return NO_ERROR;
}
} }

13
tools/tests/run.sh
View File

@ -42,19 +42,20 @@ SKDIFF_TESTDIR=tools/tests/skdiff
# Run skdiff over a variety of file pair types: identical bits, identical # Run skdiff over a variety of file pair types: identical bits, identical
# pixels, missing from baseDir, etc. # pixels, missing from baseDir, etc.
# TODO: In the near future, skdiff will return a nonzero exit code in this case.
skdiff_test "$SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/test1" skdiff_test "$SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/test1"
# Same as above but without generating HTML output files. # Same as above, except:
# TODO: In the near future, skdiff will return a nonzero exit code in this case. # - return the number of mismatching file pairs
skdiff_test "--nodiffs $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/test2" # - list filenames with each result type to stdout
# - don't generate HTML output files
skdiff_test "--failonmismatches --listfilenames --nodiffs $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/test2"
# Run skdiff over just the files that have identical bits, to validate any # Run skdiff over just the files that have identical bits, to validate any
# behavior/return value differences in this case. # behavior/return value differences in this case.
skdiff_test "--nodiffs --match identical-bits $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/identical-bits" skdiff_test "--failonmismatches --nodiffs --match identical-bits $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/identical-bits"
# Run skdiff over just the files that have identical bits or identical pixels, # Run skdiff over just the files that have identical bits or identical pixels,
# to validate any behavior/return value differences in this case. # to validate any behavior/return value differences in this case.
skdiff_test "--nodiffs --match identical-bits --match identical-pixels $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/identical-bits-or-pixels" skdiff_test "--failonmismatches --nodiffs --match identical-bits --match identical-pixels $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/identical-bits-or-pixels"
echo "All tests passed." echo "All tests passed."

2
tools/tests/skdiff/identical-bits-or-pixels/output-expected/command_line
View File

@ -1 +1 @@
out/Debug/skdiff --nodiffs --match identical-bits --match identical-pixels tools/tests/skdiff/baseDir tools/tests/skdiff/comparisonDir tools/tests/skdiff/identical-bits-or-pixels/output-actual out/Debug/skdiff --failonmismatches --nodiffs --match identical-bits --match identical-pixels tools/tests/skdiff/baseDir tools/tests/skdiff/comparisonDir tools/tests/skdiff/identical-bits-or-pixels/output-actual

4
tools/tests/skdiff/identical-bits-or-pixels/output-expected/stdout
View File

@ -1,4 +1,6 @@
baseDir is [tools/tests/skdiff/baseDir/] baseDir is [tools/tests/skdiff/baseDir/]
comparisonDir is [tools/tests/skdiff/comparisonDir/] comparisonDir is [tools/tests/skdiff/comparisonDir/]
not writing any diffs to outputDir [tools/tests/skdiff/identical-bits-or-pixels/output-actual/] not writing any diffs to outputDir [tools/tests/skdiff/identical-bits-or-pixels/output-actual/]
3 of 3 images matched. compared 3 file pairs:
2 file pairs contain exactly the same bits
1 file pairs contain the same pixel values, but not the same bits

2
tools/tests/skdiff/identical-bits/output-expected/command_line
View File

@ -1 +1 @@
out/Debug/skdiff --nodiffs --match identical-bits tools/tests/skdiff/baseDir tools/tests/skdiff/comparisonDir tools/tests/skdiff/identical-bits/output-actual out/Debug/skdiff --failonmismatches --nodiffs --match identical-bits tools/tests/skdiff/baseDir tools/tests/skdiff/comparisonDir tools/tests/skdiff/identical-bits/output-actual

3
tools/tests/skdiff/identical-bits/output-expected/stdout
View File

@ -1,4 +1,5 @@
baseDir is [tools/tests/skdiff/baseDir/] baseDir is [tools/tests/skdiff/baseDir/]
comparisonDir is [tools/tests/skdiff/comparisonDir/] comparisonDir is [tools/tests/skdiff/comparisonDir/]
not writing any diffs to outputDir [tools/tests/skdiff/identical-bits/output-actual/] not writing any diffs to outputDir [tools/tests/skdiff/identical-bits/output-actual/]
2 of 2 images matched. compared 2 file pairs:
2 file pairs contain exactly the same bits

4
tools/tests/skdiff/test1/output-expected/index.html
View File

@ -4,8 +4,8 @@
<tr><th>3 of 12 images matched exactly.<br></th> <tr><th>3 of 12 images matched exactly.<br></th>
<th>every different pixel shown in white</th> <th>every different pixel shown in white</th>
<th>color difference at each pixel</th> <th>color difference at each pixel</th>
<th>tools/tests/skdiff/baseDir/</th> <th>baseDir: tools/tests/skdiff/baseDir/</th>
<th>tools/tests/skdiff/comparisonDir/</th> <th>comparisonDir: tools/tests/skdiff/comparisonDir/</th>
</tr> </tr>
<tr> <tr>
<td><b>missing-from-baseDir.png</b><br>Missing from baseDir</td><td>N/A</td><td>N/A</td><td>N/A</td><td><a href="../../../../../tools/tests/skdiff/comparisonDir/missing-from-baseDir.png"><img src="../../../../../tools/tests/skdiff/comparisonDir/missing-from-baseDir.png" height="240px"></a></td></tr> <td><b>missing-from-baseDir.png</b><br>Missing from baseDir</td><td>N/A</td><td>N/A</td><td>N/A</td><td><a href="../../../../../tools/tests/skdiff/comparisonDir/missing-from-baseDir.png"><img src="../../../../../tools/tests/skdiff/comparisonDir/missing-from-baseDir.png" height="240px"></a></td></tr>

2
tools/tests/skdiff/test1/output-expected/return_value
View File

@ -1 +1 @@
9 0

15
tools/tests/skdiff/test1/output-expected/stdout
View File

@ -6,12 +6,13 @@ ERROR: no codec found for <tools/tests/skdiff/comparisonDir/different-bits-unkno
baseDir is [tools/tests/skdiff/baseDir/] baseDir is [tools/tests/skdiff/baseDir/]
comparisonDir is [tools/tests/skdiff/comparisonDir/] comparisonDir is [tools/tests/skdiff/comparisonDir/]
writing diffs to outputDir is [tools/tests/skdiff/test1/output-actual/] writing diffs to outputDir is [tools/tests/skdiff/test1/output-actual/]
Missing in baseDir: compared 12 file pairs:
missing-from-baseDir.png 2 file pairs contain exactly the same bits
missing-from-baseDir.xyz 1 file pairs contain the same pixel values, but not the same bits
Missing in comparisonDir: 2 file pairs have identical dimensions but some differing pixels
missing-from-comparisonDir.png 2 file pairs have differing dimensions
missing-from-comparisonDir.xyz 1 file pairs contain different bits and are not parsable images
3 of 12 images matched. 2 file pairs missing from comparisonDir
2 file pairs missing from baseDir
Maximum pixel intensity mismatch 239 Maximum pixel intensity mismatch 239
Largest area mismatch was 97.99% of pixels Largest area mismatch was 97.99% of pixels

2
tools/tests/skdiff/test2/output-expected/command_line
View File

@ -1 +1 @@
out/Debug/skdiff --nodiffs tools/tests/skdiff/baseDir tools/tests/skdiff/comparisonDir tools/tests/skdiff/test2/output-actual out/Debug/skdiff --failonmismatches --listfilenames --nodiffs tools/tests/skdiff/baseDir tools/tests/skdiff/comparisonDir tools/tests/skdiff/test2/output-actual

15
tools/tests/skdiff/test2/output-expected/stdout
View File

@ -2,12 +2,13 @@ ERROR: no codec found for basePath <tools/tests/skdiff/baseDir/different-bits-un
baseDir is [tools/tests/skdiff/baseDir/] baseDir is [tools/tests/skdiff/baseDir/]
comparisonDir is [tools/tests/skdiff/comparisonDir/] comparisonDir is [tools/tests/skdiff/comparisonDir/]
not writing any diffs to outputDir [tools/tests/skdiff/test2/output-actual/] not writing any diffs to outputDir [tools/tests/skdiff/test2/output-actual/]
Missing in baseDir: compared 12 file pairs:
missing-from-baseDir.png 2 file pairs contain exactly the same bits: identical-bits-unknown-format.xyz identical-bits.png
missing-from-baseDir.xyz 1 file pairs contain the same pixel values, but not the same bits: different-bits-identical-pixels.png
Missing in comparisonDir: 2 file pairs have identical dimensions but some differing pixels: slightly-different-pixels-same-size.png very-different-pixels-same-size.png
missing-from-comparisonDir.png 2 file pairs have differing dimensions: slightly-different-sizes.png very-different-sizes.png
missing-from-comparisonDir.xyz 1 file pairs contain different bits and are not parsable images: different-bits-unknown-format.xyz
3 of 12 images matched. 2 file pairs missing from comparisonDir: missing-from-comparisonDir.png missing-from-comparisonDir.xyz
2 file pairs missing from baseDir: missing-from-baseDir.png missing-from-baseDir.xyz
Maximum pixel intensity mismatch 239 Maximum pixel intensity mismatch 239
Largest area mismatch was 97.99% of pixels Largest area mismatch was 97.99% of pixels