gm: add self-test to confirm that ignore-failure field is respected
BUG=skia:1578 R=scroggo@google.com Review URL: https://codereview.chromium.org/23799004 git-svn-id: http://skia.googlecode.com/svn/trunk@11047 2bbb7eff-a529-9590-31e7-b0007b416f81
This commit is contained in:
epoger@google.com
parent
5a02cb48fd
commit
5852c8b06d
@ -821,12 +821,6 @@ public:
|
||||
this->fJsonActualResults_Succeeded[testName] = jsonActualResults;
|
||||
} else {
|
||||
if (ignoreFailure) {
|
||||
// TODO: Once we have added the ability to compare
|
||||
// actual results against expectations in a JSON file
|
||||
// (where we can set ignore-failure to either true or
|
||||
// false), add test cases that exercise ignored
|
||||
// failures (both for kMissingExpectations_ErrorType
|
||||
// and kExpectationsMismatch_ErrorType).
|
||||
this->fJsonActualResults_FailureIgnored[testName] =
|
||||
jsonActualResults;
|
||||
} else {
|
||||
|
||||
@ -0,0 +1 @@
|
||||
out/Debug/gm --verbose --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/json/different-pixels-ignore-some-failures.json --writeJsonSummaryPath gm/tests/outputs/ignoring-some-failures/output-actual/json-summary.txt --writePath gm/tests/outputs/ignoring-some-failures/output-actual/writePath --mismatchPath gm/tests/outputs/ignoring-some-failures/output-actual/mismatchPath --missingExpectationsPath gm/tests/outputs/ignoring-some-failures/output-actual/missingExpectationsPath
|
||||
@ -0,0 +1,26 @@
|
||||
{
|
||||
"actual-results" : {
|
||||
"failed" : {
|
||||
"8888/selftest1.png" : [ "bitmap-64bitMD5", 1209453360120438698 ]
|
||||
},
|
||||
"failure-ignored" : {
|
||||
"565/selftest1.png" : [ "bitmap-64bitMD5", 12927999507540085554 ]
|
||||
},
|
||||
"no-comparison" : null,
|
||||
"succeeded" : null
|
||||
},
|
||||
"expected-results" : {
|
||||
"565/selftest1.png" : {
|
||||
"allowed-digests" : [
|
||||
[ "bitmap-64bitMD5", 8863920166200910451 ]
|
||||
],
|
||||
"ignore-failure" : true
|
||||
},
|
||||
"8888/selftest1.png" : {
|
||||
"allowed-digests" : [
|
||||
[ "bitmap-64bitMD5", 13451349865803053525 ]
|
||||
],
|
||||
"ignore-failure" : false
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1 @@
|
||||
Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
|
||||
@ -0,0 +1 @@
|
||||
[contents of gm/tests/outputs/ignoring-some-failures/output-actual/mismatchPath/565/selftest1.png]
|
||||
@ -0,0 +1 @@
|
||||
Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
|
||||
@ -0,0 +1 @@
|
||||
[contents of gm/tests/outputs/ignoring-some-failures/output-actual/mismatchPath/8888/selftest1.png]
|
||||
@ -0,0 +1 @@
|
||||
Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
|
||||
@ -0,0 +1 @@
|
||||
Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
|
||||
@ -0,0 +1 @@
|
||||
Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
|
||||
@ -0,0 +1 @@
|
||||
Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
|
||||
@ -0,0 +1 @@
|
||||
255
|
||||
@ -0,0 +1,18 @@
|
||||
GM: These configs will be run: 8888 565
|
||||
GM: reading expectations from JSON summary file gm/tests/inputs/json/different-pixels-ignore-some-failures.json
|
||||
GM: writing to gm/tests/outputs/ignoring-some-failures/output-actual/writePath
|
||||
GM: writing mismatches to gm/tests/outputs/ignoring-some-failures/output-actual/mismatchPath
|
||||
GM: writing images without expectations to gm/tests/outputs/ignoring-some-failures/output-actual/missingExpectationsPath
|
||||
GM: drawing... selftest1 [300 200]
|
||||
GM: Ran 1 GMs
|
||||
GM: ... over 2 configs ["8888", "565"]
|
||||
GM: ... and 7 modes ["pipe", "pipe cross-process", "pipe cross-process, shared address", "replay", "rtree", "serialize", "tilegrid"]
|
||||
GM: ... so there should be a total of 9 tests.
|
||||
GM: Ran 9 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0
|
||||
GM: [*] 0 NoGpuContext:
|
||||
GM: [ ] 0 IntentionallySkipped:
|
||||
GM: [*] 0 RenderModeMismatch:
|
||||
GM: [*] 2 ExpectationsMismatch: 8888/selftest1 565/selftest1
|
||||
GM: [ ] 0 MissingExpectations:
|
||||
GM: [*] 0 WritingReferenceImage:
|
||||
GM: (results marked with [*] will cause nonzero return value)
|
||||
@ -0,0 +1 @@
|
||||
Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
|
||||
@ -0,0 +1 @@
|
||||
[contents of gm/tests/outputs/ignoring-some-failures/output-actual/writePath/565/selftest1.png]
|
||||
@ -0,0 +1 @@
|
||||
Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
|
||||
@ -0,0 +1 @@
|
||||
[contents of gm/tests/outputs/ignoring-some-failures/output-actual/writePath/8888/selftest1.png]
|
||||
@ -0,0 +1 @@
|
||||
Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
|
||||
@ -163,6 +163,14 @@ function create_inputs_dir {
|
||||
$GM_BINARY --hierarchy --match selftest1 $CONFIGS -r $THIS_IMAGE_DIR \
|
||||
--writeJsonSummaryPath $JSON_DIR/different-pixels.json
|
||||
|
||||
# Create another JSON expectations file which is identical to
|
||||
# different-pixels.json, but in which the *first* ignore-failure is changed
|
||||
# from false to true.
|
||||
OLD='"ignore-failure" : false'
|
||||
NEW='"ignore-failure" : true'
|
||||
sed -e "0,/$OLD/{s/$OLD/$NEW/}" $JSON_DIR/different-pixels.json \
|
||||
>$JSON_DIR/different-pixels-ignore-some-failures.json
|
||||
|
||||
THIS_IMAGE_DIR=$IMAGES_DIR/different-pixels-no-hierarchy
|
||||
mkdir -p $THIS_IMAGE_DIR
|
||||
$GM_BINARY --match selftest2 $CONFIGS -w $THIS_IMAGE_DIR
|
||||
@ -193,6 +201,10 @@ gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/ide
|
||||
gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/different-pixels" "$GM_OUTPUTS/compared-against-different-pixels-images"
|
||||
gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/compared-against-different-pixels-json"
|
||||
|
||||
# Compare different pixels, but with a SUBSET of the expectations marked as
|
||||
# ignore-failure.
|
||||
gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels-ignore-some-failures.json" "$GM_OUTPUTS/ignoring-some-failures"
|
||||
|
||||
# Compare generated image against an empty "expected image" dir.
|
||||
gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/empty-dir" "$GM_OUTPUTS/compared-against-empty-dir"
|
||||
|
||||
@ -205,9 +217,8 @@ gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/empty-dir"
|
||||
# Add pdf to the list of configs.
|
||||
gm_test "--verbose --hierarchy --match selftest1 $CONFIGS pdf -r $GM_INPUTS/json/identical-bytes.json" "$GM_OUTPUTS/add-config-pdf"
|
||||
|
||||
# If run without "-r", the JSON's "actual-results" section should contain
|
||||
# actual checksums marked as "failure-ignored", but the "expected-results"
|
||||
# section should be empty.
|
||||
# Test what happens if run without -r (no expected-results.json to compare
|
||||
# against).
|
||||
gm_test "--verbose --hierarchy --match selftest1 $CONFIGS" "$GM_OUTPUTS/no-readpath"
|
||||
|
||||
# Test what happens if a subset of the renderModes fail (e.g. pipe)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user