2011-07-28 14:26:00 +00:00
|
|
|
/*
|
|
|
|
* Copyright 2011 Google Inc.
|
|
|
|
*
|
|
|
|
* Use of this source code is governed by a BSD-style license that can be
|
|
|
|
* found in the LICENSE file.
|
|
|
|
*/
|
2014-06-09 19:05:34 +00:00
|
|
|
|
2009-02-27 16:24:51 +00:00
|
|
|
#include "Test.h"
|
|
|
|
|
2014-02-26 16:31:22 +00:00
|
|
|
#include "SkCommandLineFlags.h"
|
2014-01-24 20:56:26 +00:00
|
|
|
#include "SkError.h"
|
2013-04-19 13:24:28 +00:00
|
|
|
#include "SkString.h"
|
|
|
|
#include "SkTArray.h"
|
2013-04-22 16:43:07 +00:00
|
|
|
#include "SkTime.h"
|
2012-08-02 14:03:32 +00:00
|
|
|
|
|
|
|
#if SK_SUPPORT_GPU
|
2011-08-16 15:45:58 +00:00
|
|
|
#include "GrContext.h"
|
2014-10-09 12:24:15 +00:00
|
|
|
#include "gl/SkGLContext.h"
|
2012-08-02 14:03:32 +00:00
|
|
|
#else
|
|
|
|
class GrContext;
|
|
|
|
#endif
|
2011-08-16 15:45:58 +00:00
|
|
|
|
2014-02-26 16:31:22 +00:00
|
|
|
DEFINE_string2(tmpDir, t, NULL, "tmp directory for tests to use.");
|
|
|
|
|
2009-02-27 16:24:51 +00:00
|
|
|
using namespace skiatest;
|
|
|
|
|
2013-04-19 13:24:28 +00:00
|
|
|
Reporter::Reporter() : fTestCount(0) {
|
2009-02-27 16:24:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void Reporter::startTest(Test* test) {
|
|
|
|
this->onStart(test);
|
|
|
|
}
|
|
|
|
|
When running DM, write test failures to json.
Add skiatest::Failure to keep track of data about a test failure.
Reporter::reportFailed and ::onReportFailed now take Failure as a
parameter. This allows the implementation to treat the failure as it
wishes. Provide a helper to format the failure the same as prior to
the change.
Update the macros for calling reportFailed (REPORTER_ASSERT etc) to
create a Failure object.
Convert a direct call to reportFailed to the macro ERRORF.
Write Failures to Json.
Sample output when running dm on the dummy test crrev.com/705723004:
{
"test_results" : {
"failures" : [
{
"condition" : "0 > 3",
"file_name" : "../../tests/DummyTest.cpp",
"line_no" : 10,
"message" : ""
},
{
"condition" : "false",
"file_name" : "../../tests/DummyTest.cpp",
"line_no" : 4,
"message" : ""
},
{
"condition" : "1 == 3",
"file_name" : "../../tests/DummyTest.cpp",
"line_no" : 5,
"message" : "I can too count!"
},
{
"condition" : "",
"file_name" : "../../tests/DummyTest.cpp",
"line_no" : 6,
"message" : "seven is 7"
},
{
"condition" : "1 == 3",
"file_name" : "../../tests/DummyTest.cpp",
"line_no" : 14,
"message" : "I can too count!"
}
]
}
}
Report all of the failures from one test.
Previously, if one test had multiple failures, only one was reportered.
e.g:
Failures:
test Dummy: ../../tests/DummyTest.cpp:6 seven is 7
test Dummy2: ../../tests/DummyTest.cpp:10 0 > 3
test Dummy3: ../../tests/DummyTest.cpp:14 I can too count!: 1 == 3
3 failures.
Now, we get all the messages:
Failures:
test Dummy: ../../tests/DummyTest.cpp:4 false
../../tests/DummyTest.cpp:5 I can too count!: 1 == 3
../../tests/DummyTest.cpp:6 seven is 7
test Dummy2: ../../tests/DummyTest.cpp:10 0 > 3
test Dummy3: ../../tests/DummyTest.cpp:14 I can too count!: 1 == 3
3 failures.
(Note that we still state "3 failures" because 3 DM::Tasks failed.)
BUG=skia:3082
BUG=skia:2454
Review URL: https://codereview.chromium.org/694703005
2014-11-07 14:07:32 +00:00
|
|
|
void Reporter::reportFailed(const skiatest::Failure& failure) {
|
|
|
|
this->onReportFailed(failure);
|
2009-02-27 16:24:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void Reporter::endTest(Test* test) {
|
|
|
|
this->onEnd(test);
|
|
|
|
}
|
|
|
|
|
|
|
|
///////////////////////////////////////////////////////////////////////////////
|
|
|
|
|
2013-04-19 13:24:28 +00:00
|
|
|
Test::Test() : fReporter(NULL), fPassed(true) {}
|
2009-02-27 16:24:51 +00:00
|
|
|
|
|
|
|
Test::~Test() {
|
2011-02-07 15:30:46 +00:00
|
|
|
SkSafeUnref(fReporter);
|
2009-02-27 16:24:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void Test::setReporter(Reporter* r) {
|
|
|
|
SkRefCnt_SafeAssign(fReporter, r);
|
|
|
|
}
|
|
|
|
|
|
|
|
const char* Test::getName() {
|
|
|
|
if (fName.size() == 0) {
|
|
|
|
this->onGetName(&fName);
|
|
|
|
}
|
|
|
|
return fName.c_str();
|
|
|
|
}
|
|
|
|
|
2014-01-21 23:39:22 +00:00
|
|
|
class LocalReporter : public Reporter {
|
|
|
|
public:
|
|
|
|
explicit LocalReporter(Reporter* reporterToMimic) : fReporter(reporterToMimic) {}
|
|
|
|
|
|
|
|
int numFailures() const { return fFailures.count(); }
|
When running DM, write test failures to json.
Add skiatest::Failure to keep track of data about a test failure.
Reporter::reportFailed and ::onReportFailed now take Failure as a
parameter. This allows the implementation to treat the failure as it
wishes. Provide a helper to format the failure the same as prior to
the change.
Update the macros for calling reportFailed (REPORTER_ASSERT etc) to
create a Failure object.
Convert a direct call to reportFailed to the macro ERRORF.
Write Failures to Json.
Sample output when running dm on the dummy test crrev.com/705723004:
{
"test_results" : {
"failures" : [
{
"condition" : "0 > 3",
"file_name" : "../../tests/DummyTest.cpp",
"line_no" : 10,
"message" : ""
},
{
"condition" : "false",
"file_name" : "../../tests/DummyTest.cpp",
"line_no" : 4,
"message" : ""
},
{
"condition" : "1 == 3",
"file_name" : "../../tests/DummyTest.cpp",
"line_no" : 5,
"message" : "I can too count!"
},
{
"condition" : "",
"file_name" : "../../tests/DummyTest.cpp",
"line_no" : 6,
"message" : "seven is 7"
},
{
"condition" : "1 == 3",
"file_name" : "../../tests/DummyTest.cpp",
"line_no" : 14,
"message" : "I can too count!"
}
]
}
}
Report all of the failures from one test.
Previously, if one test had multiple failures, only one was reportered.
e.g:
Failures:
test Dummy: ../../tests/DummyTest.cpp:6 seven is 7
test Dummy2: ../../tests/DummyTest.cpp:10 0 > 3
test Dummy3: ../../tests/DummyTest.cpp:14 I can too count!: 1 == 3
3 failures.
Now, we get all the messages:
Failures:
test Dummy: ../../tests/DummyTest.cpp:4 false
../../tests/DummyTest.cpp:5 I can too count!: 1 == 3
../../tests/DummyTest.cpp:6 seven is 7
test Dummy2: ../../tests/DummyTest.cpp:10 0 > 3
test Dummy3: ../../tests/DummyTest.cpp:14 I can too count!: 1 == 3
3 failures.
(Note that we still state "3 failures" because 3 DM::Tasks failed.)
BUG=skia:3082
BUG=skia:2454
Review URL: https://codereview.chromium.org/694703005
2014-11-07 14:07:32 +00:00
|
|
|
const skiatest::Failure& failure(int i) const { return fFailures[i]; }
|
2014-01-21 23:39:22 +00:00
|
|
|
|
|
|
|
protected:
|
2015-01-09 18:06:39 +00:00
|
|
|
void onReportFailed(const Failure& failure) SK_OVERRIDE {
|
When running DM, write test failures to json.
Add skiatest::Failure to keep track of data about a test failure.
Reporter::reportFailed and ::onReportFailed now take Failure as a
parameter. This allows the implementation to treat the failure as it
wishes. Provide a helper to format the failure the same as prior to
the change.
Update the macros for calling reportFailed (REPORTER_ASSERT etc) to
create a Failure object.
Convert a direct call to reportFailed to the macro ERRORF.
Write Failures to Json.
Sample output when running dm on the dummy test crrev.com/705723004:
{
"test_results" : {
"failures" : [
{
"condition" : "0 > 3",
"file_name" : "../../tests/DummyTest.cpp",
"line_no" : 10,
"message" : ""
},
{
"condition" : "false",
"file_name" : "../../tests/DummyTest.cpp",
"line_no" : 4,
"message" : ""
},
{
"condition" : "1 == 3",
"file_name" : "../../tests/DummyTest.cpp",
"line_no" : 5,
"message" : "I can too count!"
},
{
"condition" : "",
"file_name" : "../../tests/DummyTest.cpp",
"line_no" : 6,
"message" : "seven is 7"
},
{
"condition" : "1 == 3",
"file_name" : "../../tests/DummyTest.cpp",
"line_no" : 14,
"message" : "I can too count!"
}
]
}
}
Report all of the failures from one test.
Previously, if one test had multiple failures, only one was reportered.
e.g:
Failures:
test Dummy: ../../tests/DummyTest.cpp:6 seven is 7
test Dummy2: ../../tests/DummyTest.cpp:10 0 > 3
test Dummy3: ../../tests/DummyTest.cpp:14 I can too count!: 1 == 3
3 failures.
Now, we get all the messages:
Failures:
test Dummy: ../../tests/DummyTest.cpp:4 false
../../tests/DummyTest.cpp:5 I can too count!: 1 == 3
../../tests/DummyTest.cpp:6 seven is 7
test Dummy2: ../../tests/DummyTest.cpp:10 0 > 3
test Dummy3: ../../tests/DummyTest.cpp:14 I can too count!: 1 == 3
3 failures.
(Note that we still state "3 failures" because 3 DM::Tasks failed.)
BUG=skia:3082
BUG=skia:2454
Review URL: https://codereview.chromium.org/694703005
2014-11-07 14:07:32 +00:00
|
|
|
fFailures.push_back(failure);
|
2014-01-21 23:39:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Proxy down to fReporter. We assume these calls are threadsafe.
|
2015-01-09 18:06:39 +00:00
|
|
|
bool allowExtendedTest() const SK_OVERRIDE {
|
2014-01-21 23:39:22 +00:00
|
|
|
return fReporter->allowExtendedTest();
|
|
|
|
}
|
|
|
|
|
2015-01-09 18:06:39 +00:00
|
|
|
void bumpTestCount() SK_OVERRIDE {
|
2014-01-21 23:39:22 +00:00
|
|
|
fReporter->bumpTestCount();
|
|
|
|
}
|
|
|
|
|
2015-01-09 18:06:39 +00:00
|
|
|
bool verbose() const SK_OVERRIDE {
|
2014-01-21 23:39:22 +00:00
|
|
|
return fReporter->verbose();
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
Reporter* fReporter; // Unowned.
|
When running DM, write test failures to json.
Add skiatest::Failure to keep track of data about a test failure.
Reporter::reportFailed and ::onReportFailed now take Failure as a
parameter. This allows the implementation to treat the failure as it
wishes. Provide a helper to format the failure the same as prior to
the change.
Update the macros for calling reportFailed (REPORTER_ASSERT etc) to
create a Failure object.
Convert a direct call to reportFailed to the macro ERRORF.
Write Failures to Json.
Sample output when running dm on the dummy test crrev.com/705723004:
{
"test_results" : {
"failures" : [
{
"condition" : "0 > 3",
"file_name" : "../../tests/DummyTest.cpp",
"line_no" : 10,
"message" : ""
},
{
"condition" : "false",
"file_name" : "../../tests/DummyTest.cpp",
"line_no" : 4,
"message" : ""
},
{
"condition" : "1 == 3",
"file_name" : "../../tests/DummyTest.cpp",
"line_no" : 5,
"message" : "I can too count!"
},
{
"condition" : "",
"file_name" : "../../tests/DummyTest.cpp",
"line_no" : 6,
"message" : "seven is 7"
},
{
"condition" : "1 == 3",
"file_name" : "../../tests/DummyTest.cpp",
"line_no" : 14,
"message" : "I can too count!"
}
]
}
}
Report all of the failures from one test.
Previously, if one test had multiple failures, only one was reportered.
e.g:
Failures:
test Dummy: ../../tests/DummyTest.cpp:6 seven is 7
test Dummy2: ../../tests/DummyTest.cpp:10 0 > 3
test Dummy3: ../../tests/DummyTest.cpp:14 I can too count!: 1 == 3
3 failures.
Now, we get all the messages:
Failures:
test Dummy: ../../tests/DummyTest.cpp:4 false
../../tests/DummyTest.cpp:5 I can too count!: 1 == 3
../../tests/DummyTest.cpp:6 seven is 7
test Dummy2: ../../tests/DummyTest.cpp:10 0 > 3
test Dummy3: ../../tests/DummyTest.cpp:14 I can too count!: 1 == 3
3 failures.
(Note that we still state "3 failures" because 3 DM::Tasks failed.)
BUG=skia:3082
BUG=skia:2454
Review URL: https://codereview.chromium.org/694703005
2014-11-07 14:07:32 +00:00
|
|
|
SkTArray<skiatest::Failure> fFailures;
|
2014-01-21 23:39:22 +00:00
|
|
|
};
|
2013-04-19 13:24:28 +00:00
|
|
|
|
|
|
|
void Test::run() {
|
2013-04-25 18:33:49 +00:00
|
|
|
// Clear the Skia error callback before running any test, to ensure that tests
|
|
|
|
// don't have unintended side effects when running more than one.
|
|
|
|
SkSetErrorCallback( NULL, NULL );
|
|
|
|
|
2013-04-19 13:24:28 +00:00
|
|
|
// Tell (likely shared) fReporter that this test has started.
|
2009-02-27 16:24:51 +00:00
|
|
|
fReporter->startTest(this);
|
2013-04-19 13:24:28 +00:00
|
|
|
|
2013-04-22 16:43:07 +00:00
|
|
|
const SkMSec start = SkTime::GetMSecs();
|
2013-04-19 13:24:28 +00:00
|
|
|
// Run the test into a LocalReporter so we know if it's passed or failed without interference
|
|
|
|
// from other tests that might share fReporter.
|
2013-04-23 11:16:32 +00:00
|
|
|
LocalReporter local(fReporter);
|
2013-04-19 13:24:28 +00:00
|
|
|
this->onRun(&local);
|
2014-01-21 23:39:22 +00:00
|
|
|
fPassed = local.numFailures() == 0;
|
2013-04-22 16:43:07 +00:00
|
|
|
fElapsed = SkTime::GetMSecs() - start;
|
2013-04-19 13:24:28 +00:00
|
|
|
|
|
|
|
// Now tell fReporter about any failures and wrap up.
|
2014-01-21 23:39:22 +00:00
|
|
|
for (int i = 0; i < local.numFailures(); i++) {
|
2013-06-18 20:50:34 +00:00
|
|
|
fReporter->reportFailed(local.failure(i));
|
2013-04-19 13:24:28 +00:00
|
|
|
}
|
2009-02-27 16:24:51 +00:00
|
|
|
fReporter->endTest(this);
|
2013-04-25 18:33:49 +00:00
|
|
|
|
2009-02-27 16:24:51 +00:00
|
|
|
}
|
|
|
|
|
2014-02-26 16:31:22 +00:00
|
|
|
SkString Test::GetTmpDir() {
|
|
|
|
const char* tmpDir = FLAGS_tmpDir.isEmpty() ? NULL : FLAGS_tmpDir[0];
|
|
|
|
return SkString(tmpDir);
|
|
|
|
}
|