AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

When running DM, write test failures to json.

Browse Source 
Add skiatest::Failure to keep track of data about a test failure.

Reporter::reportFailed and ::onReportFailed now take Failure as a
parameter. This allows the implementation to treat the failure as it
wishes. Provide a helper to format the failure the same as prior to
the change.

Update the macros for calling reportFailed (REPORTER_ASSERT etc) to
create a Failure object.

Convert a direct call to reportFailed to the macro ERRORF.

Write Failures to Json.
Sample output when running dm on the dummy test crrev.com/705723004:
{
   "test_results" : {
      "failures" : [
         {
            "condition" : "0 > 3",
            "file_name" : "../../tests/DummyTest.cpp",
            "line_no" : 10,
            "message" : ""
         },
         {
            "condition" : "false",
            "file_name" : "../../tests/DummyTest.cpp",
            "line_no" : 4,
            "message" : ""
         },
         {
            "condition" : "1 == 3",
            "file_name" : "../../tests/DummyTest.cpp",
            "line_no" : 5,
            "message" : "I can too count!"
         },
         {
            "condition" : "",
            "file_name" : "../../tests/DummyTest.cpp",
            "line_no" : 6,
            "message" : "seven is 7"
         },
         {
            "condition" : "1 == 3",
            "file_name" : "../../tests/DummyTest.cpp",
            "line_no" : 14,
            "message" : "I can too count!"
         }
      ]
   }
}

Report all of the failures from one test.
Previously, if one test had multiple failures, only one was reportered.
e.g:

Failures:
  test Dummy: ../../tests/DummyTest.cpp:6   seven is 7
  test Dummy2: ../../tests/DummyTest.cpp:10 0 > 3
  test Dummy3: ../../tests/DummyTest.cpp:14 I can too count!: 1 == 3
3 failures.

Now, we get all the messages:

Failures:
  test Dummy: ../../tests/DummyTest.cpp:4   false
        ../../tests/DummyTest.cpp:5 I can too count!: 1 == 3
        ../../tests/DummyTest.cpp:6 seven is 7
  test Dummy2: ../../tests/DummyTest.cpp:10 0 > 3
  test Dummy3: ../../tests/DummyTest.cpp:14 I can too count!: 1 == 3
3 failures.

(Note that we still state "3 failures" because 3 DM::Tasks failed.)

BUG=skia:3082
BUG=skia:2454

Review URL: https://codereview.chromium.org/694703005
This commit is contained in:
scroggo 2014-11-07 06:07:32 -08:00 committed by Commit bot
parent f0de423f09
commit 0ee2627026
7 changed files with 105 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
dm/DMJsonWriter.cpp
View File

@ -24,6 +24,14 @@ void JsonWriter::AddBitmapResult(const BitmapResult& result) {
gBitmapResults.push_back(result);
}
SkTArray<skiatest::Failure> gFailures;
SK_DECLARE_STATIC_MUTEX(gFailureLock);
void JsonWriter::AddTestFailure(const skiatest::Failure& failure) {
SkAutoMutexAcquire lock(gFailureLock);
gFailures.push_back(failure);
}
void JsonWriter::DumpJson() {
if (FLAGS_writePath.isEmpty()) {
return;
@ -52,6 +60,19 @@ void JsonWriter::DumpJson() {
}
}
{
SkAutoMutexAcquire lock(gFailureLock);
for (int i = 0; i < gFailures.count(); i++) {
Json::Value result;
result["file_name"] = gFailures[i].fileName;
result["line_no"] = gFailures[i].lineNo;
result["condition"] = gFailures[i].condition;
result["message"] = gFailures[i].message.c_str();
root["test_results"]["failures"].append(result);
}
}
SkString path = SkOSPath::Join(FLAGS_writePath[0], "dm.json");
SkFILEWStream stream(path.c_str());
stream.writeText(Json::StyledWriter().write(root).c_str());

6
dm/DMJsonWriter.h
View File

@ -9,6 +9,7 @@
#define DMJsonWriter_DEFINED
#include "SkString.h"
#include "Test.h"
namespace DM {
@ -34,6 +35,11 @@ public:
*/
static void AddBitmapResult(const BitmapResult&);
/**
* Add a Failure from a Test.
*/
static void AddTestFailure(const skiatest::Failure&);
/**
* Write all collected results to the file FLAGS_writePath[0]/dm.json.
*/

13
dm/DMTestTask.h
View File

@ -2,6 +2,7 @@
#define DMTestTask_DEFINED
#include "DMReporter.h"
#include "DMJsonWriter.h"
#include "DMTask.h"
#include "DMTaskRunner.h"
#include "SkString.h"
@ -21,8 +22,16 @@ private:
virtual bool allowExtendedTest() const SK_OVERRIDE;
virtual bool verbose() const SK_OVERRIDE;
virtual void onReportFailed(const SkString& desc) SK_OVERRIDE {
fFailure = desc;
virtual void onReportFailed(const skiatest::Failure& failure) SK_OVERRIDE {
JsonWriter::AddTestFailure(failure);
SkString newFailure;
failure.getFailureString(&newFailure);
// TODO: Better to store an array of failures?
if (!fFailure.isEmpty()) {
fFailure.append("\n\t\t");
}
fFailure.append(newFailure);
}
SkString fFailure;

2
tests/GrTRecorderTest.cpp
View File

@ -214,7 +214,7 @@ static void test_subclasses(skiatest::Reporter* reporter) {
break;
default:
reporter->reportFailed(SkString("Invalid class type"));
ERRORF(reporter, "Invalid class type");
break;
}
}

12
tests/Test.cpp
View File

@ -31,8 +31,8 @@ void Reporter::startTest(Test* test) {
this->onStart(test);
}
void Reporter::reportFailed(const SkString& desc) {
this->onReportFailed(desc);
void Reporter::reportFailed(const skiatest::Failure& failure) {
this->onReportFailed(failure);
}
void Reporter::endTest(Test* test) {
@ -63,11 +63,11 @@ public:
explicit LocalReporter(Reporter* reporterToMimic) : fReporter(reporterToMimic) {}
int numFailures() const { return fFailures.count(); }
const SkString& failure(int i) const { return fFailures[i]; }
const skiatest::Failure& failure(int i) const { return fFailures[i]; }
protected:
virtual void onReportFailed(const SkString& desc) SK_OVERRIDE {
fFailures.push_back(desc);
virtual void onReportFailed(const Failure& failure) SK_OVERRIDE {
fFailures.push_back(failure);
}
// Proxy down to fReporter. We assume these calls are threadsafe.
@ -85,7 +85,7 @@ protected:
private:
Reporter* fReporter; // Unowned.
SkTArray<SkString> fFailures;
SkTArray<skiatest::Failure> fFailures;
};
void Test::run() {

80
tests/Test.h
View File

@ -20,6 +20,40 @@ namespace skiatest {
class Test;
/**
* Information about a single failure from a Test.
*
* Not intended to be created/modified directly. To create one, use one of
*
* REPORTER_ASSERT
* REPORTER_ASSERT_MESSAGE
* ERRORF
*
* described in more detail further down in this file.
*/
struct Failure {
const char* fileName;
int lineNo;
const char* condition;
SkString message;
// Helper to combine the failure info into one string.
void getFailureString(SkString* result) const {
if (!result) {
return;
}
result->printf("%s:%d\t", fileName, lineNo);
if (!message.isEmpty()) {
result->append(message);
if (strlen(condition) > 0) {
result->append(": ");
}
}
result->append(condition);
}
};
class Reporter : public SkRefCnt {
public:
SK_DECLARE_INST_COUNT(Reporter)
@ -28,7 +62,7 @@ namespace skiatest {
int countTests() const { return fTestCount; }
void startTest(Test*);
void reportFailed(const SkString& desc);
void reportFailed(const Failure&);
void endTest(Test*);
virtual bool allowExtendedTest() const { return false; }
@ -37,7 +71,7 @@ namespace skiatest {
protected:
virtual void onStart(Test*) {}
virtual void onReportFailed(const SkString& desc) {}
virtual void onReportFailed(const Failure&) {}
virtual void onEnd(Test*) {}
private:
@ -110,31 +144,31 @@ namespace skiatest {
}
*/
#define REPORTER_ASSERT(r, cond) \
do { \
if (!(cond)) { \
SkString desc; \
desc.printf("%s:%d\t%s", __FILE__, __LINE__, #cond); \
r->reportFailed(desc); \
} \
#define REPORTER_ASSERT(r, cond) \
do { \
if (!(cond)) { \
skiatest::Failure failure = { __FILE__, __LINE__, \
#cond, SkString() }; \
r->reportFailed(failure); \
} \
} while(0)
#define REPORTER_ASSERT_MESSAGE(r, cond, message) \
do { \
if (!(cond)) { \
SkString desc; \
desc.printf("%s:%d\t%s: %s", __FILE__, __LINE__, \
message, #cond); \
r->reportFailed(desc); \
} \
#define REPORTER_ASSERT_MESSAGE(r, cond, message) \
do { \
if (!(cond)) { \
skiatest::Failure failure = { __FILE__, __LINE__, \
#cond, SkString(message) }; \
r->reportFailed(failure); \
} \
} while(0)
#define ERRORF(reporter, ...) \
do { \
SkString desc; \
desc.printf("%s:%d\t", __FILE__, __LINE__); \
desc.appendf(__VA_ARGS__) ; \
(reporter)->reportFailed(desc); \
#define ERRORF(r, ...) \
do { \
SkString desc; \
desc.appendf(__VA_ARGS__) ; \
skiatest::Failure failure = { __FILE__, __LINE__, \
"", SkString(desc) }; \
r->reportFailed(failure); \
} while(0)
#define DEF_TEST(name, reporter) \

4
tests/skia_test.cpp
View File

@ -58,7 +58,9 @@ public:
virtual bool verbose() const SK_OVERRIDE { return FLAGS_veryVerbose; }
protected:
virtual void onReportFailed(const SkString& desc) SK_OVERRIDE {
virtual void onReportFailed(const skiatest::Failure& failure) SK_OVERRIDE {
SkString desc;
failure.getFailureString(&desc);
SkDebugf("\nFAILED: %s", desc.c_str());
}