AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

SkQP: better error report workflow

Browse Source 
* Generate report in app after running all tests.
  * Add script to unpack backup.ab file.
  * Remove unused index file.
  * Streamline instructions in README.md.

Change-Id: I44c80b17332eb4496ee31578287b691bd646d71a
Reviewed-on: https://skia-review.googlesource.com/86742
Commit-Queue: Hal Canary <halcanary@google.com>
Reviewed-by: Hal Canary <halcanary@google.com>
This commit is contained in:
Hal Canary 2017-12-18 16:59:56 -05:00 committed by Skia Commit-Bot
parent 45e57732dd
commit 2a7f0aa9eb
9 changed files with 137 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/SkQPRunner.java
View File

@ -24,6 +24,7 @@ public class SkQPRunner extends Runner {
private native void nInit(AssetManager assetManager, String dataDir);
private native float nExecuteGM(int gm, int backend) throws SkQPException;
private native String[] nExecuteUnitTest(int test);
private native void nMakeReport();
private AssetManager mAssetManager;
private String[] mGMs;
@ -128,6 +129,7 @@ public class SkQPRunner extends Runner {
}
notifier.fireTestFinished(desc);
}
this.nMakeReport();
}
}

17
tools/skqp/README.md
View File

@ -35,7 +35,6 @@ How To Use SkQP on your Android device:
5. Generate the validation model data:
rm -rf platform_tools/android/apps/skqp/src/main/assets/gmkb
go get go.skia.org/infra/golden/go/search
go run tools/skqp/make_gmkb.go ~/Downloads/meta.json \
platform_tools/android/apps/skqp/src/main/assets/gmkb
@ -52,12 +51,11 @@ Run as an executable
adb push out/${arch}-rel/skqp /data/local/tmp/
adb shell "cd /data/local/tmp; ./skqp gmkb report"
2. Produce a one-page error report if there are errors:
2. Get the error report if there are errors:
rm -rf /tmp/report
if adb shell test -d /data/local/tmp/report; then
adb pull /data/local/tmp/report /tmp/
tools/skqp/make_report.py /tmp/report
tools/skqp/sysopen.py /tmp/report/report.html
fi
Run as an APK
@ -72,11 +70,8 @@ Run as an APK
2. Retrieve the report if there are any errors:
rm -rf /tmp/skqp
mkdir /tmp/skqp
adb backup -f /tmp/skqp/backup.ab org.skia.skqp
dd if=/tmp/skqp/backup.ab bs=24 skip=1 | tools/skqp/inflate.py | \
( cd /tmp/skqp; tar x )
rm /tmp/skqp/backup.ab
tools/skqp/make_report.py /tmp/skqp/apps/org.skia.skqp/f
adb backup -f /tmp/skqp.ab org.skia.skqp
# Must unlock phone and verify backup.
tools/skqp/extract_report.py /tmp/skqp.ab

29
tools/skqp/extract_report.py Executable file
View File

@ -0,0 +1,29 @@
#! /usr/bin/env python2
# Copyright 2017 Google Inc.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import StringIO
import os
import sys
import sysopen
import tarfile
import tempfile
import zlib
if __name__ == '__main__':
if len(sys.argv) != 2:
print 'usage: %s FILE.ab\n' % sys.argv[0]
exit (1)
with open(sys.argv[1], 'rb') as f:
f.read(24)
t = tarfile.open(fileobj=StringIO.StringIO(zlib.decompress(f.read())))
d = tempfile.mkdtemp(prefix='skqp_')
t.extractall(d)
p = os.path.join(d, 'apps/org.skia.skqp/f/skqp_report/report.html')
assert os.path.isfile(p)
print p
sysopen.sysopen(p)

120
tools/skqp/gm_knowledge.cpp
View File

@ -8,18 +8,23 @@
#include "gm_knowledge.h"
#include <cfloat>
#include <cstdlib>
#include <fstream>
#include <mutex>
#include <sstream>
#include <string>
#include <vector>
#include "../../src/core/SkStreamPriv.h"
#include "../../src/core/SkTSort.h"
#include "SkBitmap.h"
#include "SkCodec.h"
#include "SkOSFile.h"
#include "SkOSPath.h"
#include "SkPngEncoder.h"
#include "SkStream.h"
#include "skqp_asset_manager.h"
#define PATH_MAX_PNG "max.png"
@ -28,27 +33,6 @@
#define PATH_ERR_PNG "errors.png"
#define PATH_REPORT "report.html"
////////////////////////////////////////////////////////////////////////////////
inline void path_join_append(std::ostringstream* o) { }
template<class... Types>
void path_join_append(std::ostringstream* o, const char* v, Types... args) {
constexpr char kPathSeparator[] = "/";
*o << kPathSeparator << v;
path_join_append(o, args...);
}
template<class... Types>
std::string path_join(const char* v, Types... args) {
std::ostringstream o;
o << v;
path_join_append(&o, args...);
return o.str();
}
template<class... Types>
std::string path_join(const std::string& v, Types... args) {
return path_join(v.c_str(), args...);
}
////////////////////////////////////////////////////////////////////////////////
static int get_error(uint32_t value, uint32_t value_max, uint32_t value_min) {
@ -125,7 +109,24 @@ static SkBitmap ReadPngRgba8888FromFile(skqp::AssetManager* assetManager, const
return bitmap;
}
namespace {
struct Run {
SkString fBackend;
SkString fGM;
int fMaxerror;
int fBadpixels;
};
} // namespace
static std::vector<Run> gErrors;
static std::mutex gMutex;
namespace gmkb {
bool IsGoodGM(const char* name, skqp::AssetManager* assetManager) {
return asset_exists(assetManager, SkOSPath::Join(name, PATH_MAX_PNG).c_str())
&& asset_exists(assetManager, SkOSPath::Join(name, PATH_MIN_PNG).c_str());
}
// Assumes that for each GM foo, asset_manager has files foo/{max,min}.png
float Check(const uint32_t* pixels,
int width,
@ -135,13 +136,12 @@ float Check(const uint32_t* pixels,
skqp::AssetManager* assetManager,
const char* report_directory_path,
Error* error_out) {
using std::string;
if (width <= 0 || height <= 0) {
return set_error_code(error_out, Error::kBadInput);
}
size_t N = (unsigned)width * (unsigned)height;
string max_path = path_join(name, PATH_MAX_PNG);
string min_path = path_join(name, PATH_MIN_PNG);
SkString max_path = SkOSPath::Join(name, PATH_MAX_PNG);
SkString min_path = SkOSPath::Join(name, PATH_MIN_PNG);
SkBitmap max_image = ReadPngRgba8888FromFile(assetManager, max_path.c_str());
if (max_image.isNull()) {
return set_error_code(error_out, Error::kBadData);
@ -175,11 +175,11 @@ float Check(const uint32_t* pixels,
if (!backend) {
backend = "skia";
}
string report_directory = path_join(report_directory_path, backend);
SkString report_directory = SkOSPath::Join(report_directory_path, backend);
sk_mkdir(report_directory.c_str());
string report_subdirectory = path_join(report_directory, name);
SkString report_subdirectory = SkOSPath::Join(report_directory.c_str(), name);
sk_mkdir(report_subdirectory.c_str());
string error_path = path_join(report_subdirectory, PATH_IMG_PNG);
SkString error_path = SkOSPath::Join(report_subdirectory.c_str(), PATH_IMG_PNG);
SkAssertResult(WritePixmapToFile(rgba8888_to_pixmap(pixels, width, height),
error_path.c_str()));
SkBitmap errorBitmap;
@ -189,18 +189,52 @@ float Check(const uint32_t* pixels,
int error = get_error(pixels[i], max_pixels[i], min_pixels[i]);
errors[i] = error > 0 ? 0xFF000000 + (unsigned)error : 0x00000000;
}
error_path = path_join(report_subdirectory, PATH_ERR_PNG);
error_path = SkOSPath::Join(report_subdirectory.c_str(), PATH_ERR_PNG);
SkAssertResult(WritePixmapToFile(to_pixmap(errorBitmap), error_path.c_str()));
auto report_path = path_join(report_subdirectory, PATH_REPORT);
auto rdir = path_join("..", "..", backend, name);
SkString report_path = SkOSPath::Join(report_subdirectory.c_str(), PATH_REPORT);
auto max_path_out = path_join(report_subdirectory, PATH_MAX_PNG);
auto min_path_out = path_join(report_subdirectory, PATH_MIN_PNG);
SkString max_path_out = SkOSPath::Join(report_subdirectory.c_str(), PATH_MAX_PNG);
SkString min_path_out = SkOSPath::Join(report_subdirectory.c_str(), PATH_MIN_PNG);
(void)copy(assetManager, max_path.c_str(), max_path_out.c_str());
(void)copy(assetManager, min_path.c_str(), min_path_out.c_str());
std::lock_guard<std::mutex> lock(gMutex);
gErrors.push_back(Run{SkString(backend), SkString(name), badness, badPixelCount});
}
if (error_out) {
*error_out = Error::kNone;
}
return (float)badness;
}
bool MakeReport(const char* report_directory_path) {
SkFILEWStream out(SkOSPath::Join(report_directory_path, PATH_REPORT).c_str());
if (!out.isValid()) {
return false;
}
out.writeText(
"<!doctype html>\n"
"<html lang=\"en\">\n"
"<head>\n"
"<meta charset=\"UTF-8\">\n"
"<title>SkQP Report</title>\n"
"<style>\n"
"img { max-width:48%; border:1px green solid; }\n"
"</style>\n"
"</head>\n"
"<body>\n"
"<h1>SkQP Report</h1>\n"
"<hr>\n");
std::lock_guard<std::mutex> lock(gMutex);
for (const Run& run : gErrors) {
const SkString& backend = run.fBackend;
const SkString& gm = run.fGM;
int maxerror = run.fMaxerror;
int badpixels = run.fBadpixels;
SkString rdir = SkOSPath::Join(backend.c_str(), gm.c_str());
SkString text = SkStringPrintf(
"<h2>%s</h2>\n"
"backend: %s\n<br>\n"
"gm name: %s\n<br>\n"
"maximum error: %d\n<br>\n"
@ -210,21 +244,13 @@ float Check(const uint32_t* pixels,
"<a href=\"%s/" PATH_ERR_PNG "\">"
"<img src=\"%s/" PATH_ERR_PNG "\" alt='err'></a>\n<br>\n"
"<a href=\"%s/" PATH_MAX_PNG "\">max</a>\n<br>\n"
"<a href=\"%s/" PATH_MIN_PNG "\">min</a>\n<hr>\n",
backend, name, badness, badPixelCount,
rdir.c_str(), rdir.c_str(), rdir.c_str(), rdir.c_str(), rdir.c_str(), rdir.c_str());
SkFILEWStream(report_path.c_str()).write(text.c_str(), text.size());
"<a href=\"%s/" PATH_MIN_PNG "\">min</a>\n<hr>\n\n",
rdir.c_str(), backend.c_str(), gm.c_str(), maxerror, badpixels,
rdir.c_str(), rdir.c_str(), rdir.c_str(),
rdir.c_str(), rdir.c_str(), rdir.c_str());
out.write(text.c_str(), text.size());
}
if (error_out) {
*error_out = Error::kNone;
}
return (float)badness;
}
bool IsGoodGM(const char* name, skqp::AssetManager* assetManager) {
std::string max_path = path_join(name, PATH_MAX_PNG);
std::string min_path = path_join(name, PATH_MIN_PNG);
return asset_exists(assetManager, max_path.c_str())
&& asset_exists(assetManager, min_path.c_str());
out.writeText("</body>\n</html>\n");
return true;
}
} // namespace gmkb

6
tools/skqp/gm_knowledge.h
View File

@ -66,6 +66,12 @@ Check to see if the given test has expected results.
*/
bool IsGoodGM(const char* name, skqp::AssetManager* assetManager);
/**
Call this after running all checks.
@param report_directory_path locatation to write report to.
*/
bool MakeReport(const char* report_directory_path);
} // namespace gmkb
#endif // gm_knowledge_DEFINED

13
tools/skqp/jni/org_skia_skqp_SkQPRunner.cpp
View File

@ -13,6 +13,7 @@
#include <android/asset_manager_jni.h>
#include "gm_runner.h"
#include "gm_knowledge.h"
#include "skqp_asset_manager.h"
#include "SkStream.h"
@ -22,6 +23,7 @@ JNIEXPORT void JNICALL Java_org_skia_skqp_SkQPRunner_nInit(JNIEnv*, jobject, job
JNIEXPORT jfloat JNICALL Java_org_skia_skqp_SkQPRunner_nExecuteGM(JNIEnv*, jobject, jint, jint);
JNIEXPORT jobjectArray JNICALL Java_org_skia_skqp_SkQPRunner_nExecuteUnitTest(JNIEnv*, jobject,
jint);
JNIEXPORT void JNICALL Java_org_skia_skqp_SkQPRunner_nMakeReport(JNIEnv*, jobject);
} // extern "C"
////////////////////////////////////////////////////////////////////////////////
@ -128,7 +130,7 @@ void Java_org_skia_skqp_SkQPRunner_nInit(JNIEnv* env, jobject object, jobject as
jassert(env, gAssetManager.fMgr);
const char* dataDirString = env->GetStringUTFChars(dataDir, nullptr);
gReportDirectory = dataDirString;
gReportDirectory = std::string(dataDirString) + "/skqp_report";
env->ReleaseStringUTFChars(dataDir, dataDirString);
gBackends = gm_runner::GetSupportedBackends();
@ -191,5 +193,14 @@ jobjectArray Java_org_skia_skqp_SkQPRunner_nExecuteUnitTest(JNIEnv* env,
return (jobjectArray)env->NewGlobalRef(array);
}
void Java_org_skia_skqp_SkQPRunner_nMakeReport(JNIEnv*, jobject) {
std::string reportDirectoryPath;
{
std::lock_guard<std::mutex> lock(gMutex);
reportDirectoryPath = gReportDirectory;
}
(void)gmkb::MakeReport(reportDirectoryPath.c_str());
}
////////////////////////////////////////////////////////////////////////////////

18
tools/skqp/make_gmkb.go
View File

@ -174,8 +174,11 @@ func main() {
}
input := os.Args[1]
output := os.Args[2]
err := os.MkdirAll(output, os.ModePerm)
if err != nil && !os.IsExist(err) {
// output is removed and replaced with a clean directory.
if err := os.RemoveAll(output); err != nil && !os.IsNotExist(err) {
log.Fatal(err)
}
if err := os.MkdirAll(output, os.ModePerm); err != nil && !os.IsExist(err) {
log.Fatal(err)
}
@ -185,12 +188,6 @@ func main() {
}
sort.Sort(ExportTestRecordArray(records))
index, err := os.Create(path.Join(output, "index.txt"))
if err != nil {
log.Fatal(err)
}
defer index.Close()
var wg sync.WaitGroup
for _, record := range records {
if in(record.TestName, blacklist) {
@ -212,11 +209,6 @@ func main() {
}
fmt.Printf("\r%-60s", testName)
}(record.TestName, goodUrls, output)
_, err = fmt.Fprintf(index, "%s\n", record.TestName)
if err != nil {
log.Fatal(err)
}
}
wg.Wait()
fmt.Printf("\r%60s\n", "")

41
tools/skqp/make_report.py
View File

@ -1,41 +0,0 @@
#! /usr/bin/env python
# Copyright 2017 Google Inc.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import glob
import os
import re
import sys
import sysopen
if len(sys.argv) == 2:
if not os.path.isdir(sys.argv[1]):
exit(1)
os.chdir(sys.argv[1])
head = '''<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>SkQP Report</title>
<style>
img { max-width:48%; border:1px green solid; }
</style>
</head>
<body>
<h1>SkQP Report</h1>
<hr>
'''
reg = re.compile('="../../')
with open('report.html', 'w') as o:
o.write(head)
for x in glob.iglob('*/*/report.html'):
with open(x, 'r') as f:
o.write(reg.sub('="', f.read()))
o.write('</body>\n</html>\n')
sysopen.sysopen('report.html')

5
tools/skqp/skqp.cpp
View File

@ -5,6 +5,7 @@
* found in the LICENSE file.
*/
#include "gm_knowledge.h"
#include "gm_runner.h"
#ifdef __clang__
@ -136,5 +137,7 @@ int main(int argc, char** argv) {
gReportDirectoryPath = argv[2];
}
register_skia_tests();
return RUN_ALL_TESTS();
int ret = RUN_ALL_TESTS();
(void)gmkb::MakeReport(gReportDirectoryPath.c_str());
return ret;
}