From 5aa11fb67a72b4b3e3fce4b1ed749e1f487317bc Mon Sep 17 00:00:00 2001 From: Brian Osman Date: Mon, 8 Apr 2019 16:40:36 -0400 Subject: [PATCH] Shader serialization experiment (with Mali Offline Compiler analysis) 1) Adds a --writeShaders option to fm. When that's included, the GPU backend is run with a persistent cache (and no binary caching). Then we dump all of the fragment shaders (GLSL for now) that were created to the passed-in directory, along with a JSON digest listing the number of times each one was referenced in the cache. 2) Adds a python script that invokes the Mali Offline Compiler on a directory generated from #1, scraping the output of each compile for cycle counts and writing the collated information to stdout. Change-Id: Ie4b58ddac4f62e936707c6fec44f4fe605c213fa Reviewed-on: https://skia-review.googlesource.com/c/skia/+/206162 Commit-Queue: Brian Osman Reviewed-by: Mike Klein --- tools/fm/fm.cpp | 18 +++++++++++++ tools/gpu/MemoryCache.cpp | 56 ++++++++++++++++++++++++++++++++++++--- tools/gpu/MemoryCache.h | 16 ++++++++++- tools/malisc/malisc.py | 36 +++++++++++++++++++++++++ 4 files changed, 122 insertions(+), 4 deletions(-) create mode 100644 tools/malisc/malisc.py diff --git a/tools/fm/fm.cpp b/tools/fm/fm.cpp index 998b35fa6d..4ad1b8f611 100644 --- a/tools/fm/fm.cpp +++ b/tools/fm/fm.cpp @@ -8,7 +8,9 @@ #include "GrContextOptions.h" #include "GrContextPriv.h" #include "GrGpu.h" +#include "GrPersistentCacheUtils.h" #include "HashAndEncode.h" +#include "MemoryCache.h" #include "SkCodec.h" #include "SkColorSpace.h" #include "SkColorSpacePriv.h" @@ -62,6 +64,8 @@ static DEFINE_bool(PDFA, false, "Create PDF/A with --backend pdf?"); static DEFINE_bool (cpuDetect, true, "Detect CPU features for runtime optimizations?"); static DEFINE_string2(writePath, w, "", "Write .pngs to this directory if set."); +static DEFINE_string(writeShaders, "", "Write GLSL shaders to this directory if set."); + static DEFINE_string(key, "", "Metadata passed through to .png encoder and .json output."); static DEFINE_string(parameters, "", "Metadata passed through to .png encoder and .json output."); @@ -366,6 +370,12 @@ int main(int argc, char** argv) { GrContextOptions baseOptions; SetCtxOptionsFromCommonFlags(&baseOptions); + sk_gpu_test::MemoryCache memoryCache; + if (!FLAGS_writeShaders.isEmpty()) { + baseOptions.fPersistentCache = &memoryCache; + baseOptions.fDisallowGLSLBinaryCaching = true; + } + SkTHashMap gm_factories; for (skiagm::GMFactory factory : skiagm::GMRegistry::Range()) { std::unique_ptr gm{factory(nullptr)}; @@ -573,5 +583,13 @@ int main(int argc, char** argv) { (int)std::chrono::duration_cast(elapsed).count()); } + if (!FLAGS_writeShaders.isEmpty()) { + sk_mkdir(FLAGS_writeShaders[0]); + GrBackendApi api = + GrContextFactory::ContextTypeBackend((GrContextFactory::ContextType)backend); + memoryCache.writeShadersToDisk(FLAGS_writeShaders[0], api); + + } + return 0; } diff --git a/tools/gpu/MemoryCache.cpp b/tools/gpu/MemoryCache.cpp index 7fc3e1b5d0..771d1a1b5c 100644 --- a/tools/gpu/MemoryCache.cpp +++ b/tools/gpu/MemoryCache.cpp @@ -5,8 +5,12 @@ * found in the LICENSE file. */ +#include "GrPersistentCacheUtils.h" #include "MemoryCache.h" #include "SkBase64.h" +#include "SkJSONWriter.h" +#include "SkMD5.h" +#include "SkTHash.h" // Change this to 1 to log cache hits/misses/stores using SkDebugf. #define LOG_MEMORY_CACHE 0 @@ -40,9 +44,10 @@ sk_sp MemoryCache::load(const SkData& key) { } if (LOG_MEMORY_CACHE) { SkDebugf("Load Key: %s\n\tFound Data: %s\n\n", data_to_str(key).c_str(), - data_to_str(*result->second).c_str()); + data_to_str(*result->second.fData).c_str()); } - return result->second; + result->second.fHitCount++; + return result->second.fData; } void MemoryCache::store(const SkData& key, const SkData& data) { @@ -50,7 +55,52 @@ void MemoryCache::store(const SkData& key, const SkData& data) { SkDebugf("Store Key: %s\n\tData: %s\n\n", data_to_str(key).c_str(), data_to_str(data).c_str()); } - fMap[Key(key)] = SkData::MakeWithCopy(data.data(), data.size()); + fMap[Key(key)] = Value(data); +} + +void MemoryCache::writeShadersToDisk(const char* path, GrBackendApi api) { + if (GrBackendApi::kOpenGL != api) { + // TODO: Add SPIRV support, too. + return; + } + + // Default extensions detected by the Mali Offline Compiler + const char* extensions[kGrShaderTypeCount] = { "vert", "geom", "frag" }; + + // For now, we only dump fragment shaders. They are the biggest factor in performance, and + // the shaders for other stages tend to be heavily reused. + SkString jsonPath = SkStringPrintf("%s/%s.json", path, extensions[kFragment_GrShaderType]); + SkFILEWStream jsonFile(jsonPath.c_str()); + SkJSONWriter writer(&jsonFile, SkJSONWriter::Mode::kPretty); + writer.beginArray(); + + for (auto it = fMap.begin(); it != fMap.end(); ++it) { + SkMD5 hash; + hash.write(it->first.fKey->bytes(), it->first.fKey->size()); + SkMD5::Digest digest = hash.finish(); + SkString md5; + for (int i = 0; i < 16; ++i) { + md5.appendf("%02x", digest.data[i]); + } + + // Write [ hash, hitCount ] to JSON digest + writer.beginArray(nullptr, false); + writer.appendString(md5.c_str()); + writer.appendS32(it->second.fHitCount); + writer.endArray(); + + SkReader32 reader(it->second.fData->data(), it->second.fData->size()); + SkSL::Program::Inputs inputsIgnored; + SkSL::String glsl[kGrShaderTypeCount]; + GrPersistentCacheUtils::UnpackCachedGLSL(reader, &inputsIgnored, glsl); + + SkString filename = SkStringPrintf("%s/%s.%s", path, md5.c_str(), + extensions[kFragment_GrShaderType]); + SkFILEWStream file(filename.c_str()); + file.write(glsl[kFragment_GrShaderType].c_str(), glsl[kFragment_GrShaderType].size()); + } + + writer.endArray(); } } // namespace sk_gpu_test diff --git a/tools/gpu/MemoryCache.h b/tools/gpu/MemoryCache.h index dea1ccdafb..119ea8dede 100644 --- a/tools/gpu/MemoryCache.h +++ b/tools/gpu/MemoryCache.h @@ -33,6 +33,8 @@ public: int numCacheMisses() const { return fCacheMissCnt; } void resetNumCacheMisses() { fCacheMissCnt = 0; } + void writeShadersToDisk(const char* path, GrBackendApi backend); + private: struct Key { Key() = default; @@ -46,6 +48,18 @@ private: sk_sp fKey; }; + struct Value { + Value() = default; + Value(const SkData& data) + : fData(SkData::MakeWithCopy(data.data(), data.size())) + , fHitCount(1) {} + Value(const Value& that) = default; + Value& operator=(const Value&) = default; + + sk_sp fData; + int fHitCount; + }; + struct Hash { using argument_type = Key; using result_type = uint32_t; @@ -55,7 +69,7 @@ private: }; int fCacheMissCnt = 0; - std::unordered_map, Hash> fMap; + std::unordered_map fMap; }; } // namespace sk_gpu_test diff --git a/tools/malisc/malisc.py b/tools/malisc/malisc.py new file mode 100644 index 0000000000..000405fc8e --- /dev/null +++ b/tools/malisc/malisc.py @@ -0,0 +1,36 @@ +# Copyright 2019 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import json +import os +import subprocess +import sys + +if len(sys.argv) != 3: + print sys.argv[0], ' ' + sys.exit(1) + +compiler = sys.argv[1] +folder = sys.argv[2] + +FRAG_JSON = os.path.join(folder, 'frag.json') + +with open(FRAG_JSON) as f: + frags = json.load(f) + +if not frags: + print 'No JSON data' + sys.exit(1) + +for fragAndCount in frags: + source = os.path.join(folder, fragAndCount[0] + '.frag') + try: + output = subprocess.check_output([compiler, source]) + except subprocess.CalledProcessError: + continue + for line in output.splitlines(): + if line.startswith('Instructions Emitted'): + inst = line.split(':')[1].split() + print '{0} {1} {2} {3} {4}'.format( + fragAndCount[0], fragAndCount[1], inst[0], inst[1], inst[2])