AuroraMiddleware/SPIRV-Tools
1
0
Fork 0
mirror of https://github.com/KhronosGroup/SPIRV-Tools synced 2024-11-29 14:31:04 +00:00
Code Issues Packages Projects Releases Wiki Activity
0f335cf87e
SPIRV-Tools/tools/stats/stats_analyzer.cpp
David Neto 0c13467161 Consistently include latest spirv.h header file. 
Use indirection through latest_version_spirv.h

Also, when generating enum tables, use the unified1 JSON grammar since
it now has FragmentFullyCoveredEXT but the other JSON grammars don't.
They are starting to fall behind.
2018-02-27 18:47:29 -05:00

875 lines
27 KiB
C++
Raw Blame History

// Copyright (c) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "stats_analyzer.h"
#include <algorithm>
#include <cassert>
#include <cstring>
#include <iostream>
#include <sstream>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include "latest_version_spirv_header.h"
#include "source/comp/markv_model.h"
#include "source/enum_string_mapping.h"
#include "source/opcode.h"
#include "source/operand.h"
#include "source/spirv_constant.h"
#include "source/util/huffman_codec.h"
using libspirv::SpirvStats;
using spvutils::HuffmanCodec;
namespace {
// Signals that the value is not in the coding scheme and a fallback method
// needs to be used.
const uint64_t kMarkvNoneOfTheAbove =
spvtools::MarkvModel::GetMarkvNoneOfTheAbove();
inline uint32_t CombineOpcodeAndNumOperands(uint32_t opcode,
uint32_t num_operands) {
return opcode | (num_operands << 16);
}
// Returns all SPIR-V v1.2 opcodes.
std::vector<uint32_t> GetAllOpcodes() {
return std::vector<uint32_t>({
SpvOpNop,
SpvOpUndef,
SpvOpSourceContinued,
SpvOpSource,
SpvOpSourceExtension,
SpvOpName,
SpvOpMemberName,
SpvOpString,
SpvOpLine,
SpvOpExtension,
SpvOpExtInstImport,
SpvOpExtInst,
SpvOpMemoryModel,
SpvOpEntryPoint,
SpvOpExecutionMode,
SpvOpCapability,
SpvOpTypeVoid,
SpvOpTypeBool,
SpvOpTypeInt,
SpvOpTypeFloat,
SpvOpTypeVector,
SpvOpTypeMatrix,
SpvOpTypeImage,
SpvOpTypeSampler,
SpvOpTypeSampledImage,
SpvOpTypeArray,
SpvOpTypeRuntimeArray,
SpvOpTypeStruct,
SpvOpTypeOpaque,
SpvOpTypePointer,
SpvOpTypeFunction,
SpvOpTypeEvent,
SpvOpTypeDeviceEvent,
SpvOpTypeReserveId,
SpvOpTypeQueue,
SpvOpTypePipe,
SpvOpTypeForwardPointer,
SpvOpConstantTrue,
SpvOpConstantFalse,
SpvOpConstant,
SpvOpConstantComposite,
SpvOpConstantSampler,
SpvOpConstantNull,
SpvOpSpecConstantTrue,
SpvOpSpecConstantFalse,
SpvOpSpecConstant,
SpvOpSpecConstantComposite,
SpvOpSpecConstantOp,
SpvOpFunction,
SpvOpFunctionParameter,
SpvOpFunctionEnd,
SpvOpFunctionCall,
SpvOpVariable,
SpvOpImageTexelPointer,
SpvOpLoad,
SpvOpStore,
SpvOpCopyMemory,
SpvOpCopyMemorySized,
SpvOpAccessChain,
SpvOpInBoundsAccessChain,
SpvOpPtrAccessChain,
SpvOpArrayLength,
SpvOpGenericPtrMemSemantics,
SpvOpInBoundsPtrAccessChain,
SpvOpDecorate,
SpvOpMemberDecorate,
SpvOpDecorationGroup,
SpvOpGroupDecorate,
SpvOpGroupMemberDecorate,
SpvOpVectorExtractDynamic,
SpvOpVectorInsertDynamic,
SpvOpVectorShuffle,
SpvOpCompositeConstruct,
SpvOpCompositeExtract,
SpvOpCompositeInsert,
SpvOpCopyObject,
SpvOpTranspose,
SpvOpSampledImage,
SpvOpImageSampleImplicitLod,
SpvOpImageSampleExplicitLod,
SpvOpImageSampleDrefImplicitLod,
SpvOpImageSampleDrefExplicitLod,
SpvOpImageSampleProjImplicitLod,
SpvOpImageSampleProjExplicitLod,
SpvOpImageSampleProjDrefImplicitLod,
SpvOpImageSampleProjDrefExplicitLod,
SpvOpImageFetch,
SpvOpImageGather,
SpvOpImageDrefGather,
SpvOpImageRead,
SpvOpImageWrite,
SpvOpImage,
SpvOpImageQueryFormat,
SpvOpImageQueryOrder,
SpvOpImageQuerySizeLod,
SpvOpImageQuerySize,
SpvOpImageQueryLod,
SpvOpImageQueryLevels,
SpvOpImageQuerySamples,
SpvOpConvertFToU,
SpvOpConvertFToS,
SpvOpConvertSToF,
SpvOpConvertUToF,
SpvOpUConvert,
SpvOpSConvert,
SpvOpFConvert,
SpvOpQuantizeToF16,
SpvOpConvertPtrToU,
SpvOpSatConvertSToU,
SpvOpSatConvertUToS,
SpvOpConvertUToPtr,
SpvOpPtrCastToGeneric,
SpvOpGenericCastToPtr,
SpvOpGenericCastToPtrExplicit,
SpvOpBitcast,
SpvOpSNegate,
SpvOpFNegate,
SpvOpIAdd,
SpvOpFAdd,
SpvOpISub,
SpvOpFSub,
SpvOpIMul,
SpvOpFMul,
SpvOpUDiv,
SpvOpSDiv,
SpvOpFDiv,
SpvOpUMod,
SpvOpSRem,
SpvOpSMod,
SpvOpFRem,
SpvOpFMod,
SpvOpVectorTimesScalar,
SpvOpMatrixTimesScalar,
SpvOpVectorTimesMatrix,
SpvOpMatrixTimesVector,
SpvOpMatrixTimesMatrix,
SpvOpOuterProduct,
SpvOpDot,
SpvOpIAddCarry,
SpvOpISubBorrow,
SpvOpUMulExtended,
SpvOpSMulExtended,
SpvOpAny,
SpvOpAll,
SpvOpIsNan,
SpvOpIsInf,
SpvOpIsFinite,
SpvOpIsNormal,
SpvOpSignBitSet,
SpvOpLessOrGreater,
SpvOpOrdered,
SpvOpUnordered,
SpvOpLogicalEqual,
SpvOpLogicalNotEqual,
SpvOpLogicalOr,
SpvOpLogicalAnd,
SpvOpLogicalNot,
SpvOpSelect,
SpvOpIEqual,
SpvOpINotEqual,
SpvOpUGreaterThan,
SpvOpSGreaterThan,
SpvOpUGreaterThanEqual,
SpvOpSGreaterThanEqual,
SpvOpULessThan,
SpvOpSLessThan,
SpvOpULessThanEqual,
SpvOpSLessThanEqual,
SpvOpFOrdEqual,
SpvOpFUnordEqual,
SpvOpFOrdNotEqual,
SpvOpFUnordNotEqual,
SpvOpFOrdLessThan,
SpvOpFUnordLessThan,
SpvOpFOrdGreaterThan,
SpvOpFUnordGreaterThan,
SpvOpFOrdLessThanEqual,
SpvOpFUnordLessThanEqual,
SpvOpFOrdGreaterThanEqual,
SpvOpFUnordGreaterThanEqual,
SpvOpShiftRightLogical,
SpvOpShiftRightArithmetic,
SpvOpShiftLeftLogical,
SpvOpBitwiseOr,
SpvOpBitwiseXor,
SpvOpBitwiseAnd,
SpvOpNot,
SpvOpBitFieldInsert,
SpvOpBitFieldSExtract,
SpvOpBitFieldUExtract,
SpvOpBitReverse,
SpvOpBitCount,
SpvOpDPdx,
SpvOpDPdy,
SpvOpFwidth,
SpvOpDPdxFine,
SpvOpDPdyFine,
SpvOpFwidthFine,
SpvOpDPdxCoarse,
SpvOpDPdyCoarse,
SpvOpFwidthCoarse,
SpvOpEmitVertex,
SpvOpEndPrimitive,
SpvOpEmitStreamVertex,
SpvOpEndStreamPrimitive,
SpvOpControlBarrier,
SpvOpMemoryBarrier,
SpvOpAtomicLoad,
SpvOpAtomicStore,
SpvOpAtomicExchange,
SpvOpAtomicCompareExchange,
SpvOpAtomicCompareExchangeWeak,
SpvOpAtomicIIncrement,
SpvOpAtomicIDecrement,
SpvOpAtomicIAdd,
SpvOpAtomicISub,
SpvOpAtomicSMin,
SpvOpAtomicUMin,
SpvOpAtomicSMax,
SpvOpAtomicUMax,
SpvOpAtomicAnd,
SpvOpAtomicOr,
SpvOpAtomicXor,
SpvOpPhi,
SpvOpLoopMerge,
SpvOpSelectionMerge,
SpvOpLabel,
SpvOpBranch,
SpvOpBranchConditional,
SpvOpSwitch,
SpvOpKill,
SpvOpReturn,
SpvOpReturnValue,
SpvOpUnreachable,
SpvOpLifetimeStart,
SpvOpLifetimeStop,
SpvOpGroupAsyncCopy,
SpvOpGroupWaitEvents,
SpvOpGroupAll,
SpvOpGroupAny,
SpvOpGroupBroadcast,
SpvOpGroupIAdd,
SpvOpGroupFAdd,
SpvOpGroupFMin,
SpvOpGroupUMin,
SpvOpGroupSMin,
SpvOpGroupFMax,
SpvOpGroupUMax,
SpvOpGroupSMax,
SpvOpReadPipe,
SpvOpWritePipe,
SpvOpReservedReadPipe,
SpvOpReservedWritePipe,
SpvOpReserveReadPipePackets,
SpvOpReserveWritePipePackets,
SpvOpCommitReadPipe,
SpvOpCommitWritePipe,
SpvOpIsValidReserveId,
SpvOpGetNumPipePackets,
SpvOpGetMaxPipePackets,
SpvOpGroupReserveReadPipePackets,
SpvOpGroupReserveWritePipePackets,
SpvOpGroupCommitReadPipe,
SpvOpGroupCommitWritePipe,
SpvOpEnqueueMarker,
SpvOpEnqueueKernel,
SpvOpGetKernelNDrangeSubGroupCount,
SpvOpGetKernelNDrangeMaxSubGroupSize,
SpvOpGetKernelWorkGroupSize,
SpvOpGetKernelPreferredWorkGroupSizeMultiple,
SpvOpRetainEvent,
SpvOpReleaseEvent,
SpvOpCreateUserEvent,
SpvOpIsValidEvent,
SpvOpSetUserEventStatus,
SpvOpCaptureEventProfilingInfo,
SpvOpGetDefaultQueue,
SpvOpBuildNDRange,
SpvOpImageSparseSampleImplicitLod,
SpvOpImageSparseSampleExplicitLod,
SpvOpImageSparseSampleDrefImplicitLod,
SpvOpImageSparseSampleDrefExplicitLod,
SpvOpImageSparseSampleProjImplicitLod,
SpvOpImageSparseSampleProjExplicitLod,
SpvOpImageSparseSampleProjDrefImplicitLod,
SpvOpImageSparseSampleProjDrefExplicitLod,
SpvOpImageSparseFetch,
SpvOpImageSparseGather,
SpvOpImageSparseDrefGather,
SpvOpImageSparseTexelsResident,
SpvOpNoLine,
SpvOpAtomicFlagTestAndSet,
SpvOpAtomicFlagClear,
SpvOpImageSparseRead,
SpvOpSizeOf,
SpvOpTypePipeStorage,
SpvOpConstantPipeStorage,
SpvOpCreatePipeFromPipeStorage,
SpvOpGetKernelLocalSizeForSubgroupCount,
SpvOpGetKernelMaxNumSubgroups,
SpvOpTypeNamedBarrier,
SpvOpNamedBarrierInitialize,
SpvOpMemoryNamedBarrier,
SpvOpModuleProcessed,
SpvOpExecutionModeId,
SpvOpDecorateId,
SpvOpSubgroupBallotKHR,
SpvOpSubgroupFirstInvocationKHR,
SpvOpSubgroupAllKHR,
SpvOpSubgroupAnyKHR,
SpvOpSubgroupAllEqualKHR,
SpvOpSubgroupReadInvocationKHR,
});
}
std::string GetVersionString(uint32_t word) {
std::stringstream ss;
ss << "Version " << SPV_SPIRV_VERSION_MAJOR_PART(word) << "."
<< SPV_SPIRV_VERSION_MINOR_PART(word);
return ss.str();
}
std::string GetGeneratorString(uint32_t word) {
return spvGeneratorStr(SPV_GENERATOR_TOOL_PART(word));
}
std::string GetOpcodeString(uint32_t word) {
return spvOpcodeString(static_cast<SpvOp>(word));
}
std::string GetCapabilityString(uint32_t word) {
return libspirv::CapabilityToString(static_cast<SpvCapability>(word));
}
template <class T>
std::string KeyIsLabel(T key) {
std::stringstream ss;
ss << key;
return ss.str();
}
template <class Key>
std::unordered_map<Key, double> GetRecall(
const std::unordered_map<Key, uint32_t>& hist, uint64_t total) {
std::unordered_map<Key, double> freq;
for (const auto& pair : hist) {
const double frequency =
static_cast<double>(pair.second) / static_cast<double>(total);
freq.emplace(pair.first, frequency);
}
return freq;
}
template <class Key>
std::unordered_map<Key, double> GetPrevalence(
const std::unordered_map<Key, uint32_t>& hist) {
uint64_t total = 0;
for (const auto& pair : hist) {
total += pair.second;
}
return GetRecall(hist, total);
}
// Writes |freq| to |out| sorted by frequency in the following format:
// LABEL3 70%
// LABEL1 20%
// LABEL2 10%
// |label_from_key| is used to convert |Key| to label.
template <class Key>
void WriteFreq(std::ostream& out, const std::unordered_map<Key, double>& freq,
std::string (*label_from_key)(Key), double threshold = 0.001) {
std::vector<std::pair<Key, double>> sorted_freq(freq.begin(), freq.end());
std::sort(sorted_freq.begin(), sorted_freq.end(),
[](const std::pair<Key, double>& left,
const std::pair<Key, double>& right) {
return left.second > right.second;
});
for (const auto& pair : sorted_freq) {
if (pair.second < threshold) break;
out << label_from_key(pair.first) << " " << pair.second * 100.0 << "%"
<< std::endl;
}
}
// Writes |hist| to |out| sorted by count in the following format:
// LABEL3 100
// LABEL1 50
// LABEL2 10
// |label_from_key| is used to convert |Key| to label.
template <class Key>
void WriteHist(std::ostream& out, const std::unordered_map<Key, uint32_t>& hist,
std::string (*label_from_key)(Key)) {
std::vector<std::pair<Key, uint32_t>> sorted_hist(hist.begin(), hist.end());
std::sort(sorted_hist.begin(), sorted_hist.end(),
[](const std::pair<Key, uint32_t>& left,
const std::pair<Key, uint32_t>& right) {
return left.second > right.second;
});
for (const auto& pair : sorted_hist) {
out << label_from_key(pair.first) << " " << pair.second << std::endl;
}
}
} // namespace
StatsAnalyzer::StatsAnalyzer(const SpirvStats& stats) : stats_(stats) {
num_modules_ = 0;
for (const auto& pair : stats_.version_hist) {
num_modules_ += pair.second;
}
version_freq_ = GetRecall(stats_.version_hist, num_modules_);
generator_freq_ = GetRecall(stats_.generator_hist, num_modules_);
capability_freq_ = GetRecall(stats_.capability_hist, num_modules_);
extension_freq_ = GetRecall(stats_.extension_hist, num_modules_);
opcode_freq_ = GetPrevalence(stats_.opcode_hist);
}
void StatsAnalyzer::WriteVersion(std::ostream& out) {
WriteFreq(out, version_freq_, GetVersionString);
}
void StatsAnalyzer::WriteGenerator(std::ostream& out) {
WriteFreq(out, generator_freq_, GetGeneratorString);
}
void StatsAnalyzer::WriteCapability(std::ostream& out) {
WriteFreq(out, capability_freq_, GetCapabilityString);
}
void StatsAnalyzer::WriteExtension(std::ostream& out) {
WriteFreq(out, extension_freq_, KeyIsLabel);
}
void StatsAnalyzer::WriteOpcode(std::ostream& out) {
out << "Total unique opcodes used: " << opcode_freq_.size() << std::endl;
WriteFreq(out, opcode_freq_, GetOpcodeString);
}
void StatsAnalyzer::WriteConstantLiterals(std::ostream& out) {
out << "Constant literals" << std::endl;
out << "Float 32" << std::endl;
WriteFreq(out, GetPrevalence(stats_.f32_constant_hist), KeyIsLabel);
out << std::endl << "Float 64" << std::endl;
WriteFreq(out, GetPrevalence(stats_.f64_constant_hist), KeyIsLabel);
out << std::endl << "Unsigned int 16" << std::endl;
WriteFreq(out, GetPrevalence(stats_.u16_constant_hist), KeyIsLabel);
out << std::endl << "Signed int 16" << std::endl;
WriteFreq(out, GetPrevalence(stats_.s16_constant_hist), KeyIsLabel);
out << std::endl << "Unsigned int 32" << std::endl;
WriteFreq(out, GetPrevalence(stats_.u32_constant_hist), KeyIsLabel);
out << std::endl << "Signed int 32" << std::endl;
WriteFreq(out, GetPrevalence(stats_.s32_constant_hist), KeyIsLabel);
out << std::endl << "Unsigned int 64" << std::endl;
WriteFreq(out, GetPrevalence(stats_.u64_constant_hist), KeyIsLabel);
out << std::endl << "Signed int 64" << std::endl;
WriteFreq(out, GetPrevalence(stats_.s64_constant_hist), KeyIsLabel);
}
void StatsAnalyzer::WriteOpcodeMarkov(std::ostream& out) {
if (stats_.opcode_markov_hist.empty()) return;
const std::unordered_map<uint32_t, std::unordered_map<uint32_t, uint32_t>>&
cue_to_hist = stats_.opcode_markov_hist[0];
// Sort by prevalence of the opcodes in opcode_freq_ (descending).
std::vector<std::pair<uint32_t, std::unordered_map<uint32_t, uint32_t>>>
sorted_cue_to_hist(cue_to_hist.begin(), cue_to_hist.end());
std::sort(
sorted_cue_to_hist.begin(), sorted_cue_to_hist.end(),
[this](const std::pair<uint32_t, std::unordered_map<uint32_t, uint32_t>>&
left,
const std::pair<uint32_t, std::unordered_map<uint32_t, uint32_t>>&
right) {
const double lf = opcode_freq_[left.first];
const double rf = opcode_freq_[right.first];
if (lf == rf) return right.first > left.first;
return lf > rf;
});
for (const auto& kv : sorted_cue_to_hist) {
const uint32_t cue = kv.first;
const double kFrequentEnoughToAnalyze = 0.0001;
if (opcode_freq_[cue] < kFrequentEnoughToAnalyze) continue;
const std::unordered_map<uint32_t, uint32_t>& hist = kv.second;
uint32_t total = 0;
for (const auto& pair : hist) {
total += pair.second;
}
std::vector<std::pair<uint32_t, uint32_t>> sorted_hist(hist.begin(),
hist.end());
std::sort(sorted_hist.begin(), sorted_hist.end(),
[](const std::pair<uint32_t, uint32_t>& left,
const std::pair<uint32_t, uint32_t>& right) {
if (left.second == right.second)
return right.first > left.first;
return left.second > right.second;
});
for (const auto& pair : sorted_hist) {
const double prior = opcode_freq_[pair.first];
const double posterior =
static_cast<double>(pair.second) / static_cast<double>(total);
out << GetOpcodeString(cue) << " -> " << GetOpcodeString(pair.first)
<< " " << posterior * 100 << "% (base rate " << prior * 100
<< "%, pair occurrences " << pair.second << ")" << std::endl;
}
}
}
void StatsAnalyzer::WriteCodegenOpcodeHist(std::ostream& out) {
auto all_opcodes = GetAllOpcodes();
// uint64_t is used because kMarkvNoneOfTheAbove is outside of uint32_t range.
out << "std::map<uint64_t, uint32_t> GetOpcodeHist() {\n"
<< " return std::map<uint64_t, uint32_t>({\n";
uint32_t total = 0;
for (const auto& kv : stats_.opcode_hist) {
total += kv.second;
}
for (uint32_t opcode : all_opcodes) {
const auto it = stats_.opcode_hist.find(opcode);
const uint32_t count = it == stats_.opcode_hist.end() ? 0 : it->second;
const double kMaxValue = 1000.0;
uint32_t value = uint32_t(kMaxValue * double(count) / double(total));
if (value == 0) value = 1;
out << " { SpvOp" << GetOpcodeString(opcode) << ", " << value << " },\n";
}
// Add kMarkvNoneOfTheAbove as a signal for unknown opcode.
out << " { kMarkvNoneOfTheAbove, " << 10 << " },\n";
out << " });\n}\n";
}
void StatsAnalyzer::WriteCodegenOpcodeAndNumOperandsHist(std::ostream& out) {
out << "std::map<uint64_t, uint32_t> GetOpcodeAndNumOperandsHist() {\n"
<< " return std::map<uint64_t, uint32_t>({\n";
uint32_t total = 0;
for (const auto& kv : stats_.opcode_and_num_operands_hist) {
total += kv.second;
}
uint32_t left_out = 0;
for (const auto& kv : stats_.opcode_and_num_operands_hist) {
const uint32_t count = kv.second;
const double kFrequentEnoughToAnalyze = 0.001;
const uint32_t opcode_and_num_operands = kv.first;
const uint32_t opcode = opcode_and_num_operands & 0xFFFF;
const uint32_t num_operands = opcode_and_num_operands >> 16;
if (opcode == SpvOpTypeStruct ||
double(count) / double(total) < kFrequentEnoughToAnalyze) {
left_out += count;
continue;
}
out << " { CombineOpcodeAndNumOperands(SpvOp"
<< spvOpcodeString(SpvOp(opcode)) << ", " << num_operands << "), "
<< count << " },\n";
}
// Heuristic.
const uint32_t none_of_the_above = std::max(1, int(left_out + total * 0.01));
out << " { kMarkvNoneOfTheAbove, " << none_of_the_above << " },\n";
out << " });\n}\n";
}
void StatsAnalyzer::WriteCodegenOpcodeAndNumOperandsMarkovHuffmanCodecs(
std::ostream& out) {
out << "std::map<uint32_t, std::unique_ptr<HuffmanCodec<uint64_t>>>\n"
<< "GetOpcodeAndNumOperandsMarkovHuffmanCodecs() {\n"
<< " std::map<uint32_t, std::unique_ptr<HuffmanCodec<uint64_t>>> "
<< "codecs;\n";
for (const auto& kv : stats_.opcode_and_num_operands_markov_hist) {
const uint32_t prev_opcode = kv.first;
const double kFrequentEnoughToAnalyze = 0.001;
if (opcode_freq_[prev_opcode] < kFrequentEnoughToAnalyze) continue;
const std::unordered_map<uint32_t, uint32_t>& hist = kv.second;
uint32_t total = 0;
for (const auto& pair : hist) {
total += pair.second;
}
uint32_t left_out = 0;
std::map<uint64_t, uint32_t> processed_hist;
for (const auto& pair : hist) {
const uint32_t opcode_and_num_operands = pair.first;
const uint32_t opcode = opcode_and_num_operands & 0xFFFF;
if (opcode == SpvOpTypeStruct) continue;
const uint32_t num_operands = opcode_and_num_operands >> 16;
const uint32_t count = pair.second;
const double posterior_freq = double(count) / double(total);
if (opcode_freq_[opcode] < kFrequentEnoughToAnalyze &&
posterior_freq < kFrequentEnoughToAnalyze) {
left_out += count;
continue;
}
processed_hist.emplace(CombineOpcodeAndNumOperands(opcode, num_operands),
count);
}
// Heuristic.
processed_hist.emplace(kMarkvNoneOfTheAbove,
std::max(1, int(left_out + total * 0.01)));
HuffmanCodec<uint64_t> codec(processed_hist);
out << " {\n";
out << " std::unique_ptr<HuffmanCodec<uint64_t>> "
<< "codec(new HuffmanCodec<uint64_t>";
out << codec.SerializeToText(4);
out << ");\n" << std::endl;
out << " codecs.emplace(SpvOp" << GetOpcodeString(prev_opcode)
<< ", std::move(codec));\n";
out << " }\n\n";
}
out << " return codecs;\n}\n";
}
void StatsAnalyzer::WriteCodegenLiteralStringHuffmanCodecs(std::ostream& out) {
out << "std::map<uint32_t, std::unique_ptr<HuffmanCodec<std::string>>>\n"
<< "GetLiteralStringHuffmanCodecs() {\n"
<< " std::map<uint32_t, std::unique_ptr<HuffmanCodec<std::string>>> "
<< "codecs;\n";
for (const auto& kv : stats_.literal_strings_hist) {
const uint32_t opcode = kv.first;
if (opcode == SpvOpName || opcode == SpvOpMemberName) continue;
const double kOpcodeFrequentEnoughToAnalyze = 0.001;
if (opcode_freq_[opcode] < kOpcodeFrequentEnoughToAnalyze) continue;
const std::unordered_map<std::string, uint32_t>& hist = kv.second;
uint32_t total = 0;
for (const auto& pair : hist) {
total += pair.second;
}
uint32_t left_out = 0;
std::map<std::string, uint32_t> processed_hist;
for (const auto& pair : hist) {
const uint32_t count = pair.second;
const double freq = double(count) / double(total);
const double kStringFrequentEnoughToAnalyze = 0.001;
if (freq < kStringFrequentEnoughToAnalyze) {
left_out += count;
continue;
}
processed_hist.emplace(pair.first, count);
}
// Heuristic.
processed_hist.emplace("kMarkvNoneOfTheAbove",
std::max(1, int(left_out + total * 0.01)));
HuffmanCodec<std::string> codec(processed_hist);
out << " {\n";
out << " std::unique_ptr<HuffmanCodec<std::string>> "
<< "codec(new HuffmanCodec<std::string>";
out << codec.SerializeToText(4);
out << ");\n" << std::endl;
out << " codecs.emplace(SpvOp" << spvOpcodeString(SpvOp(opcode))
<< ", std::move(codec));\n";
out << " }\n\n";
}
out << " return codecs;\n}\n";
}
void StatsAnalyzer::WriteCodegenNonIdWordHuffmanCodecs(std::ostream& out) {
out << "std::map<std::pair<uint32_t, uint32_t>, "
<< "std::unique_ptr<HuffmanCodec<uint64_t>>>\n"
<< "GetNonIdWordHuffmanCodecs() {\n"
<< " std::map<std::pair<uint32_t, uint32_t>, "
<< "std::unique_ptr<HuffmanCodec<uint64_t>>> codecs;\n";
for (const auto& kv : stats_.operand_slot_non_id_words_hist) {
const auto& opcode_and_index = kv.first;
const uint32_t opcode = opcode_and_index.first;
const uint32_t index = opcode_and_index.second;
const double kOpcodeFrequentEnoughToAnalyze = 0.001;
if (opcode_freq_[opcode] < kOpcodeFrequentEnoughToAnalyze) continue;
const std::map<uint32_t, uint32_t>& hist = kv.second;
uint32_t total = 0;
for (const auto& pair : hist) {
total += pair.second;
}
uint32_t left_out = 0;
std::map<uint64_t, uint32_t> processed_hist;
for (const auto& pair : hist) {
const uint32_t word = pair.first;
const uint32_t count = pair.second;
const double freq = double(count) / double(total);
const double kWordFrequentEnoughToAnalyze = 0.003;
if (freq < kWordFrequentEnoughToAnalyze) {
left_out += count;
continue;
}
processed_hist.emplace(word, count);
}
// Heuristic.
processed_hist.emplace(kMarkvNoneOfTheAbove,
std::max(1, int(left_out + total * 0.01)));
HuffmanCodec<uint64_t> codec(processed_hist);
out << " {\n";
out << " std::unique_ptr<HuffmanCodec<uint64_t>> "
<< "codec(new HuffmanCodec<uint64_t>";
out << codec.SerializeToText(4);
out << ");\n" << std::endl;
out << " codecs.emplace(std::pair<uint32_t, uint32_t>(SpvOp"
<< spvOpcodeString(SpvOp(opcode)) << ", " << index
<< "), std::move(codec));\n";
out << " }\n\n";
}
out << " return codecs;\n}\n";
}
void StatsAnalyzer::WriteCodegenIdDescriptorHuffmanCodecs(std::ostream& out) {
out << "std::map<std::pair<uint32_t, uint32_t>, "
<< "std::unique_ptr<HuffmanCodec<uint64_t>>>\n"
<< "GetIdDescriptorHuffmanCodecs() {\n"
<< " std::map<std::pair<uint32_t, uint32_t>, "
<< "std::unique_ptr<HuffmanCodec<uint64_t>>> codecs;\n";
std::unordered_set<uint32_t> descriptors_with_coding_scheme;
for (const auto& kv : stats_.operand_slot_id_descriptor_hist) {
const auto& opcode_and_index = kv.first;
const uint32_t opcode = opcode_and_index.first;
const uint32_t index = opcode_and_index.second;
const double kOpcodeFrequentEnoughToAnalyze = 0.003;
if (opcode_freq_[opcode] < kOpcodeFrequentEnoughToAnalyze) continue;
const std::map<uint32_t, uint32_t>& hist = kv.second;
uint32_t total = 0;
for (const auto& pair : hist) {
total += pair.second;
}
uint32_t left_out = 0;
std::map<uint64_t, uint32_t> processed_hist;
for (const auto& pair : hist) {
const uint32_t descriptor = pair.first;
const uint32_t count = pair.second;
const double freq = double(count) / double(total);
const double kDescriptorFrequentEnoughToAnalyze = 0.003;
if (freq < kDescriptorFrequentEnoughToAnalyze) {
left_out += count;
continue;
}
processed_hist.emplace(descriptor, count);
descriptors_with_coding_scheme.insert(descriptor);
}
// Heuristic.
processed_hist.emplace(kMarkvNoneOfTheAbove,
std::max(1, int(left_out + total * 0.01)));
HuffmanCodec<uint64_t> codec(processed_hist);
out << " {\n";
out << " std::unique_ptr<HuffmanCodec<uint64_t>> "
<< "codec(new HuffmanCodec<uint64_t>";
out << codec.SerializeToText(4);
out << ");\n" << std::endl;
out << " codecs.emplace(std::pair<uint32_t, uint32_t>(SpvOp"
<< spvOpcodeString(SpvOp(opcode)) << ", " << index
<< "), std::move(codec));\n";
out << " }\n\n";
}
out << " return codecs;\n}\n";
out << "\nstd::unordered_set<uint32_t> GetDescriptorsWithCodingScheme() {\n"
<< " std::unordered_set<uint32_t> descriptors_with_coding_scheme = {\n";
for (uint32_t descriptor : descriptors_with_coding_scheme) {
out << " " << descriptor << ",\n";
}
out << " };\n";
out << " return descriptors_with_coding_scheme;\n}\n";
}
Reference in New Issue View Git Blame Copy Permalink