AuroraMiddleware/SPIRV-Tools
1
0
Fork 0
mirror of https://github.com/KhronosGroup/SPIRV-Tools synced 2025-01-14 02:10:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove codegen related code. (#1819)

Browse Source 
This CL removes the code related to code generation from the stats
module.
This commit is contained in:
dan sinclair 2018-08-13 10:40:55 -04:00 committed by GitHub
parent b6319c3a43
commit 70de4a35aa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 2 additions and 781 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
tools/comp/markv_model_shader.cpp
View File

@ -36,23 +36,6 @@ inline uint32_t CombineOpcodeAndNumOperands(uint32_t opcode,
return opcode | (num_operands << 16);
}
// The following file contains autogenerated statistical coding rules.
// Can be generated by running spirv-stats on representative corpus of shaders
// with flags:
// --codegen_opcode_and_num_operands_hist
// --codegen_opcode_and_num_operands_markov_huffman_codecs
// --codegen_literal_string_huffman_codecs
// --codegen_non_id_word_huffman_codecs
// --codegen_id_descriptor_huffman_codecs
//
// Example:
// find <SHADER_CORPUS_DIR> -type f -print0 | xargs -0 -s 2000000
// ~/SPIRV-Tools/build/tools/spirv-stats -v
// --codegen_opcode_and_num_operands_hist
// --codegen_opcode_and_num_operands_markov_huffman_codecs
// --codegen_literal_string_huffman_codecs --codegen_non_id_word_huffman_codecs
// --codegen_id_descriptor_huffman_codecs -o
// ~/SPIRV-Tools/source/comp/markv_autogen.inc
#include "tools/comp/markv_model_shader_default_autogen.inc"
} // namespace

88
tools/stats/stats.cpp
View File

@ -50,35 +50,6 @@ Options:
-v, --verbose
Print additional info to stderr.
--codegen_opcode_hist
Output generated C++ code for opcode histogram.
This flag disables non-C++ output.
--codegen_opcode_and_num_operands_hist
Output generated C++ code for opcode_and_num_operands
histogram.
This flag disables non-C++ output.
--codegen_opcode_and_num_operands_markov_huffman_codecs
Output generated C++ code for Huffman codecs of
opcode_and_num_operands Markov chain.
This flag disables non-C++ output.
--codegen_literal_string_huffman_codecs
Output generated C++ code for Huffman codecs for
literal strings.
This flag disables non-C++ output.
--codegen_non_id_word_huffman_codecs
Output generated C++ code for Huffman codecs for
single-word non-id slots.
This flag disables non-C++ output.
--codegen_id_descriptor_huffman_codecs
Output generated C++ code for Huffman codecs for
common id descriptors.
This flag disables non-C++ output.
)",
argv0, argv0, argv0);
}
@ -113,12 +84,6 @@ int main(int argc, char** argv) {
bool expect_output_path = false;
bool verbose = false;
bool export_text = true;
bool codegen_opcode_hist = false;
bool codegen_opcode_and_num_operands_hist = false;
bool codegen_opcode_and_num_operands_markov_huffman_codecs = false;
bool codegen_literal_string_huffman_codecs = false;
bool codegen_non_id_word_huffman_codecs = false;
bool codegen_id_descriptor_huffman_codecs = false;
std::vector<const char*> paths;
const char* output_path = nullptr;
@ -130,29 +95,6 @@ int main(int argc, char** argv) {
PrintUsage(argv[0]);
continue_processing = false;
return_code = 0;
} else if (0 == strcmp(cur_arg, "--codegen_opcode_hist")) {
codegen_opcode_hist = true;
export_text = false;
} else if (0 ==
strcmp(cur_arg, "--codegen_opcode_and_num_operands_hist")) {
codegen_opcode_and_num_operands_hist = true;
export_text = false;
} else if (strcmp(
"--codegen_opcode_and_num_operands_markov_huffman_codecs",
cur_arg) == 0) {
codegen_opcode_and_num_operands_markov_huffman_codecs = true;
export_text = false;
} else if (0 ==
strcmp(cur_arg, "--codegen_literal_string_huffman_codecs")) {
codegen_literal_string_huffman_codecs = true;
export_text = false;
} else if (0 == strcmp(cur_arg, "--codegen_non_id_word_huffman_codecs")) {
codegen_non_id_word_huffman_codecs = true;
export_text = false;
} else if (0 ==
strcmp(cur_arg, "--codegen_id_descriptor_huffman_codecs")) {
codegen_id_descriptor_huffman_codecs = true;
export_text = false;
} else if (0 == strcmp(cur_arg, "--verbose") ||
0 == strcmp(cur_arg, "-v")) {
verbose = true;
@ -240,35 +182,5 @@ int main(int argc, char** argv) {
analyzer.WriteConstantLiterals(out);
}
if (codegen_opcode_hist) {
out << std::endl;
analyzer.WriteCodegenOpcodeHist(out);
}
if (codegen_opcode_and_num_operands_hist) {
out << std::endl;
analyzer.WriteCodegenOpcodeAndNumOperandsHist(out);
}
if (codegen_opcode_and_num_operands_markov_huffman_codecs) {
out << std::endl;
analyzer.WriteCodegenOpcodeAndNumOperandsMarkovHuffmanCodecs(out);
}
if (codegen_literal_string_huffman_codecs) {
out << std::endl;
analyzer.WriteCodegenLiteralStringHuffmanCodecs(out);
}
if (codegen_non_id_word_huffman_codecs) {
out << std::endl;
analyzer.WriteCodegenNonIdWordHuffmanCodecs(out);
}
if (codegen_id_descriptor_huffman_codecs) {
out << std::endl;
analyzer.WriteCodegenIdDescriptorHuffmanCodecs(out);
}
return 0;
}

649
tools/stats/stats_analyzer.cpp
View File

@ -32,341 +32,16 @@
#include "source/opcode.h"
#include "source/operand.h"
#include "source/spirv_constant.h"
#include "source/util/huffman_codec.h"
namespace {
using spvtools::SpirvStats;
using spvtools::utils::HuffmanCodec;
// Signals that the value is not in the coding scheme and a fallback method
// needs to be used.
const uint64_t kMarkvNoneOfTheAbove =
spvtools::comp::MarkvModel::GetMarkvNoneOfTheAbove();
inline uint32_t CombineOpcodeAndNumOperands(uint32_t opcode,
uint32_t num_operands) {
return opcode | (num_operands << 16);
}
// Returns all SPIR-V v1.2 opcodes.
std::vector<uint32_t> GetAllOpcodes() {
return std::vector<uint32_t>({
SpvOpNop,
SpvOpUndef,
SpvOpSourceContinued,
SpvOpSource,
SpvOpSourceExtension,
SpvOpName,
SpvOpMemberName,
SpvOpString,
SpvOpLine,
SpvOpExtension,
SpvOpExtInstImport,
SpvOpExtInst,
SpvOpMemoryModel,
SpvOpEntryPoint,
SpvOpExecutionMode,
SpvOpCapability,
SpvOpTypeVoid,
SpvOpTypeBool,
SpvOpTypeInt,
SpvOpTypeFloat,
SpvOpTypeVector,
SpvOpTypeMatrix,
SpvOpTypeImage,
SpvOpTypeSampler,
SpvOpTypeSampledImage,
SpvOpTypeArray,
SpvOpTypeRuntimeArray,
SpvOpTypeStruct,
SpvOpTypeOpaque,
SpvOpTypePointer,
SpvOpTypeFunction,
SpvOpTypeEvent,
SpvOpTypeDeviceEvent,
SpvOpTypeReserveId,
SpvOpTypeQueue,
SpvOpTypePipe,
SpvOpTypeForwardPointer,
SpvOpConstantTrue,
SpvOpConstantFalse,
SpvOpConstant,
SpvOpConstantComposite,
SpvOpConstantSampler,
SpvOpConstantNull,
SpvOpSpecConstantTrue,
SpvOpSpecConstantFalse,
SpvOpSpecConstant,
SpvOpSpecConstantComposite,
SpvOpSpecConstantOp,
SpvOpFunction,
SpvOpFunctionParameter,
SpvOpFunctionEnd,
SpvOpFunctionCall,
SpvOpVariable,
SpvOpImageTexelPointer,
SpvOpLoad,
SpvOpStore,
SpvOpCopyMemory,
SpvOpCopyMemorySized,
SpvOpAccessChain,
SpvOpInBoundsAccessChain,
SpvOpPtrAccessChain,
SpvOpArrayLength,
SpvOpGenericPtrMemSemantics,
SpvOpInBoundsPtrAccessChain,
SpvOpDecorate,
SpvOpMemberDecorate,
SpvOpDecorationGroup,
SpvOpGroupDecorate,
SpvOpGroupMemberDecorate,
SpvOpVectorExtractDynamic,
SpvOpVectorInsertDynamic,
SpvOpVectorShuffle,
SpvOpCompositeConstruct,
SpvOpCompositeExtract,
SpvOpCompositeInsert,
SpvOpCopyObject,
SpvOpTranspose,
SpvOpSampledImage,
SpvOpImageSampleImplicitLod,
SpvOpImageSampleExplicitLod,
SpvOpImageSampleDrefImplicitLod,
SpvOpImageSampleDrefExplicitLod,
SpvOpImageSampleProjImplicitLod,
SpvOpImageSampleProjExplicitLod,
SpvOpImageSampleProjDrefImplicitLod,
SpvOpImageSampleProjDrefExplicitLod,
SpvOpImageFetch,
SpvOpImageGather,
SpvOpImageDrefGather,
SpvOpImageRead,
SpvOpImageWrite,
SpvOpImage,
SpvOpImageQueryFormat,
SpvOpImageQueryOrder,
SpvOpImageQuerySizeLod,
SpvOpImageQuerySize,
SpvOpImageQueryLod,
SpvOpImageQueryLevels,
SpvOpImageQuerySamples,
SpvOpConvertFToU,
SpvOpConvertFToS,
SpvOpConvertSToF,
SpvOpConvertUToF,
SpvOpUConvert,
SpvOpSConvert,
SpvOpFConvert,
SpvOpQuantizeToF16,
SpvOpConvertPtrToU,
SpvOpSatConvertSToU,
SpvOpSatConvertUToS,
SpvOpConvertUToPtr,
SpvOpPtrCastToGeneric,
SpvOpGenericCastToPtr,
SpvOpGenericCastToPtrExplicit,
SpvOpBitcast,
SpvOpSNegate,
SpvOpFNegate,
SpvOpIAdd,
SpvOpFAdd,
SpvOpISub,
SpvOpFSub,
SpvOpIMul,
SpvOpFMul,
SpvOpUDiv,
SpvOpSDiv,
SpvOpFDiv,
SpvOpUMod,
SpvOpSRem,
SpvOpSMod,
SpvOpFRem,
SpvOpFMod,
SpvOpVectorTimesScalar,
SpvOpMatrixTimesScalar,
SpvOpVectorTimesMatrix,
SpvOpMatrixTimesVector,
SpvOpMatrixTimesMatrix,
SpvOpOuterProduct,
SpvOpDot,
SpvOpIAddCarry,
SpvOpISubBorrow,
SpvOpUMulExtended,
SpvOpSMulExtended,
SpvOpAny,
SpvOpAll,
SpvOpIsNan,
SpvOpIsInf,
SpvOpIsFinite,
SpvOpIsNormal,
SpvOpSignBitSet,
SpvOpLessOrGreater,
SpvOpOrdered,
SpvOpUnordered,
SpvOpLogicalEqual,
SpvOpLogicalNotEqual,
SpvOpLogicalOr,
SpvOpLogicalAnd,
SpvOpLogicalNot,
SpvOpSelect,
SpvOpIEqual,
SpvOpINotEqual,
SpvOpUGreaterThan,
SpvOpSGreaterThan,
SpvOpUGreaterThanEqual,
SpvOpSGreaterThanEqual,
SpvOpULessThan,
SpvOpSLessThan,
SpvOpULessThanEqual,
SpvOpSLessThanEqual,
SpvOpFOrdEqual,
SpvOpFUnordEqual,
SpvOpFOrdNotEqual,
SpvOpFUnordNotEqual,
SpvOpFOrdLessThan,
SpvOpFUnordLessThan,
SpvOpFOrdGreaterThan,
SpvOpFUnordGreaterThan,
SpvOpFOrdLessThanEqual,
SpvOpFUnordLessThanEqual,
SpvOpFOrdGreaterThanEqual,
SpvOpFUnordGreaterThanEqual,
SpvOpShiftRightLogical,
SpvOpShiftRightArithmetic,
SpvOpShiftLeftLogical,
SpvOpBitwiseOr,
SpvOpBitwiseXor,
SpvOpBitwiseAnd,
SpvOpNot,
SpvOpBitFieldInsert,
SpvOpBitFieldSExtract,
SpvOpBitFieldUExtract,
SpvOpBitReverse,
SpvOpBitCount,
SpvOpDPdx,
SpvOpDPdy,
SpvOpFwidth,
SpvOpDPdxFine,
SpvOpDPdyFine,
SpvOpFwidthFine,
SpvOpDPdxCoarse,
SpvOpDPdyCoarse,
SpvOpFwidthCoarse,
SpvOpEmitVertex,
SpvOpEndPrimitive,
SpvOpEmitStreamVertex,
SpvOpEndStreamPrimitive,
SpvOpControlBarrier,
SpvOpMemoryBarrier,
SpvOpAtomicLoad,
SpvOpAtomicStore,
SpvOpAtomicExchange,
SpvOpAtomicCompareExchange,
SpvOpAtomicCompareExchangeWeak,
SpvOpAtomicIIncrement,
SpvOpAtomicIDecrement,
SpvOpAtomicIAdd,
SpvOpAtomicISub,
SpvOpAtomicSMin,
SpvOpAtomicUMin,
SpvOpAtomicSMax,
SpvOpAtomicUMax,
SpvOpAtomicAnd,
SpvOpAtomicOr,
SpvOpAtomicXor,
SpvOpPhi,
SpvOpLoopMerge,
SpvOpSelectionMerge,
SpvOpLabel,
SpvOpBranch,
SpvOpBranchConditional,
SpvOpSwitch,
SpvOpKill,
SpvOpReturn,
SpvOpReturnValue,
SpvOpUnreachable,
SpvOpLifetimeStart,
SpvOpLifetimeStop,
SpvOpGroupAsyncCopy,
SpvOpGroupWaitEvents,
SpvOpGroupAll,
SpvOpGroupAny,
SpvOpGroupBroadcast,
SpvOpGroupIAdd,
SpvOpGroupFAdd,
SpvOpGroupFMin,
SpvOpGroupUMin,
SpvOpGroupSMin,
SpvOpGroupFMax,
SpvOpGroupUMax,
SpvOpGroupSMax,
SpvOpReadPipe,
SpvOpWritePipe,
SpvOpReservedReadPipe,
SpvOpReservedWritePipe,
SpvOpReserveReadPipePackets,
SpvOpReserveWritePipePackets,
SpvOpCommitReadPipe,
SpvOpCommitWritePipe,
SpvOpIsValidReserveId,
SpvOpGetNumPipePackets,
SpvOpGetMaxPipePackets,
SpvOpGroupReserveReadPipePackets,
SpvOpGroupReserveWritePipePackets,
SpvOpGroupCommitReadPipe,
SpvOpGroupCommitWritePipe,
SpvOpEnqueueMarker,
SpvOpEnqueueKernel,
SpvOpGetKernelNDrangeSubGroupCount,
SpvOpGetKernelNDrangeMaxSubGroupSize,
SpvOpGetKernelWorkGroupSize,
SpvOpGetKernelPreferredWorkGroupSizeMultiple,
SpvOpRetainEvent,
SpvOpReleaseEvent,
SpvOpCreateUserEvent,
SpvOpIsValidEvent,
SpvOpSetUserEventStatus,
SpvOpCaptureEventProfilingInfo,
SpvOpGetDefaultQueue,
SpvOpBuildNDRange,
SpvOpImageSparseSampleImplicitLod,
SpvOpImageSparseSampleExplicitLod,
SpvOpImageSparseSampleDrefImplicitLod,
SpvOpImageSparseSampleDrefExplicitLod,
SpvOpImageSparseSampleProjImplicitLod,
SpvOpImageSparseSampleProjExplicitLod,
SpvOpImageSparseSampleProjDrefImplicitLod,
SpvOpImageSparseSampleProjDrefExplicitLod,
SpvOpImageSparseFetch,
SpvOpImageSparseGather,
SpvOpImageSparseDrefGather,
SpvOpImageSparseTexelsResident,
SpvOpNoLine,
SpvOpAtomicFlagTestAndSet,
SpvOpAtomicFlagClear,
SpvOpImageSparseRead,
SpvOpSizeOf,
SpvOpTypePipeStorage,
SpvOpConstantPipeStorage,
SpvOpCreatePipeFromPipeStorage,
SpvOpGetKernelLocalSizeForSubgroupCount,
SpvOpGetKernelMaxNumSubgroups,
SpvOpTypeNamedBarrier,
SpvOpNamedBarrierInitialize,
SpvOpMemoryNamedBarrier,
SpvOpModuleProcessed,
SpvOpExecutionModeId,
SpvOpDecorateId,
SpvOpSubgroupBallotKHR,
SpvOpSubgroupFirstInvocationKHR,
SpvOpSubgroupAllKHR,
SpvOpSubgroupAnyKHR,
SpvOpSubgroupAllEqualKHR,
SpvOpSubgroupReadInvocationKHR,
});
}
std::string GetVersionString(uint32_t word) {
std::stringstream ss;
ss << "Version " << SPV_SPIRV_VERSION_MAJOR_PART(word) << "."
@ -423,7 +98,7 @@ std::unordered_map<Key, double> GetPrevalence(
// |label_from_key| is used to convert |Key| to label.
template <class Key>
void WriteFreq(std::ostream& out, const std::unordered_map<Key, double>& freq,
std::string (*label_from_key)(Key), double threshold = 0.001) {
std::string (*label_from_key)(Key)) {
std::vector<std::pair<Key, double>> sorted_freq(freq.begin(), freq.end());
std::sort(sorted_freq.begin(), sorted_freq.end(),
[](const std::pair<Key, double>& left,
@ -432,32 +107,12 @@ void WriteFreq(std::ostream& out, const std::unordered_map<Key, double>& freq,
});
for (const auto& pair : sorted_freq) {
if (pair.second < threshold) break;
if (pair.second < 0.001) break;
out << label_from_key(pair.first) << " " << pair.second * 100.0 << "%"
<< std::endl;
}
}
// Writes |hist| to |out| sorted by count in the following format:
// LABEL3 100
// LABEL1 50
// LABEL2 10
// |label_from_key| is used to convert |Key| to label.
template <class Key>
void WriteHist(std::ostream& out, const std::unordered_map<Key, uint32_t>& hist,
std::string (*label_from_key)(Key)) {
std::vector<std::pair<Key, uint32_t>> sorted_hist(hist.begin(), hist.end());
std::sort(sorted_hist.begin(), sorted_hist.end(),
[](const std::pair<Key, uint32_t>& left,
const std::pair<Key, uint32_t>& right) {
return left.second > right.second;
});
for (const auto& pair : sorted_hist) {
out << label_from_key(pair.first) << " " << pair.second << std::endl;
}
}
} // namespace
StatsAnalyzer::StatsAnalyzer(const SpirvStats& stats) : stats_(stats) {
@ -575,303 +230,3 @@ void StatsAnalyzer::WriteOpcodeMarkov(std::ostream& out) {
}
}
}
void StatsAnalyzer::WriteCodegenOpcodeHist(std::ostream& out) {
auto all_opcodes = GetAllOpcodes();
// uint64_t is used because kMarkvNoneOfTheAbove is outside of uint32_t range.
out << "std::map<uint64_t, uint32_t> GetOpcodeHist() {\n"
<< " return std::map<uint64_t, uint32_t>({\n";
uint32_t total = 0;
for (const auto& kv : stats_.opcode_hist) {
total += kv.second;
}
for (uint32_t opcode : all_opcodes) {
const auto it = stats_.opcode_hist.find(opcode);
const uint32_t count = it == stats_.opcode_hist.end() ? 0 : it->second;
const double kMaxValue = 1000.0;
uint32_t value = uint32_t(kMaxValue * double(count) / double(total));
if (value == 0) value = 1;
out << " { SpvOp" << GetOpcodeString(opcode) << ", " << value << " },\n";
}
// Add kMarkvNoneOfTheAbove as a signal for unknown opcode.
out << " { kMarkvNoneOfTheAbove, " << 10 << " },\n";
out << " });\n}\n";
}
void StatsAnalyzer::WriteCodegenOpcodeAndNumOperandsHist(std::ostream& out) {
out << "std::map<uint64_t, uint32_t> GetOpcodeAndNumOperandsHist() {\n"
<< " return std::map<uint64_t, uint32_t>({\n";
uint32_t total = 0;
for (const auto& kv : stats_.opcode_and_num_operands_hist) {
total += kv.second;
}
uint32_t left_out = 0;
for (const auto& kv : stats_.opcode_and_num_operands_hist) {
const uint32_t count = kv.second;
const double kFrequentEnoughToAnalyze = 0.001;
const uint32_t opcode_and_num_operands = kv.first;
const uint32_t opcode = opcode_and_num_operands & 0xFFFF;
const uint32_t num_operands = opcode_and_num_operands >> 16;
if (opcode == SpvOpTypeStruct ||
double(count) / double(total) < kFrequentEnoughToAnalyze) {
left_out += count;
continue;
}
out << " { CombineOpcodeAndNumOperands(SpvOp"
<< spvOpcodeString(SpvOp(opcode)) << ", " << num_operands << "), "
<< count << " },\n";
}
// Heuristic.
const uint32_t none_of_the_above = std::max(1, int(left_out + total * 0.01));
out << " { kMarkvNoneOfTheAbove, " << none_of_the_above << " },\n";
out << " });\n}\n";
}
void StatsAnalyzer::WriteCodegenOpcodeAndNumOperandsMarkovHuffmanCodecs(
std::ostream& out) {
out << "std::map<uint32_t, std::unique_ptr<HuffmanCodec<uint64_t>>>\n"
<< "GetOpcodeAndNumOperandsMarkovHuffmanCodecs() {\n"
<< " std::map<uint32_t, std::unique_ptr<HuffmanCodec<uint64_t>>> "
<< "codecs;\n";
for (const auto& kv : stats_.opcode_and_num_operands_markov_hist) {
const uint32_t prev_opcode = kv.first;
const double kFrequentEnoughToAnalyze = 0.001;
if (opcode_freq_[prev_opcode] < kFrequentEnoughToAnalyze) continue;
const std::unordered_map<uint32_t, uint32_t>& hist = kv.second;
uint32_t total = 0;
for (const auto& pair : hist) {
total += pair.second;
}
uint32_t left_out = 0;
std::map<uint64_t, uint32_t> processed_hist;
for (const auto& pair : hist) {
const uint32_t opcode_and_num_operands = pair.first;
const uint32_t opcode = opcode_and_num_operands & 0xFFFF;
if (opcode == SpvOpTypeStruct) continue;
const uint32_t num_operands = opcode_and_num_operands >> 16;
const uint32_t count = pair.second;
const double posterior_freq = double(count) / double(total);
if (opcode_freq_[opcode] < kFrequentEnoughToAnalyze &&
posterior_freq < kFrequentEnoughToAnalyze) {
left_out += count;
continue;
}
processed_hist.emplace(CombineOpcodeAndNumOperands(opcode, num_operands),
count);
}
// Heuristic.
processed_hist.emplace(kMarkvNoneOfTheAbove,
std::max(1, int(left_out + total * 0.01)));
HuffmanCodec<uint64_t> codec(processed_hist);
out << " {\n";
out << " std::unique_ptr<HuffmanCodec<uint64_t>> "
<< "codec(new HuffmanCodec<uint64_t>";
out << codec.SerializeToText(4);
out << ");\n" << std::endl;
out << " codecs.emplace(SpvOp" << GetOpcodeString(prev_opcode)
<< ", std::move(codec));\n";
out << " }\n\n";
}
out << " return codecs;\n}\n";
}
void StatsAnalyzer::WriteCodegenLiteralStringHuffmanCodecs(std::ostream& out) {
out << "std::map<uint32_t, std::unique_ptr<HuffmanCodec<std::string>>>\n"
<< "GetLiteralStringHuffmanCodecs() {\n"
<< " std::map<uint32_t, std::unique_ptr<HuffmanCodec<std::string>>> "
<< "codecs;\n";
for (const auto& kv : stats_.literal_strings_hist) {
const uint32_t opcode = kv.first;
if (opcode == SpvOpName || opcode == SpvOpMemberName) continue;
const double kOpcodeFrequentEnoughToAnalyze = 0.001;
if (opcode_freq_[opcode] < kOpcodeFrequentEnoughToAnalyze) continue;
const std::unordered_map<std::string, uint32_t>& hist = kv.second;
uint32_t total = 0;
for (const auto& pair : hist) {
total += pair.second;
}
uint32_t left_out = 0;
std::map<std::string, uint32_t> processed_hist;
for (const auto& pair : hist) {
const uint32_t count = pair.second;
const double freq = double(count) / double(total);
const double kStringFrequentEnoughToAnalyze = 0.001;
if (freq < kStringFrequentEnoughToAnalyze) {
left_out += count;
continue;
}
processed_hist.emplace(pair.first, count);
}
// Heuristic.
processed_hist.emplace("kMarkvNoneOfTheAbove",
std::max(1, int(left_out + total * 0.01)));
HuffmanCodec<std::string> codec(processed_hist);
out << " {\n";
out << " std::unique_ptr<HuffmanCodec<std::string>> "
<< "codec(new HuffmanCodec<std::string>";
out << codec.SerializeToText(4);
out << ");\n" << std::endl;
out << " codecs.emplace(SpvOp" << spvOpcodeString(SpvOp(opcode))
<< ", std::move(codec));\n";
out << " }\n\n";
}
out << " return codecs;\n}\n";
}
void StatsAnalyzer::WriteCodegenNonIdWordHuffmanCodecs(std::ostream& out) {
out << "std::map<std::pair<uint32_t, uint32_t>, "
<< "std::unique_ptr<HuffmanCodec<uint64_t>>>\n"
<< "GetNonIdWordHuffmanCodecs() {\n"
<< " std::map<std::pair<uint32_t, uint32_t>, "
<< "std::unique_ptr<HuffmanCodec<uint64_t>>> codecs;\n";
for (const auto& kv : stats_.operand_slot_non_id_words_hist) {
const auto& opcode_and_index = kv.first;
const uint32_t opcode = opcode_and_index.first;
const uint32_t index = opcode_and_index.second;
const double kOpcodeFrequentEnoughToAnalyze = 0.001;
if (opcode_freq_[opcode] < kOpcodeFrequentEnoughToAnalyze) continue;
const std::map<uint32_t, uint32_t>& hist = kv.second;
uint32_t total = 0;
for (const auto& pair : hist) {
total += pair.second;
}
uint32_t left_out = 0;
std::map<uint64_t, uint32_t> processed_hist;
for (const auto& pair : hist) {
const uint32_t word = pair.first;
const uint32_t count = pair.second;
const double freq = double(count) / double(total);
const double kWordFrequentEnoughToAnalyze = 0.003;
if (freq < kWordFrequentEnoughToAnalyze) {
left_out += count;
continue;
}
processed_hist.emplace(word, count);
}
// Heuristic.
processed_hist.emplace(kMarkvNoneOfTheAbove,
std::max(1, int(left_out + total * 0.01)));
HuffmanCodec<uint64_t> codec(processed_hist);
out << " {\n";
out << " std::unique_ptr<HuffmanCodec<uint64_t>> "
<< "codec(new HuffmanCodec<uint64_t>";
out << codec.SerializeToText(4);
out << ");\n" << std::endl;
out << " codecs.emplace(std::pair<uint32_t, uint32_t>(SpvOp"
<< spvOpcodeString(SpvOp(opcode)) << ", " << index
<< "), std::move(codec));\n";
out << " }\n\n";
}
out << " return codecs;\n}\n";
}
void StatsAnalyzer::WriteCodegenIdDescriptorHuffmanCodecs(std::ostream& out) {
out << "std::map<std::pair<uint32_t, uint32_t>, "
<< "std::unique_ptr<HuffmanCodec<uint64_t>>>\n"
<< "GetIdDescriptorHuffmanCodecs() {\n"
<< " std::map<std::pair<uint32_t, uint32_t>, "
<< "std::unique_ptr<HuffmanCodec<uint64_t>>> codecs;\n";
std::unordered_set<uint32_t> descriptors_with_coding_scheme;
for (const auto& kv : stats_.operand_slot_id_descriptor_hist) {
const auto& opcode_and_index = kv.first;
const uint32_t opcode = opcode_and_index.first;
const uint32_t index = opcode_and_index.second;
const double kOpcodeFrequentEnoughToAnalyze = 0.003;
if (opcode_freq_[opcode] < kOpcodeFrequentEnoughToAnalyze) continue;
const std::map<uint32_t, uint32_t>& hist = kv.second;
uint32_t total = 0;
for (const auto& pair : hist) {
total += pair.second;
}
uint32_t left_out = 0;
std::map<uint64_t, uint32_t> processed_hist;
for (const auto& pair : hist) {
const uint32_t descriptor = pair.first;
const uint32_t count = pair.second;
const double freq = double(count) / double(total);
const double kDescriptorFrequentEnoughToAnalyze = 0.003;
if (freq < kDescriptorFrequentEnoughToAnalyze) {
left_out += count;
continue;
}
processed_hist.emplace(descriptor, count);
descriptors_with_coding_scheme.insert(descriptor);
}
// Heuristic.
processed_hist.emplace(kMarkvNoneOfTheAbove,
std::max(1, int(left_out + total * 0.01)));
HuffmanCodec<uint64_t> codec(processed_hist);
out << " {\n";
out << " std::unique_ptr<HuffmanCodec<uint64_t>> "
<< "codec(new HuffmanCodec<uint64_t>";
out << codec.SerializeToText(4);
out << ");\n" << std::endl;
out << " codecs.emplace(std::pair<uint32_t, uint32_t>(SpvOp"
<< spvOpcodeString(SpvOp(opcode)) << ", " << index
<< "), std::move(codec));\n";
out << " }\n\n";
}
out << " return codecs;\n}\n";
out << "\nstd::unordered_set<uint32_t> GetDescriptorsWithCodingScheme() {\n"
<< " std::unordered_set<uint32_t> descriptors_with_coding_scheme = {\n";
for (uint32_t descriptor : descriptors_with_coding_scheme) {
out << " " << descriptor << ",\n";
}
out << " };\n";
out << " return descriptors_with_coding_scheme;\n}\n";
}

29
tools/stats/stats_analyzer.h
View File

@ -37,35 +37,6 @@ class StatsAnalyzer {
// level.
void WriteOpcodeMarkov(std::ostream& out);
// Writes C++ code containing a function returning opcode histogram.
void WriteCodegenOpcodeHist(std::ostream& out);
// Writes C++ code containing a function returning opcode_and_num_operands
// histogram.
void WriteCodegenOpcodeAndNumOperandsHist(std::ostream& out);
// Writes C++ code containing a function returning a map of Huffman codecs
// for opcode_and_num_operands. Each Huffman codec is created for a specific
// previous opcode.
// TODO(atgoo@github.com) Write code which would contain pregenerated Huffman
// codecs, instead of code which would generate them every time.
void WriteCodegenOpcodeAndNumOperandsMarkovHuffmanCodecs(std::ostream& out);
// Writes C++ code containing a function returning a map of Huffman codecs
// for literal strings. Each Huffman codec is created for a specific opcode.
// I.e. OpExtension and OpExtInstImport would use different codecs.
void WriteCodegenLiteralStringHuffmanCodecs(std::ostream& out);
// Writes C++ code containing a function returning a map of Huffman codecs
// for single-word non-id operands. Each Huffman codec is created for a
// specific operand slot (opcode and operand number).
void WriteCodegenNonIdWordHuffmanCodecs(std::ostream& out);
// Writes C++ code containing a function returning a map of Huffman codecs
// for common id descriptors. Each Huffman codec is created for a
// specific operand slot (opcode and operand number).
void WriteCodegenIdDescriptorHuffmanCodecs(std::ostream& out);
private:
const spvtools::SpirvStats& stats_;