AuroraMiddleware/SPIRV-Tools
1
0
Fork 0
mirror of https://github.com/KhronosGroup/SPIRV-Tools synced 2024-11-22 11:40:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add new compression algorithm and models

Browse Source 
Add new "short descriptor" algorithm to MARK-V codec.

Add three shader compression models:
lite - fast, poor compression
mid - balanced
max - best compression
This commit is contained in:
Andrey Tuganov 2017-11-14 17:02:42 -05:00 committed by David Neto
parent a771713e42
commit 250a235a8d
14 changed files with 676 additions and 431 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

231
source/comp/markv_codec.cpp
View File

@ -60,10 +60,10 @@
#include "val/validation_state.h"
#include "validate.h"
using libspirv::DiagnosticStream;
using libspirv::IdDescriptorCollection;
using libspirv::Instruction;
using libspirv::ValidationState_t;
using libspirv::DiagnosticStream;
using spvutils::BitReaderWord64;
using spvutils::BitWriterWord64;
using spvutils::HuffmanCodec;
@ -131,10 +131,10 @@ enum : uint64_t {
kMtfFunctionTypeWithReturnTypeBegin = 0x70000,
// All function objects which return specific type.
kMtfFunctionWithReturnTypeBegin = 0x80000,
// All float vectors of specific size.
kMtfFloatVectorOfSizeBegin = 0x90000,
// Id descriptor space (32-bit).
kMtfIdDescriptorSpaceBegin = 0x100000000,
// Short id descriptor space (max 16-bit).
kMtfShortIdDescriptorSpaceBegin = 0x90000,
// Long id descriptor space (32-bit).
kMtfLongIdDescriptorSpaceBegin = 0x100000000,
};
// Signals that the value is not in the coding scheme and a fallback method
@ -152,6 +152,20 @@ const size_t kCommentNumWhitespaces = 2;
const size_t kByteBreakAfterInstIfLessThanUntilNextByte = 8;
const uint32_t kShortDescriptorNumBits = 8;
// Custom hash function used to produce short descriptors.
uint32_t ShortHashU32Array(const std::vector<uint32_t>& words) {
// The hash function is a sum of hashes of each word seeded by word index.
// Knuth's multiplicative hash is used to hash the words.
const uint32_t kKnuthMulHash = 2654435761;
uint32_t val = 0;
for (uint32_t i = 0; i < words.size(); ++i) {
val += (words[i] + i + 123) * kKnuthMulHash;
}
return 1 + val % ((1 << kShortDescriptorNumBits) - 1);
}
// Returns a set of mtf rank codecs based on a plausible hand-coded
// distribution.
std::map<uint64_t, std::unique_ptr<HuffmanCodec<uint32_t>>>
@ -263,7 +277,7 @@ size_t GetNumBitsToNextByte(size_t bit_pos) { return (8 - (bit_pos % 8)) % 8; }
// Defines and returns current MARK-V version.
uint32_t GetMarkvVersion() {
const uint32_t kVersionMajor = 1;
const uint32_t kVersionMinor = 3;
const uint32_t kVersionMinor = 4;
return kVersionMinor | (kVersionMajor << 16);
}
@ -372,6 +386,7 @@ class MarkvCodecBase {
: validator_options_(validator_options),
grammar_(context),
model_(model),
short_id_descriptors_(ShortHashU32Array),
mtf_huffman_codecs_(GetMtfHuffmanCodecs()),
context_(context),
vstate_(validator_options
@ -425,11 +440,6 @@ class MarkvCodecBase {
return kMtfVectorOfComponentTypeBegin + type_id;
}
// Returns mtf handle for float vectors of specific size.
uint64_t GetMtfFloatVectorOfSize(uint32_t size) const {
return kMtfFloatVectorOfSizeBegin + size;
}
// Returns mtf handle for vector type of specific size.
uint64_t GetMtfTypeVectorOfSize(uint32_t size) const {
return kMtfTypeVectorOfSizeBegin + size;
@ -450,9 +460,14 @@ class MarkvCodecBase {
return kMtfFunctionWithReturnTypeBegin + type_id;
}
// Returns mtf handle for the given id descriptor.
uint64_t GetMtfIdDescriptor(uint32_t descriptor) const {
return kMtfIdDescriptorSpaceBegin + descriptor;
// Returns mtf handle for the given long id descriptor.
uint64_t GetMtfLongIdDescriptor(uint32_t descriptor) const {
return kMtfLongIdDescriptorSpaceBegin + descriptor;
}
// Returns mtf handle for the given short id descriptor.
uint64_t GetMtfShortIdDescriptor(uint32_t descriptor) const {
return kMtfShortIdDescriptorSpaceBegin + descriptor;
}
// Process data from the current instruction. This would update MTFs and
@ -501,6 +516,18 @@ class MarkvCodecBase {
return it->second.get();
}
// Promotes id in all move-to-front sequences if ids can be shared by multiple
// sequences.
void PromoteIfNeeded(uint32_t id) {
if (!model_->AnyDescriptorHasCodingScheme() &&
model_->id_fallback_strategy() ==
MarkvModel::IdFallbackStrategy::kShortDescriptor) {
// Move-to-front sequences do not share ids. Nothing to do.
return;
}
multi_mtf_.Promote(id);
}
spv_validator_options validator_options_ = nullptr;
const libspirv::AssemblyGrammar grammar_;
MarkvHeader header_;
@ -537,8 +564,19 @@ class MarkvCodecBase {
// List of instructions in the order they are given in the module.
std::vector<std::unique_ptr<const Instruction>> instructions_;
// Container/computer for id descriptors.
IdDescriptorCollection id_descriptors_;
// Container/computer for long (32-bit) id descriptors.
IdDescriptorCollection long_id_descriptors_;
// Container/computer for short id descriptors.
// Short descriptors are stored in uint32_t, but their actual bit width is
// defined with kShortDescriptorNumBits.
// It doesn't seem logical to have a different computer for short id
// descriptors, since one could actually map/truncate long descriptors.
// But as short descriptors have collisions, the efficiency of
// compression depends on the collision pattern, and short descriptors
// produced by function ShortHashU32Array have been empirically proven to
// produce better results.
IdDescriptorCollection short_id_descriptors_;
// Huffman codecs for move-to-front ranks. The map key is mtf handle. Doesn't
// need to contain a different codec for every handle as most use one and the
@ -855,8 +893,11 @@ void MarkvCodecBase::ProcessCurInstruction() {
if (opcode == SpvOpFunction) {
cur_function_id_ = inst_.result_id;
cur_function_return_type_ = inst_.type_id;
if (model_->id_fallback_strategy() ==
MarkvModel::IdFallbackStrategy::kRuleBased) {
multi_mtf_.Insert(GetMtfFunctionWithReturnType(inst_.type_id),
inst_.result_id);
}
// Store function parameter types in a queue, so that we know which types
// to expect in the following OpFunctionParameter instructions.
@ -886,15 +927,16 @@ void MarkvCodecBase::ProcessCurInstruction() {
// have no type Id, and will map to 0. The result Id for a
// type-generating instruction (e.g. OpTypeInt) maps to itself.
auto insertion_result = id_to_type_id_.emplace(
inst_.result_id,
spvOpcodeGeneratesType(SpvOp(inst_.opcode)) ? inst_.result_id
inst_.result_id, spvOpcodeGeneratesType(SpvOp(inst_.opcode))
? inst_.result_id
: inst_.type_id);
(void)insertion_result;
assert(insertion_result.second);
}
// Add result_id to MTFs.
if (model_->id_fallback_strategy() ==
MarkvModel::IdFallbackStrategy::kRuleBased) {
switch (opcode) {
case SpvOpTypeFloat:
case SpvOpTypeInt:
@ -1017,10 +1059,23 @@ void MarkvCodecBase::ProcessCurInstruction() {
multi_mtf_.Insert(kMtfTypeNonFunction, inst_.result_id);
}
}
}
const uint32_t descriptor = id_descriptors_.ProcessInstruction(inst_);
if (model_->DescriptorHasCodingScheme(descriptor))
multi_mtf_.Insert(GetMtfIdDescriptor(descriptor), inst_.result_id);
if (model_->AnyDescriptorHasCodingScheme()) {
const uint32_t long_descriptor =
long_id_descriptors_.ProcessInstruction(inst_);
if (model_->DescriptorHasCodingScheme(long_descriptor))
multi_mtf_.Insert(GetMtfLongIdDescriptor(long_descriptor),
inst_.result_id);
}
if (model_->id_fallback_strategy() ==
MarkvModel::IdFallbackStrategy::kShortDescriptor) {
const uint32_t short_descriptor =
short_id_descriptors_.ProcessInstruction(inst_);
multi_mtf_.Insert(GetMtfShortIdDescriptor(short_descriptor),
inst_.result_id);
}
}
uint64_t MarkvCodecBase::GetRuleBasedMtf() {
@ -1052,7 +1107,6 @@ uint64_t MarkvCodecBase::GetRuleBasedMtf() {
case SpvOpFMod:
case SpvOpFNegate: {
if (operand_index_ == 0) return kMtfTypeFloatScalarOrVector;
return GetMtfIdOfType(inst_.type_id);
}
@ -1637,21 +1691,21 @@ spv_result_t MarkvDecoder::DecodeMtfRankHuffman(uint64_t mtf,
}
spv_result_t MarkvEncoder::EncodeIdWithDescriptor(uint32_t id) {
// Get the descriptor for id.
const uint32_t long_descriptor = long_id_descriptors_.GetDescriptor(id);
auto* codec =
model_->GetIdDescriptorHuffmanCodec(inst_.opcode, operand_index_);
if (!codec) return SPV_UNSUPPORTED;
uint64_t bits = 0;
size_t num_bits = 0;
// Get the descriptor for id.
const uint32_t descriptor = id_descriptors_.GetDescriptor(id);
if (descriptor && codec->Encode(descriptor, &bits, &num_bits)) {
uint64_t mtf = kMtfNone;
if (long_descriptor && codec &&
codec->Encode(long_descriptor, &bits, &num_bits)) {
// If the descriptor exists and is in the table, write the descriptor and
// proceed to encoding the rank.
writer_.WriteBits(bits, num_bits);
mtf = GetMtfLongIdDescriptor(long_descriptor);
} else {
if (codec) {
// The descriptor doesn't exist or we have no coding for it. Write
// kMarkvNoneOfTheAbove and go to fallback method.
if (!codec->Encode(kMarkvNoneOfTheAbove, &bits, &num_bits))
@ -1661,33 +1715,63 @@ spv_result_t MarkvEncoder::EncodeIdWithDescriptor(uint32_t id) {
<< operand_index_ << " is missing kMarkvNoneOfTheAbove";
writer_.WriteBits(bits, num_bits);
}
if (model_->id_fallback_strategy() !=
MarkvModel::IdFallbackStrategy::kShortDescriptor) {
return SPV_UNSUPPORTED;
}
const uint32_t short_descriptor = short_id_descriptors_.GetDescriptor(id);
writer_.WriteBits(short_descriptor, kShortDescriptorNumBits);
if (short_descriptor == 0) {
// Forward declared id.
return SPV_UNSUPPORTED;
}
mtf = GetMtfShortIdDescriptor(short_descriptor);
}
// Descriptor has been encoded. Now encode the rank of the id in the
// associated mtf sequence.
const uint64_t mtf = GetMtfIdDescriptor(descriptor);
return EncodeExistingId(mtf, id);
}
spv_result_t MarkvDecoder::DecodeIdWithDescriptor(uint32_t* id) {
auto* codec =
model_->GetIdDescriptorHuffmanCodec(inst_.opcode, operand_index_);
if (!codec) return SPV_UNSUPPORTED;
uint64_t mtf = kMtfNone;
if (codec) {
uint64_t decoded_value = 0;
if (!codec->DecodeFromStream(GetReadBitCallback(), &decoded_value))
return Diag(SPV_ERROR_INTERNAL)
<< "Failed to decode descriptor with Huffman";
if (decoded_value == kMarkvNoneOfTheAbove) return SPV_UNSUPPORTED;
if (decoded_value != kMarkvNoneOfTheAbove) {
const uint32_t long_descriptor = uint32_t(decoded_value);
mtf = GetMtfLongIdDescriptor(long_descriptor);
}
}
// If descriptor exists then the id was encoded through descriptor mtf.
const uint32_t descriptor = uint32_t(decoded_value);
assert(descriptor == decoded_value);
assert(descriptor);
if (mtf == kMtfNone) {
if (model_->id_fallback_strategy() !=
MarkvModel::IdFallbackStrategy::kShortDescriptor) {
return SPV_UNSUPPORTED;
}
uint64_t decoded_value = 0;
if (!reader_.ReadBits(&decoded_value, kShortDescriptorNumBits))
return Diag(SPV_ERROR_INTERNAL) << "Failed to read short descriptor";
const uint32_t short_descriptor = uint32_t(decoded_value);
if (short_descriptor == 0) {
// Forward declared id.
return SPV_UNSUPPORTED;
}
mtf = GetMtfShortIdDescriptor(short_descriptor);
}
const uint64_t mtf = GetMtfIdDescriptor(descriptor);
return DecodeExistingId(mtf, id);
}
@ -1735,10 +1819,14 @@ spv_result_t MarkvEncoder::EncodeRefId(uint32_t id) {
// If can't be done continue with other methods.
}
// Encode using rule-based mtf.
uint64_t mtf = GetRuleBasedMtf();
const bool can_forward_declare = spvOperandCanBeForwardDeclaredFunction(
SpvOp(inst_.opcode))(operand_index_);
uint32_t rank = 0;
if (model_->id_fallback_strategy() ==
MarkvModel::IdFallbackStrategy::kRuleBased) {
// Encode using rule-based mtf.
uint64_t mtf = GetRuleBasedMtf();
if (mtf != kMtfNone && !can_forward_declare) {
assert(multi_mtf_.HasValue(kMtfAll, id));
@ -1747,8 +1835,6 @@ spv_result_t MarkvEncoder::EncodeRefId(uint32_t id) {
if (mtf == kMtfNone) mtf = kMtfAll;
uint32_t rank = 0;
if (!multi_mtf_.RankFromValue(mtf, id, &rank)) {
// This is the first occurrence of a forward declared id.
multi_mtf_.Insert(kMtfAll, id);
@ -1758,6 +1844,18 @@ spv_result_t MarkvEncoder::EncodeRefId(uint32_t id) {
}
return EncodeMtfRankHuffman(rank, mtf, kMtfAll);
} else {
assert(can_forward_declare);
if (!multi_mtf_.RankFromValue(kMtfForwardDeclared, id, &rank)) {
// This is the first occurrence of a forward declared id.
multi_mtf_.Insert(kMtfForwardDeclared, id);
rank = 0;
}
writer_.WriteVariableWidthU32(rank, model_->mtf_rank_chunk_length());
return SPV_SUCCESS;
}
}
spv_result_t MarkvDecoder::DecodeRefId(uint32_t* id) {
@ -1766,20 +1864,19 @@ spv_result_t MarkvDecoder::DecodeRefId(uint32_t* id) {
if (result != SPV_UNSUPPORTED) return result;
}
uint64_t mtf = GetRuleBasedMtf();
const bool can_forward_declare = spvOperandCanBeForwardDeclaredFunction(
SpvOp(inst_.opcode))(operand_index_);
uint32_t rank = 0;
*id = 0;
if (model_->id_fallback_strategy() ==
MarkvModel::IdFallbackStrategy::kRuleBased) {
uint64_t mtf = GetRuleBasedMtf();
if (mtf != kMtfNone && !can_forward_declare) {
return DecodeExistingId(mtf, id);
}
if (mtf == kMtfNone) mtf = kMtfAll;
*id = 0;
uint32_t rank = 0;
{
const spv_result_t result = DecodeMtfRankHuffman(mtf, kMtfAll, &rank);
if (result != SPV_SUCCESS) return result;
@ -1796,7 +1893,23 @@ spv_result_t MarkvDecoder::DecodeRefId(uint32_t* id) {
if (!multi_mtf_.ValueFromRank(mtf, rank, id))
return Diag(SPV_ERROR_INTERNAL) << "MTF rank out of bounds";
}
} else {
assert(can_forward_declare);
if (!reader_.ReadVariableWidthU32(&rank, model_->mtf_rank_chunk_length()))
return Diag(SPV_ERROR_INTERNAL)
<< "Failed to decode MTF rank with varint";
if (rank == 0) {
// This is the first occurrence of a forward declared id.
*id = GetIdBound();
SetIdBound(*id + 1);
multi_mtf_.Insert(kMtfForwardDeclared, *id);
} else {
if (!multi_mtf_.ValueFromRank(kMtfForwardDeclared, rank, id))
return Diag(SPV_ERROR_INTERNAL) << "MTF rank out of bounds";
}
}
assert(*id);
return SPV_SUCCESS;
}
@ -1816,6 +1929,9 @@ spv_result_t MarkvEncoder::EncodeTypeId() {
// If can't be done continue with other methods.
}
assert(model_->id_fallback_strategy() ==
MarkvModel::IdFallbackStrategy::kRuleBased);
uint64_t mtf = GetRuleBasedMtf();
assert(!spvOperandCanBeForwardDeclaredFunction(SpvOp(inst_.opcode))(
operand_index_));
@ -1842,6 +1958,9 @@ spv_result_t MarkvDecoder::DecodeTypeId() {
if (result != SPV_UNSUPPORTED) return result;
}
assert(model_->id_fallback_strategy() ==
MarkvModel::IdFallbackStrategy::kRuleBased);
uint64_t mtf = GetRuleBasedMtf();
assert(!spvOperandCanBeForwardDeclaredFunction(SpvOp(inst_.opcode))(
operand_index_));
@ -1878,9 +1997,12 @@ spv_result_t MarkvEncoder::EncodeResultId() {
}
}
if (model_->id_fallback_strategy() ==
MarkvModel::IdFallbackStrategy::kRuleBased) {
if (!rank) {
multi_mtf_.Insert(kMtfAll, inst_.result_id);
}
}
return SPV_SUCCESS;
}
@ -1924,9 +2046,12 @@ spv_result_t MarkvDecoder::DecodeResultId() {
SetIdBound(inst_.result_id + 1);
}
if (model_->id_fallback_strategy() ==
MarkvModel::IdFallbackStrategy::kRuleBased) {
if (!rank) {
multi_mtf_.Insert(kMtfAll, inst_.result_id);
}
}
return SPV_SUCCESS;
}
@ -2078,7 +2203,7 @@ spv_result_t MarkvEncoder::EncodeInstruction(
if (result != SPV_SUCCESS) return result;
}
multi_mtf_.Promote(id);
PromoteIfNeeded(id);
break;
}
@ -2248,7 +2373,7 @@ spv_result_t MarkvDecoder::DecodeOperand(
inst_words_.push_back(inst_.result_id);
SetIdBound(std::max(GetIdBound(), inst_.result_id + 1));
multi_mtf_.Promote(inst_.result_id);
PromoteIfNeeded(inst_.result_id);
break;
}
@ -2258,7 +2383,7 @@ spv_result_t MarkvDecoder::DecodeOperand(
inst_words_.push_back(inst_.type_id);
SetIdBound(std::max(GetIdBound(), inst_.type_id + 1));
multi_mtf_.Promote(inst_.type_id);
PromoteIfNeeded(inst_.type_id);
break;
}
@ -2291,7 +2416,7 @@ spv_result_t MarkvDecoder::DecodeOperand(
inst_words_.push_back(id);
SetIdBound(std::max(GetIdBound(), id + 1));
multi_mtf_.Promote(id);
PromoteIfNeeded(id);
break;
}
@ -2793,4 +2918,4 @@ spv_result_t MarkvToSpirv(
return SPV_SUCCESS;
}
} // namespave spvtools
} // namespace spvtools

57
source/comp/markv_model.h
View File

@ -32,7 +32,46 @@ class MarkvModel {
public:
MarkvModel()
: operand_chunk_lengths_(
static_cast<size_t>(SPV_OPERAND_TYPE_NUM_OPERAND_TYPES), 0) {}
static_cast<size_t>(SPV_OPERAND_TYPE_NUM_OPERAND_TYPES), 0) {
// Set default values.
operand_chunk_lengths_[SPV_OPERAND_TYPE_TYPE_ID] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_RESULT_ID] = 8;
operand_chunk_lengths_[SPV_OPERAND_TYPE_ID] = 8;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SCOPE_ID] = 8;
operand_chunk_lengths_[SPV_OPERAND_TYPE_MEMORY_SEMANTICS_ID] = 8;
operand_chunk_lengths_[SPV_OPERAND_TYPE_LITERAL_INTEGER] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_LITERAL_INTEGER] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_CAPABILITY] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SOURCE_LANGUAGE] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_EXECUTION_MODEL] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_ADDRESSING_MODEL] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_MEMORY_MODEL] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_EXECUTION_MODE] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_STORAGE_CLASS] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_DIMENSIONALITY] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SAMPLER_ADDRESSING_MODE] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SAMPLER_FILTER_MODE] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SAMPLER_IMAGE_FORMAT] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_FP_ROUNDING_MODE] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_LINKAGE_TYPE] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_ACCESS_QUALIFIER] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_ACCESS_QUALIFIER] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_FUNCTION_PARAMETER_ATTRIBUTE] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_DECORATION] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_BUILT_IN] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_GROUP_OPERATION] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_KERNEL_ENQ_FLAGS] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_KERNEL_PROFILING_INFO] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_FP_FAST_MATH_MODE] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_FUNCTION_CONTROL] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_LOOP_CONTROL] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_IMAGE] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_IMAGE] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_MEMORY_ACCESS] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SELECTION_CONTROL] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_EXTENSION_INSTRUCTION_NUMBER] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_TYPED_LITERAL_NUMBER] = 6;
}
uint32_t model_type() const { return model_type_; }
uint32_t model_version() const { return model_version_; }
@ -47,6 +86,15 @@ class MarkvModel {
uint32_t s64_chunk_length() const { return s64_chunk_length_; }
uint32_t s64_block_exponent() const { return s64_block_exponent_; }
enum class IdFallbackStrategy {
kRuleBased = 0,
kShortDescriptor,
};
IdFallbackStrategy id_fallback_strategy() const {
return id_fallback_strategy_;
}
// Returns a codec for common opcode_and_num_operands words for the given
// previous opcode. May return nullptr if the codec doesn't exist.
const spvutils::HuffmanCodec<uint64_t>*
@ -99,6 +147,11 @@ class MarkvModel {
return descriptors_with_coding_scheme_.count(descriptor);
}
// Checks if any descriptor has a coding scheme.
bool AnyDescriptorHasCodingScheme() const {
return !descriptors_with_coding_scheme_.empty();
}
// Returns chunk length used for variable length encoding of spirv operand
// words.
uint32_t GetOperandVariableWidthChunkLength(spv_operand_type_t type) const {
@ -167,6 +220,8 @@ class MarkvModel {
uint32_t s64_chunk_length_ = 8;
uint32_t s64_block_exponent_ = 10;
IdFallbackStrategy id_fallback_strategy_ = IdFallbackStrategy::kShortDescriptor;
uint32_t model_type_ = 0;
uint32_t model_version_ = 0;
};

5
source/id_descriptor.cpp
View File

@ -63,7 +63,10 @@ uint32_t IdDescriptorCollection::ProcessInstruction(
}
}
const uint32_t descriptor = HashU32Array(words_);
uint32_t descriptor = custom_hash_func_ ?
custom_hash_func_(words_) : HashU32Array(words_);
if (descriptor == 0)
descriptor = 1;
assert(descriptor);
words_.clear();

10
source/id_descriptor.h
View File

@ -22,13 +22,19 @@
namespace libspirv {
using CustomHashFunc = std::function<uint32_t(const std::vector<uint32_t>&)>;
// Computes and stores id descriptors.
//
// Descriptors are computed as hash of all words in the instruction where ids
// were substituted with previously computed descriptors.
class IdDescriptorCollection {
public:
IdDescriptorCollection() { words_.reserve(16); }
explicit IdDescriptorCollection(
CustomHashFunc custom_hash_func = CustomHashFunc())
: custom_hash_func_(custom_hash_func) {
words_.reserve(16);
}
// Computes descriptor for the result id of the given instruction and
// registers it in id_to_descriptor_. Returns the computed descriptor.
@ -46,6 +52,8 @@ class IdDescriptorCollection {
private:
std::unordered_map<uint32_t, uint32_t> id_to_descriptor_;
std::function<uint32_t(const std::vector<uint32_t>&)> custom_hash_func_;
// Scratch buffer used for hashing. Class member to optimize on allocation.
std::vector<uint32_t> words_;
};

2
test/comp/CMakeLists.txt
View File

@ -22,7 +22,7 @@ if(SPIRV_BUILD_COMPRESSION)
SRCS
markv_codec_test.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../tools/comp/markv_model_factory.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../tools/comp/markv_model_shader_default.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../tools/comp/markv_model_shader.cpp
${VAL_TEST_COMMON_SRCS}
LIBS SPIRV-Tools-comp ${SPIRV_TOOLS}
)

152
test/comp/markv_codec_test.cpp
View File

@ -26,7 +26,9 @@
namespace {
using spvtools::MarkvModelType;
using spvtest::ScopedContext;
using MarkvTest = ::testing::TestWithParam<MarkvModelType>;
void DiagnosticsMessageHandler(spv_message_level_t level, const char*,
const spv_position_t& position,
@ -85,10 +87,11 @@ void Disassemble(const std::vector<uint32_t>& words,
// Encodes/decodes |original|, assembles/dissasembles |original|, then compares
// the results of the two operations.
void TestEncodeDecode(const std::string& original_text) {
void TestEncodeDecode(MarkvModelType model_type,
const std::string& original_text) {
ScopedContext ctx(SPV_ENV_UNIVERSAL_1_2);
std::unique_ptr<spvtools::MarkvModel> model =
spvtools::CreateMarkvModel(spvtools::kMarkvModelShaderDefault);
spvtools::CreateMarkvModel(model_type);
spvtools::MarkvCodecOptions options;
std::vector<uint32_t> expected_binary;
@ -133,7 +136,8 @@ void TestEncodeDecode(const std::string& original_text) {
EXPECT_EQ(expected_text, decoded_text) << encoder_comments.str();
}
void TestEncodeDecodeShaderMainBody(const std::string& body) {
void TestEncodeDecodeShaderMainBody(MarkvModelType model_type,
const std::string& body) {
const std::string prefix =
R"(
OpCapability Shader
@ -216,11 +220,11 @@ R"(
OpReturn
OpFunctionEnd)";
TestEncodeDecode(prefix + body + suffix);
TestEncodeDecode(model_type, prefix + body + suffix);
}
TEST(Markv, U32Literal) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, U32Literal) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
OpCapability Linkage
OpMemoryModel Logical GLSL450
@ -231,8 +235,8 @@ OpMemoryModel Logical GLSL450
)");
}
TEST(Markv, S32Literal) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, S32Literal) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
OpCapability Linkage
OpMemoryModel Logical GLSL450
@ -245,8 +249,8 @@ OpMemoryModel Logical GLSL450
)");
}
TEST(Markv, U64Literal) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, U64Literal) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
OpCapability Linkage
OpCapability Int64
@ -258,8 +262,8 @@ OpMemoryModel Logical GLSL450
)");
}
TEST(Markv, S64Literal) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, S64Literal) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
OpCapability Linkage
OpCapability Int64
@ -273,8 +277,8 @@ OpMemoryModel Logical GLSL450
)");
}
TEST(Markv, U16Literal) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, U16Literal) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
OpCapability Linkage
OpCapability Int16
@ -286,8 +290,8 @@ OpMemoryModel Logical GLSL450
)");
}
TEST(Markv, S16Literal) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, S16Literal) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
OpCapability Linkage
OpCapability Int16
@ -301,8 +305,8 @@ OpMemoryModel Logical GLSL450
)");
}
TEST(Markv, F32Literal) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, F32Literal) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
OpCapability Linkage
OpMemoryModel Logical GLSL450
@ -314,8 +318,8 @@ OpMemoryModel Logical GLSL450
)");
}
TEST(Markv, F64Literal) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, F64Literal) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
OpCapability Linkage
OpCapability Float64
@ -328,8 +332,8 @@ OpMemoryModel Logical GLSL450
)");
}
TEST(Markv, F16Literal) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, F16Literal) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
OpCapability Linkage
OpCapability Float16
@ -342,8 +346,8 @@ OpMemoryModel Logical GLSL450
)");
}
TEST(Markv, StringLiteral) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, StringLiteral) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
OpCapability Linkage
OpExtension "SPV_KHR_16bit_storage"
@ -354,8 +358,8 @@ OpMemoryModel Logical GLSL450
)");
}
TEST(Markv, WithFunction) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, WithFunction) {
TestEncodeDecode(GetParam(), R"(
OpCapability Addresses
OpCapability Kernel
OpCapability GenericPointer
@ -376,8 +380,8 @@ OpFunctionEnd
)");
}
TEST(Markv, WithMultipleFunctions) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, WithMultipleFunctions) {
TestEncodeDecode(GetParam(), R"(
OpCapability Addresses
OpCapability Kernel
OpCapability GenericPointer
@ -409,8 +413,8 @@ OpFunctionEnd
)");
}
TEST(Markv, ForwardDeclaredId) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, ForwardDeclaredId) {
TestEncodeDecode(GetParam(), R"(
OpCapability Addresses
OpCapability Kernel
OpCapability GenericPointer
@ -430,8 +434,8 @@ OpFunctionEnd
)");
}
TEST(Markv, WithSwitch) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, WithSwitch) {
TestEncodeDecode(GetParam(), R"(
OpCapability Addresses
OpCapability Kernel
OpCapability GenericPointer
@ -460,8 +464,8 @@ OpFunctionEnd
)");
}
TEST(Markv, WithLoop) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, WithLoop) {
TestEncodeDecode(GetParam(), R"(
OpCapability Addresses
OpCapability Kernel
OpCapability GenericPointer
@ -485,8 +489,8 @@ OpFunctionEnd
)");
}
TEST(Markv, WithDecorate) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, WithDecorate) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
OpCapability Linkage
OpMemoryModel Logical GLSL450
@ -497,8 +501,8 @@ OpDecorate %1 Uniform
)");
}
TEST(Markv, WithExtInst) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, WithExtInst) {
TestEncodeDecode(GetParam(), R"(
OpCapability Addresses
OpCapability Kernel
OpCapability GenericPointer
@ -517,8 +521,8 @@ OpFunctionEnd
)");
}
TEST(Markv, F32Mul) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, F32Mul) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%val1 = OpFMul %f32 %f32_0 %f32_1
%val2 = OpFMul %f32 %f32_2 %f32_0
%val3 = OpFMul %f32 %f32_pi %f32_2
@ -526,8 +530,8 @@ TEST(Markv, F32Mul) {
)");
}
TEST(Markv, U32Mul) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, U32Mul) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%val1 = OpIMul %u32 %u32_0 %u32_1
%val2 = OpIMul %u32 %u32_2 %u32_0
%val3 = OpIMul %u32 %u32_3 %u32_2
@ -535,8 +539,8 @@ TEST(Markv, U32Mul) {
)");
}
TEST(Markv, S32Mul) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, S32Mul) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%val1 = OpIMul %s32 %s32_0 %s32_1
%val2 = OpIMul %s32 %s32_2 %s32_0
%val3 = OpIMul %s32 %s32_m1 %s32_2
@ -544,8 +548,8 @@ TEST(Markv, S32Mul) {
)");
}
TEST(Markv, F32Add) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, F32Add) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%val1 = OpFAdd %f32 %f32_0 %f32_1
%val2 = OpFAdd %f32 %f32_2 %f32_0
%val3 = OpFAdd %f32 %f32_pi %f32_2
@ -553,8 +557,8 @@ TEST(Markv, F32Add) {
)");
}
TEST(Markv, U32Add) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, U32Add) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%val1 = OpIAdd %u32 %u32_0 %u32_1
%val2 = OpIAdd %u32 %u32_2 %u32_0
%val3 = OpIAdd %u32 %u32_3 %u32_2
@ -562,8 +566,8 @@ TEST(Markv, U32Add) {
)");
}
TEST(Markv, S32Add) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, S32Add) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%val1 = OpIAdd %s32 %s32_0 %s32_1
%val2 = OpIAdd %s32 %s32_2 %s32_0
%val3 = OpIAdd %s32 %s32_m1 %s32_2
@ -571,8 +575,8 @@ TEST(Markv, S32Add) {
)");
}
TEST(Markv, F32Dot) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, F32Dot) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%dot2_1 = OpDot %f32 %f32vec2_01 %f32vec2_12
%dot2_2 = OpDot %f32 %f32vec2_01 %f32vec2_01
%dot2_3 = OpDot %f32 %f32vec2_12 %f32vec2_12
@ -585,8 +589,8 @@ TEST(Markv, F32Dot) {
)");
}
TEST(Markv, F32VectorCompositeConstruct) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, F32VectorCompositeConstruct) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%cc1 = OpCompositeConstruct %f32vec4 %f32vec2_01 %f32vec2_12
%cc2 = OpCompositeConstruct %f32vec3 %f32vec2_01 %f32_2
%cc3 = OpCompositeConstruct %f32vec2 %f32_1 %f32_2
@ -594,8 +598,8 @@ TEST(Markv, F32VectorCompositeConstruct) {
)");
}
TEST(Markv, U32VectorCompositeConstruct) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, U32VectorCompositeConstruct) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%cc1 = OpCompositeConstruct %u32vec4 %u32vec2_01 %u32vec2_12
%cc2 = OpCompositeConstruct %u32vec3 %u32vec2_01 %u32_2
%cc3 = OpCompositeConstruct %u32vec2 %u32_1 %u32_2
@ -603,8 +607,8 @@ TEST(Markv, U32VectorCompositeConstruct) {
)");
}
TEST(Markv, S32VectorCompositeConstruct) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, S32VectorCompositeConstruct) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%cc1 = OpCompositeConstruct %u32vec4 %u32vec2_01 %u32vec2_12
%cc2 = OpCompositeConstruct %u32vec3 %u32vec2_01 %u32_2
%cc3 = OpCompositeConstruct %u32vec2 %u32_1 %u32_2
@ -612,15 +616,15 @@ TEST(Markv, S32VectorCompositeConstruct) {
)");
}
TEST(Markv, F32VectorCompositeExtract) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, F32VectorCompositeExtract) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%f32vec4_3210 = OpCompositeConstruct %f32vec4 %f32_3 %f32_2 %f32_1 %f32_0
%f32vec3_013 = OpCompositeExtract %f32vec3 %f32vec4_0123 0 1 3
)");
}
TEST(Markv, F32VectorComparison) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, F32VectorComparison) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%f32vec4_3210 = OpCompositeConstruct %f32vec4 %f32_3 %f32_2 %f32_1 %f32_0
%c1 = OpFOrdEqual %boolvec4 %f32vec4_0123 %f32vec4_3210
%c2 = OpFUnordEqual %boolvec4 %f32vec4_0123 %f32vec4_3210
@ -637,24 +641,24 @@ TEST(Markv, F32VectorComparison) {
)");
}
TEST(Markv, VectorShuffle) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, VectorShuffle) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%f32vec4_3210 = OpCompositeConstruct %f32vec4 %f32_3 %f32_2 %f32_1 %f32_0
%sh1 = OpVectorShuffle %f32vec2 %f32vec4_0123 %f32vec4_3210 3 6
%sh2 = OpVectorShuffle %f32vec3 %f32vec2_01 %f32vec4_3210 0 3 4
)");
}
TEST(Markv, VectorTimesScalar) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, VectorTimesScalar) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%f32vec4_3210 = OpCompositeConstruct %f32vec4 %f32_3 %f32_2 %f32_1 %f32_0
%res1 = OpVectorTimesScalar %f32vec4 %f32vec4_0123 %f32_2
%res2 = OpVectorTimesScalar %f32vec4 %f32vec4_3210 %f32_2
)");
}
TEST(Markv, SpirvSpecSample) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, SpirvSpecSample) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
%1 = OpExtInstImport "GLSL.std.450"
OpMemoryModel Logical GLSL450
@ -775,8 +779,8 @@ TEST(Markv, SpirvSpecSample) {
)");
}
TEST(Markv, SampleFromDeadBranchEliminationTest) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, SampleFromDeadBranchEliminationTest) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
%1 = OpExtInstImport "GLSL.std.450"
OpMemoryModel Logical GLSL450
@ -815,4 +819,12 @@ OpFunctionEnd
)");
}
INSTANTIATE_TEST_CASE_P(
AllMarkvModels, MarkvTest,
::testing::ValuesIn(std::vector<MarkvModelType>{
spvtools::kMarkvModelShaderLite,
spvtools::kMarkvModelShaderMid,
spvtools::kMarkvModelShaderMax,
}),);
} // namespace

2
tools/CMakeLists.txt
View File

@ -64,7 +64,7 @@ if (NOT ${SPIRV_SKIP_EXECUTABLES})
add_spvtools_tool(TARGET spirv-markv
SRCS comp/markv.cpp
comp/markv_model_factory.cpp
comp/markv_model_shader_default.cpp
comp/markv_model_shader.cpp
LIBS SPIRV-Tools-comp SPIRV-Tools-opt ${SPIRV_TOOLS})
target_include_directories(spirv-markv PRIVATE ${spirv-tools_SOURCE_DIR}
${SPIRV_HEADER_INCLUDE_DIR})

25
tools/comp/markv.cpp
View File

@ -72,6 +72,12 @@ Options:
--comments Write codec comments to stderr.
--version Display MARK-V codec version.
--validate Validate SPIR-V while encoding or decoding.
--model=<model-name>
Compression model, possible values:
shader_lite - fast, poor compression ratio
shader_mid - balanced
shader_max - best compression ratio
Default: shader_lite
-o <filename> Set the output filename.
Output goes to standard output if this option is
@ -132,6 +138,8 @@ int main(int argc, char** argv) {
bool want_comments = false;
bool validate_spirv_binary = false;
spvtools::MarkvModelType model_type = spvtools::kMarkvModelUnknown;
for (int argi = 2; argi < argc; ++argi) {
if ('-' == argv[argi][0]) {
switch (argv[argi][1]) {
@ -158,6 +166,18 @@ int main(int argc, char** argv) {
return 1;
} else if (0 == strcmp(argv[argi], "--validate")) {
validate_spirv_binary = true;
} else if (0 == strcmp(argv[argi], "--model=shader_lite")) {
if (model_type != spvtools::kMarkvModelUnknown)
fprintf(stderr, "error: More than one model specified\n");
model_type = spvtools::kMarkvModelShaderLite;
} else if (0 == strcmp(argv[argi], "--model=shader_mid")) {
if (model_type != spvtools::kMarkvModelUnknown)
fprintf(stderr, "error: More than one model specified\n");
model_type = spvtools::kMarkvModelShaderMid;
} else if (0 == strcmp(argv[argi], "--model=shader_max")) {
if (model_type != spvtools::kMarkvModelUnknown)
fprintf(stderr, "error: More than one model specified\n");
model_type = spvtools::kMarkvModelShaderMax;
} else {
print_usage(argv[0]);
return 1;
@ -186,6 +206,9 @@ int main(int argc, char** argv) {
}
}
if (model_type == spvtools::kMarkvModelUnknown)
model_type = spvtools::kMarkvModelShaderLite;
const auto no_comments = spvtools::MarkvLogConsumer();
const auto output_to_stderr = [](const std::string& str) {
std::cerr << str;
@ -194,7 +217,7 @@ int main(int argc, char** argv) {
ScopedContext ctx(kSpvEnv);
std::unique_ptr<spvtools::MarkvModel> model =
spvtools::CreateMarkvModel(spvtools::kMarkvModelShaderDefault);
spvtools::CreateMarkvModel(model_type);
std::vector<uint32_t> spirv;
std::vector<uint8_t> markv;

19
tools/comp/markv_model_factory.cpp
View File

@ -13,17 +13,30 @@
// limitations under the License.
#include "markv_model_factory.h"
#include "markv_model_shader_default.h"
#include "markv_model_shader.h"
namespace spvtools {
std::unique_ptr<MarkvModel> CreateMarkvModel(MarkvModelType type) {
std::unique_ptr<MarkvModel> model;
switch (type) {
case kMarkvModelShaderDefault: {
model.reset(new MarkvModelShaderDefault());
case kMarkvModelShaderLite: {
model.reset(new MarkvModelShaderLite());
break;
}
case kMarkvModelShaderMid: {
model.reset(new MarkvModelShaderMid());
break;
}
case kMarkvModelShaderMax: {
model.reset(new MarkvModelShaderMax());
break;
}
case kMarkvModelUnknown: {
assert(0 && "kMarkvModelUnknown supplied to CreateMarkvModel");
return model;
}
}
model->SetModelType(static_cast<uint32_t>(type));

5
tools/comp/markv_model_factory.h
View File

@ -22,7 +22,10 @@
namespace spvtools {
enum MarkvModelType {
kMarkvModelShaderDefault = 1,
kMarkvModelUnknown = 0,
kMarkvModelShaderLite,
kMarkvModelShaderMid,
kMarkvModelShaderMax,
};
std::unique_ptr<MarkvModel> CreateMarkvModel(MarkvModelType type);

100
tools/comp/markv_model_shader.cpp Normal file
View File

@ -0,0 +1,100 @@
// Copyright (c) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "markv_model_shader.h"
#include <algorithm>
#include <map>
#include <memory>
#include <unordered_map>
#include <unordered_set>
#include <vector>
using spvutils::HuffmanCodec;
namespace spvtools {
namespace {
// Signals that the value is not in the coding scheme and a fallback method
// needs to be used.
const uint64_t kMarkvNoneOfTheAbove = MarkvModel::GetMarkvNoneOfTheAbove();
inline uint32_t CombineOpcodeAndNumOperands(uint32_t opcode,
uint32_t num_operands) {
return opcode | (num_operands << 16);
}
// The following file contains autogenerated statistical coding rules.
// Can be generated by running spirv-stats on representative corpus of shaders
// with flags:
// --codegen_opcode_and_num_operands_hist
// --codegen_opcode_and_num_operands_markov_huffman_codecs
// --codegen_literal_string_huffman_codecs
// --codegen_non_id_word_huffman_codecs
// --codegen_id_descriptor_huffman_codecs
//
// Example:
// find <SHADER_CORPUS_DIR> -type f -print0 | xargs -0 -s 2000000
// ~/SPIRV-Tools/build/tools/spirv-stats -v
// --codegen_opcode_and_num_operands_hist
// --codegen_opcode_and_num_operands_markov_huffman_codecs
// --codegen_literal_string_huffman_codecs --codegen_non_id_word_huffman_codecs
// --codegen_id_descriptor_huffman_codecs -o
// ~/SPIRV-Tools/source/comp/markv_autogen.inc
#include "markv_model_shader_default_autogen.inc"
} // namespace
MarkvModelShaderLite::MarkvModelShaderLite() {
const uint16_t kVersionNumber = 1;
SetModelVersion(kVersionNumber);
opcode_and_num_operands_huffman_codec_.reset(
new HuffmanCodec<uint64_t>(GetOpcodeAndNumOperandsHist()));
id_fallback_strategy_ = IdFallbackStrategy::kShortDescriptor;
}
MarkvModelShaderMid::MarkvModelShaderMid() {
const uint16_t kVersionNumber = 1;
SetModelVersion(kVersionNumber);
opcode_and_num_operands_huffman_codec_.reset(
new HuffmanCodec<uint64_t>(GetOpcodeAndNumOperandsHist()));
non_id_word_huffman_codecs_ = GetNonIdWordHuffmanCodecs();
id_descriptor_huffman_codecs_ = GetIdDescriptorHuffmanCodecs();
descriptors_with_coding_scheme_ = GetDescriptorsWithCodingScheme();
literal_string_huffman_codecs_ = GetLiteralStringHuffmanCodecs();
id_fallback_strategy_ = IdFallbackStrategy::kShortDescriptor;
}
MarkvModelShaderMax::MarkvModelShaderMax() {
const uint16_t kVersionNumber = 1;
SetModelVersion(kVersionNumber);
opcode_and_num_operands_huffman_codec_.reset(
new HuffmanCodec<uint64_t>(GetOpcodeAndNumOperandsHist()));
opcode_and_num_operands_markov_huffman_codecs_ =
GetOpcodeAndNumOperandsMarkovHuffmanCodecs();
non_id_word_huffman_codecs_ = GetNonIdWordHuffmanCodecs();
id_descriptor_huffman_codecs_ = GetIdDescriptorHuffmanCodecs();
descriptors_with_coding_scheme_ = GetDescriptorsWithCodingScheme();
literal_string_huffman_codecs_ = GetLiteralStringHuffmanCodecs();
id_fallback_strategy_ = IdFallbackStrategy::kRuleBased;
}
} // namespace spvtools

45
tools/comp/markv_model_shader.h Normal file
View File

@ -0,0 +1,45 @@
// Copyright (c) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef SPIRV_TOOLS_MARKV_MODEL_SHADER_H_
#define SPIRV_TOOLS_MARKV_MODEL_SHADER_H_
#include "source/comp/markv_model.h"
namespace spvtools {
// MARK-V shader compression model, which only uses fast and lightweight
// algorithms, which do not require training and are not heavily dependent on
// SPIR-V grammar. Compression ratio is worse than by other models.
class MarkvModelShaderLite : public MarkvModel {
public:
MarkvModelShaderLite();
};
// MARK-V shader compression model with balanced compression ratio and runtime
// performance.
class MarkvModelShaderMid : public MarkvModel {
public:
MarkvModelShaderMid();
};
// MARK-V shader compression model designed for maximum compression.
class MarkvModelShaderMax : public MarkvModel {
public:
MarkvModelShaderMax();
};
} // namespace spvtools
#endif // SPIRV_TOOLS_MARKV_MODEL_SHADER_H_

112
tools/comp/markv_model_shader_default.cpp
View File

@ -1,112 +0,0 @@
// Copyright (c) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "markv_model_shader_default.h"
#include <algorithm>
#include <map>
#include <memory>
#include <unordered_map>
#include <unordered_set>
#include <vector>
using spvutils::HuffmanCodec;
namespace spvtools {
namespace {
// Signals that the value is not in the coding scheme and a fallback method
// needs to be used.
const uint64_t kMarkvNoneOfTheAbove = MarkvModel::GetMarkvNoneOfTheAbove();
inline uint32_t CombineOpcodeAndNumOperands(uint32_t opcode,
uint32_t num_operands) {
return opcode | (num_operands << 16);
}
// The following file contains autogenerated statistical coding rules.
// Generated by running spirv-stats on representative corpus of shaders with
// flags:
// --codegen_opcode_and_num_operands_hist
// --codegen_opcode_and_num_operands_markov_huffman_codecs
// --codegen_literal_string_huffman_codecs
// --codegen_non_id_word_huffman_codecs
// --codegen_id_descriptor_huffman_codecs
//
// Example:
// find <SHADER_CORPUS_DIR> -type f -print0 | xargs -0 -s 2000000
// ~/SPIRV-Tools/build/tools/spirv-stats -v
// --codegen_opcode_and_num_operands_hist
// --codegen_opcode_and_num_operands_markov_huffman_codecs
// --codegen_literal_string_huffman_codecs --codegen_non_id_word_huffman_codecs
// --codegen_id_descriptor_huffman_codecs -o
// ~/SPIRV-Tools/source/comp/markv_autogen.inc
#include "markv_model_shader_default_autogen.inc"
} // namespace
MarkvModelShaderDefault::MarkvModelShaderDefault() {
const uint16_t kVersionNumber = 0;
SetModelVersion(kVersionNumber);
opcode_and_num_operands_huffman_codec_.reset(
new HuffmanCodec<uint64_t>(GetOpcodeAndNumOperandsHist()));
opcode_and_num_operands_markov_huffman_codecs_ =
GetOpcodeAndNumOperandsMarkovHuffmanCodecs();
non_id_word_huffman_codecs_ = GetNonIdWordHuffmanCodecs();
id_descriptor_huffman_codecs_ = GetIdDescriptorHuffmanCodecs();
descriptors_with_coding_scheme_ = GetDescriptorsWithCodingScheme();
literal_string_huffman_codecs_ = GetLiteralStringHuffmanCodecs();
operand_chunk_lengths_[SPV_OPERAND_TYPE_TYPE_ID] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_RESULT_ID] = 8;
operand_chunk_lengths_[SPV_OPERAND_TYPE_ID] = 8;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SCOPE_ID] = 8;
operand_chunk_lengths_[SPV_OPERAND_TYPE_MEMORY_SEMANTICS_ID] = 8;
operand_chunk_lengths_[SPV_OPERAND_TYPE_LITERAL_INTEGER] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_LITERAL_INTEGER] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_CAPABILITY] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SOURCE_LANGUAGE] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_EXECUTION_MODEL] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_ADDRESSING_MODEL] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_MEMORY_MODEL] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_EXECUTION_MODE] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_STORAGE_CLASS] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_DIMENSIONALITY] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SAMPLER_ADDRESSING_MODE] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SAMPLER_FILTER_MODE] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SAMPLER_IMAGE_FORMAT] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_FP_ROUNDING_MODE] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_LINKAGE_TYPE] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_ACCESS_QUALIFIER] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_ACCESS_QUALIFIER] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_FUNCTION_PARAMETER_ATTRIBUTE] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_DECORATION] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_BUILT_IN] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_GROUP_OPERATION] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_KERNEL_ENQ_FLAGS] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_KERNEL_PROFILING_INFO] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_FP_FAST_MATH_MODE] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_FUNCTION_CONTROL] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_LOOP_CONTROL] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_IMAGE] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_IMAGE] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_MEMORY_ACCESS] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SELECTION_CONTROL] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_EXTENSION_INSTRUCTION_NUMBER] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_TYPED_LITERAL_NUMBER] = 6;
}
} // namespace spvtools

30
tools/comp/markv_model_shader_default.h
View File

@ -1,30 +0,0 @@
// Copyright (c) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef SPIRV_TOOLS_COMP_DEFAULT_SHADER_MARKV_MODEL_H_
#define SPIRV_TOOLS_COMP_DEFAULT_SHADER_MARKV_MODEL_H_
#include "source/comp/markv_model.h"
namespace spvtools {
// MARK-V model designed to be a default model for shader compression.
class MarkvModelShaderDefault : public MarkvModel {
public:
MarkvModelShaderDefault();
};
} // namespace spvtools
#endif // SPIRV_TOOLS_COMP_DEFAULT_SHADER_MARKV_MODEL_H_