Add new compression algorithm and models

Add new "short descriptor" algorithm to MARK-V codec.

Add three shader compression models:
lite - fast, poor compression
mid - balanced
max - best compression
This commit is contained in:
Andrey Tuganov 2017-11-14 17:02:42 -05:00 committed by David Neto
parent a771713e42
commit 250a235a8d
14 changed files with 676 additions and 431 deletions

View File

@ -60,10 +60,10 @@
#include "val/validation_state.h"
#include "validate.h"
using libspirv::DiagnosticStream;
using libspirv::IdDescriptorCollection;
using libspirv::Instruction;
using libspirv::ValidationState_t;
using libspirv::DiagnosticStream;
using spvutils::BitReaderWord64;
using spvutils::BitWriterWord64;
using spvutils::HuffmanCodec;
@ -131,10 +131,10 @@ enum : uint64_t {
kMtfFunctionTypeWithReturnTypeBegin = 0x70000,
// All function objects which return specific type.
kMtfFunctionWithReturnTypeBegin = 0x80000,
// All float vectors of specific size.
kMtfFloatVectorOfSizeBegin = 0x90000,
// Id descriptor space (32-bit).
kMtfIdDescriptorSpaceBegin = 0x100000000,
// Short id descriptor space (max 16-bit).
kMtfShortIdDescriptorSpaceBegin = 0x90000,
// Long id descriptor space (32-bit).
kMtfLongIdDescriptorSpaceBegin = 0x100000000,
};
// Signals that the value is not in the coding scheme and a fallback method
@ -152,6 +152,20 @@ const size_t kCommentNumWhitespaces = 2;
const size_t kByteBreakAfterInstIfLessThanUntilNextByte = 8;
const uint32_t kShortDescriptorNumBits = 8;
// Custom hash function used to produce short descriptors.
uint32_t ShortHashU32Array(const std::vector<uint32_t>& words) {
// The hash function is a sum of hashes of each word seeded by word index.
// Knuth's multiplicative hash is used to hash the words.
const uint32_t kKnuthMulHash = 2654435761;
uint32_t val = 0;
for (uint32_t i = 0; i < words.size(); ++i) {
val += (words[i] + i + 123) * kKnuthMulHash;
}
return 1 + val % ((1 << kShortDescriptorNumBits) - 1);
}
// Returns a set of mtf rank codecs based on a plausible hand-coded
// distribution.
std::map<uint64_t, std::unique_ptr<HuffmanCodec<uint32_t>>>
@ -263,7 +277,7 @@ size_t GetNumBitsToNextByte(size_t bit_pos) { return (8 - (bit_pos % 8)) % 8; }
// Defines and returns current MARK-V version.
uint32_t GetMarkvVersion() {
const uint32_t kVersionMajor = 1;
const uint32_t kVersionMinor = 3;
const uint32_t kVersionMinor = 4;
return kVersionMinor | (kVersionMajor << 16);
}
@ -372,6 +386,7 @@ class MarkvCodecBase {
: validator_options_(validator_options),
grammar_(context),
model_(model),
short_id_descriptors_(ShortHashU32Array),
mtf_huffman_codecs_(GetMtfHuffmanCodecs()),
context_(context),
vstate_(validator_options
@ -425,11 +440,6 @@ class MarkvCodecBase {
return kMtfVectorOfComponentTypeBegin + type_id;
}
// Returns mtf handle for float vectors of specific size.
uint64_t GetMtfFloatVectorOfSize(uint32_t size) const {
return kMtfFloatVectorOfSizeBegin + size;
}
// Returns mtf handle for vector type of specific size.
uint64_t GetMtfTypeVectorOfSize(uint32_t size) const {
return kMtfTypeVectorOfSizeBegin + size;
@ -450,9 +460,14 @@ class MarkvCodecBase {
return kMtfFunctionWithReturnTypeBegin + type_id;
}
// Returns mtf handle for the given id descriptor.
uint64_t GetMtfIdDescriptor(uint32_t descriptor) const {
return kMtfIdDescriptorSpaceBegin + descriptor;
// Returns mtf handle for the given long id descriptor.
uint64_t GetMtfLongIdDescriptor(uint32_t descriptor) const {
return kMtfLongIdDescriptorSpaceBegin + descriptor;
}
// Returns mtf handle for the given short id descriptor.
uint64_t GetMtfShortIdDescriptor(uint32_t descriptor) const {
return kMtfShortIdDescriptorSpaceBegin + descriptor;
}
// Process data from the current instruction. This would update MTFs and
@ -501,6 +516,18 @@ class MarkvCodecBase {
return it->second.get();
}
// Promotes id in all move-to-front sequences if ids can be shared by multiple
// sequences.
void PromoteIfNeeded(uint32_t id) {
if (!model_->AnyDescriptorHasCodingScheme() &&
model_->id_fallback_strategy() ==
MarkvModel::IdFallbackStrategy::kShortDescriptor) {
// Move-to-front sequences do not share ids. Nothing to do.
return;
}
multi_mtf_.Promote(id);
}
spv_validator_options validator_options_ = nullptr;
const libspirv::AssemblyGrammar grammar_;
MarkvHeader header_;
@ -537,8 +564,19 @@ class MarkvCodecBase {
// List of instructions in the order they are given in the module.
std::vector<std::unique_ptr<const Instruction>> instructions_;
// Container/computer for id descriptors.
IdDescriptorCollection id_descriptors_;
// Container/computer for long (32-bit) id descriptors.
IdDescriptorCollection long_id_descriptors_;
// Container/computer for short id descriptors.
// Short descriptors are stored in uint32_t, but their actual bit width is
// defined with kShortDescriptorNumBits.
// It doesn't seem logical to have a different computer for short id
// descriptors, since one could actually map/truncate long descriptors.
// But as short descriptors have collisions, the efficiency of
// compression depends on the collision pattern, and short descriptors
// produced by function ShortHashU32Array have been empirically proven to
// produce better results.
IdDescriptorCollection short_id_descriptors_;
// Huffman codecs for move-to-front ranks. The map key is mtf handle. Doesn't
// need to contain a different codec for every handle as most use one and the
@ -855,8 +893,11 @@ void MarkvCodecBase::ProcessCurInstruction() {
if (opcode == SpvOpFunction) {
cur_function_id_ = inst_.result_id;
cur_function_return_type_ = inst_.type_id;
multi_mtf_.Insert(GetMtfFunctionWithReturnType(inst_.type_id),
inst_.result_id);
if (model_->id_fallback_strategy() ==
MarkvModel::IdFallbackStrategy::kRuleBased) {
multi_mtf_.Insert(GetMtfFunctionWithReturnType(inst_.type_id),
inst_.result_id);
}
// Store function parameter types in a queue, so that we know which types
// to expect in the following OpFunctionParameter instructions.
@ -886,141 +927,155 @@ void MarkvCodecBase::ProcessCurInstruction() {
// have no type Id, and will map to 0. The result Id for a
// type-generating instruction (e.g. OpTypeInt) maps to itself.
auto insertion_result = id_to_type_id_.emplace(
inst_.result_id,
spvOpcodeGeneratesType(SpvOp(inst_.opcode)) ? inst_.result_id
: inst_.type_id);
inst_.result_id, spvOpcodeGeneratesType(SpvOp(inst_.opcode))
? inst_.result_id
: inst_.type_id);
(void)insertion_result;
assert(insertion_result.second);
}
// Add result_id to MTFs.
switch (opcode) {
case SpvOpTypeFloat:
case SpvOpTypeInt:
case SpvOpTypeBool:
case SpvOpTypeVector:
case SpvOpTypePointer:
case SpvOpExtInstImport:
case SpvOpTypeSampledImage:
case SpvOpTypeImage:
case SpvOpTypeSampler:
multi_mtf_.Insert(GetMtfIdGeneratedByOpcode(opcode), inst_.result_id);
break;
default:
break;
}
if (spvOpcodeIsComposite(opcode)) {
multi_mtf_.Insert(kMtfTypeComposite, inst_.result_id);
}
if (opcode == SpvOpLabel) {
multi_mtf_.InsertOrPromote(kMtfLabel, inst_.result_id);
}
if (opcode == SpvOpTypeInt) {
multi_mtf_.Insert(kMtfTypeScalar, inst_.result_id);
multi_mtf_.Insert(kMtfTypeIntScalarOrVector, inst_.result_id);
}
if (opcode == SpvOpTypeFloat) {
multi_mtf_.Insert(kMtfTypeScalar, inst_.result_id);
multi_mtf_.Insert(kMtfTypeFloatScalarOrVector, inst_.result_id);
}
if (opcode == SpvOpTypeBool) {
multi_mtf_.Insert(kMtfTypeScalar, inst_.result_id);
multi_mtf_.Insert(kMtfTypeBoolScalarOrVector, inst_.result_id);
}
if (opcode == SpvOpTypeVector) {
const uint32_t component_type_id = inst_.words[2];
const uint32_t size = inst_.words[3];
if (multi_mtf_.HasValue(GetMtfIdGeneratedByOpcode(SpvOpTypeFloat),
component_type_id)) {
multi_mtf_.Insert(kMtfTypeFloatScalarOrVector, inst_.result_id);
} else if (multi_mtf_.HasValue(GetMtfIdGeneratedByOpcode(SpvOpTypeInt),
component_type_id)) {
multi_mtf_.Insert(kMtfTypeIntScalarOrVector, inst_.result_id);
} else if (multi_mtf_.HasValue(GetMtfIdGeneratedByOpcode(SpvOpTypeBool),
component_type_id)) {
multi_mtf_.Insert(kMtfTypeBoolScalarOrVector, inst_.result_id);
}
multi_mtf_.Insert(GetMtfTypeVectorOfSize(size), inst_.result_id);
}
if (inst_.opcode == SpvOpTypeFunction) {
const uint32_t return_type = inst_.words[2];
multi_mtf_.Insert(kMtfTypeReturnedByFunction, return_type);
multi_mtf_.Insert(GetMtfFunctionTypeWithReturnType(return_type),
inst_.result_id);
}
if (inst_.type_id) {
const Instruction* type_inst = FindDef(inst_.type_id);
assert(type_inst);
multi_mtf_.Insert(kMtfObject, inst_.result_id);
multi_mtf_.Insert(GetMtfIdOfType(inst_.type_id), inst_.result_id);
if (multi_mtf_.HasValue(kMtfTypeFloatScalarOrVector, inst_.type_id)) {
multi_mtf_.Insert(kMtfFloatScalarOrVector, inst_.result_id);
}
if (multi_mtf_.HasValue(kMtfTypeIntScalarOrVector, inst_.type_id))
multi_mtf_.Insert(kMtfIntScalarOrVector, inst_.result_id);
if (multi_mtf_.HasValue(kMtfTypeBoolScalarOrVector, inst_.type_id))
multi_mtf_.Insert(kMtfBoolScalarOrVector, inst_.result_id);
if (multi_mtf_.HasValue(kMtfTypeComposite, inst_.type_id))
multi_mtf_.Insert(kMtfComposite, inst_.result_id);
switch (type_inst->opcode()) {
if (model_->id_fallback_strategy() ==
MarkvModel::IdFallbackStrategy::kRuleBased) {
switch (opcode) {
case SpvOpTypeFloat:
case SpvOpTypeInt:
case SpvOpTypeBool:
case SpvOpTypePointer:
case SpvOpTypeVector:
case SpvOpTypeImage:
case SpvOpTypePointer:
case SpvOpExtInstImport:
case SpvOpTypeSampledImage:
case SpvOpTypeImage:
case SpvOpTypeSampler:
multi_mtf_.Insert(
GetMtfIdWithTypeGeneratedByOpcode(type_inst->opcode()),
inst_.result_id);
multi_mtf_.Insert(GetMtfIdGeneratedByOpcode(opcode), inst_.result_id);
break;
default:
break;
}
if (type_inst->opcode() == SpvOpTypeVector) {
const uint32_t component_type = type_inst->word(2);
multi_mtf_.Insert(GetMtfVectorOfComponentType(component_type),
if (spvOpcodeIsComposite(opcode)) {
multi_mtf_.Insert(kMtfTypeComposite, inst_.result_id);
}
if (opcode == SpvOpLabel) {
multi_mtf_.InsertOrPromote(kMtfLabel, inst_.result_id);
}
if (opcode == SpvOpTypeInt) {
multi_mtf_.Insert(kMtfTypeScalar, inst_.result_id);
multi_mtf_.Insert(kMtfTypeIntScalarOrVector, inst_.result_id);
}
if (opcode == SpvOpTypeFloat) {
multi_mtf_.Insert(kMtfTypeScalar, inst_.result_id);
multi_mtf_.Insert(kMtfTypeFloatScalarOrVector, inst_.result_id);
}
if (opcode == SpvOpTypeBool) {
multi_mtf_.Insert(kMtfTypeScalar, inst_.result_id);
multi_mtf_.Insert(kMtfTypeBoolScalarOrVector, inst_.result_id);
}
if (opcode == SpvOpTypeVector) {
const uint32_t component_type_id = inst_.words[2];
const uint32_t size = inst_.words[3];
if (multi_mtf_.HasValue(GetMtfIdGeneratedByOpcode(SpvOpTypeFloat),
component_type_id)) {
multi_mtf_.Insert(kMtfTypeFloatScalarOrVector, inst_.result_id);
} else if (multi_mtf_.HasValue(GetMtfIdGeneratedByOpcode(SpvOpTypeInt),
component_type_id)) {
multi_mtf_.Insert(kMtfTypeIntScalarOrVector, inst_.result_id);
} else if (multi_mtf_.HasValue(GetMtfIdGeneratedByOpcode(SpvOpTypeBool),
component_type_id)) {
multi_mtf_.Insert(kMtfTypeBoolScalarOrVector, inst_.result_id);
}
multi_mtf_.Insert(GetMtfTypeVectorOfSize(size), inst_.result_id);
}
if (inst_.opcode == SpvOpTypeFunction) {
const uint32_t return_type = inst_.words[2];
multi_mtf_.Insert(kMtfTypeReturnedByFunction, return_type);
multi_mtf_.Insert(GetMtfFunctionTypeWithReturnType(return_type),
inst_.result_id);
}
if (type_inst->opcode() == SpvOpTypePointer) {
assert(type_inst->operands().size() > 2);
assert(type_inst->words().size() > type_inst->operands()[2].offset);
const uint32_t data_type =
type_inst->word(type_inst->operands()[2].offset);
multi_mtf_.Insert(GetMtfPointerToType(data_type), inst_.result_id);
if (inst_.type_id) {
const Instruction* type_inst = FindDef(inst_.type_id);
assert(type_inst);
if (multi_mtf_.HasValue(kMtfTypeComposite, data_type))
multi_mtf_.Insert(kMtfTypePointerToComposite, inst_.result_id);
multi_mtf_.Insert(kMtfObject, inst_.result_id);
multi_mtf_.Insert(GetMtfIdOfType(inst_.type_id), inst_.result_id);
if (multi_mtf_.HasValue(kMtfTypeFloatScalarOrVector, inst_.type_id)) {
multi_mtf_.Insert(kMtfFloatScalarOrVector, inst_.result_id);
}
if (multi_mtf_.HasValue(kMtfTypeIntScalarOrVector, inst_.type_id))
multi_mtf_.Insert(kMtfIntScalarOrVector, inst_.result_id);
if (multi_mtf_.HasValue(kMtfTypeBoolScalarOrVector, inst_.type_id))
multi_mtf_.Insert(kMtfBoolScalarOrVector, inst_.result_id);
if (multi_mtf_.HasValue(kMtfTypeComposite, inst_.type_id))
multi_mtf_.Insert(kMtfComposite, inst_.result_id);
switch (type_inst->opcode()) {
case SpvOpTypeInt:
case SpvOpTypeBool:
case SpvOpTypePointer:
case SpvOpTypeVector:
case SpvOpTypeImage:
case SpvOpTypeSampledImage:
case SpvOpTypeSampler:
multi_mtf_.Insert(
GetMtfIdWithTypeGeneratedByOpcode(type_inst->opcode()),
inst_.result_id);
break;
default:
break;
}
if (type_inst->opcode() == SpvOpTypeVector) {
const uint32_t component_type = type_inst->word(2);
multi_mtf_.Insert(GetMtfVectorOfComponentType(component_type),
inst_.result_id);
}
if (type_inst->opcode() == SpvOpTypePointer) {
assert(type_inst->operands().size() > 2);
assert(type_inst->words().size() > type_inst->operands()[2].offset);
const uint32_t data_type =
type_inst->word(type_inst->operands()[2].offset);
multi_mtf_.Insert(GetMtfPointerToType(data_type), inst_.result_id);
if (multi_mtf_.HasValue(kMtfTypeComposite, data_type))
multi_mtf_.Insert(kMtfTypePointerToComposite, inst_.result_id);
}
}
if (spvOpcodeGeneratesType(opcode)) {
if (opcode != SpvOpTypeFunction) {
multi_mtf_.Insert(kMtfTypeNonFunction, inst_.result_id);
}
}
}
if (spvOpcodeGeneratesType(opcode)) {
if (opcode != SpvOpTypeFunction) {
multi_mtf_.Insert(kMtfTypeNonFunction, inst_.result_id);
}
if (model_->AnyDescriptorHasCodingScheme()) {
const uint32_t long_descriptor =
long_id_descriptors_.ProcessInstruction(inst_);
if (model_->DescriptorHasCodingScheme(long_descriptor))
multi_mtf_.Insert(GetMtfLongIdDescriptor(long_descriptor),
inst_.result_id);
}
const uint32_t descriptor = id_descriptors_.ProcessInstruction(inst_);
if (model_->DescriptorHasCodingScheme(descriptor))
multi_mtf_.Insert(GetMtfIdDescriptor(descriptor), inst_.result_id);
if (model_->id_fallback_strategy() ==
MarkvModel::IdFallbackStrategy::kShortDescriptor) {
const uint32_t short_descriptor =
short_id_descriptors_.ProcessInstruction(inst_);
multi_mtf_.Insert(GetMtfShortIdDescriptor(short_descriptor),
inst_.result_id);
}
}
uint64_t MarkvCodecBase::GetRuleBasedMtf() {
@ -1052,7 +1107,6 @@ uint64_t MarkvCodecBase::GetRuleBasedMtf() {
case SpvOpFMod:
case SpvOpFNegate: {
if (operand_index_ == 0) return kMtfTypeFloatScalarOrVector;
return GetMtfIdOfType(inst_.type_id);
}
@ -1070,7 +1124,7 @@ uint64_t MarkvCodecBase::GetRuleBasedMtf() {
return kMtfIntScalarOrVector;
}
// TODO(atgoo@github.com) Add OpConvertFToU and other opcodes.
// TODO(atgoo@github.com) Add OpConvertFToU and other opcodes.
case SpvOpFOrdEqual:
case SpvOpFUnordEqual:
@ -1637,57 +1691,87 @@ spv_result_t MarkvDecoder::DecodeMtfRankHuffman(uint64_t mtf,
}
spv_result_t MarkvEncoder::EncodeIdWithDescriptor(uint32_t id) {
// Get the descriptor for id.
const uint32_t long_descriptor = long_id_descriptors_.GetDescriptor(id);
auto* codec =
model_->GetIdDescriptorHuffmanCodec(inst_.opcode, operand_index_);
if (!codec) return SPV_UNSUPPORTED;
uint64_t bits = 0;
size_t num_bits = 0;
// Get the descriptor for id.
const uint32_t descriptor = id_descriptors_.GetDescriptor(id);
if (descriptor && codec->Encode(descriptor, &bits, &num_bits)) {
uint64_t mtf = kMtfNone;
if (long_descriptor && codec &&
codec->Encode(long_descriptor, &bits, &num_bits)) {
// If the descriptor exists and is in the table, write the descriptor and
// proceed to encoding the rank.
writer_.WriteBits(bits, num_bits);
mtf = GetMtfLongIdDescriptor(long_descriptor);
} else {
// The descriptor doesn't exist or we have no coding for it. Write
// kMarkvNoneOfTheAbove and go to fallback method.
if (!codec->Encode(kMarkvNoneOfTheAbove, &bits, &num_bits))
return Diag(SPV_ERROR_INTERNAL)
<< "Descriptor Huffman table for "
<< spvOpcodeString(SpvOp(inst_.opcode)) << " operand index "
<< operand_index_ << " is missing kMarkvNoneOfTheAbove";
if (codec) {
// The descriptor doesn't exist or we have no coding for it. Write
// kMarkvNoneOfTheAbove and go to fallback method.
if (!codec->Encode(kMarkvNoneOfTheAbove, &bits, &num_bits))
return Diag(SPV_ERROR_INTERNAL)
<< "Descriptor Huffman table for "
<< spvOpcodeString(SpvOp(inst_.opcode)) << " operand index "
<< operand_index_ << " is missing kMarkvNoneOfTheAbove";
writer_.WriteBits(bits, num_bits);
return SPV_UNSUPPORTED;
writer_.WriteBits(bits, num_bits);
}
if (model_->id_fallback_strategy() !=
MarkvModel::IdFallbackStrategy::kShortDescriptor) {
return SPV_UNSUPPORTED;
}
const uint32_t short_descriptor = short_id_descriptors_.GetDescriptor(id);
writer_.WriteBits(short_descriptor, kShortDescriptorNumBits);
if (short_descriptor == 0) {
// Forward declared id.
return SPV_UNSUPPORTED;
}
mtf = GetMtfShortIdDescriptor(short_descriptor);
}
// Descriptor has been encoded. Now encode the rank of the id in the
// associated mtf sequence.
const uint64_t mtf = GetMtfIdDescriptor(descriptor);
return EncodeExistingId(mtf, id);
}
spv_result_t MarkvDecoder::DecodeIdWithDescriptor(uint32_t* id) {
auto* codec =
model_->GetIdDescriptorHuffmanCodec(inst_.opcode, operand_index_);
if (!codec) return SPV_UNSUPPORTED;
uint64_t decoded_value = 0;
if (!codec->DecodeFromStream(GetReadBitCallback(), &decoded_value))
return Diag(SPV_ERROR_INTERNAL)
<< "Failed to decode descriptor with Huffman";
uint64_t mtf = kMtfNone;
if (codec) {
uint64_t decoded_value = 0;
if (!codec->DecodeFromStream(GetReadBitCallback(), &decoded_value))
return Diag(SPV_ERROR_INTERNAL)
<< "Failed to decode descriptor with Huffman";
if (decoded_value == kMarkvNoneOfTheAbove) return SPV_UNSUPPORTED;
if (decoded_value != kMarkvNoneOfTheAbove) {
const uint32_t long_descriptor = uint32_t(decoded_value);
mtf = GetMtfLongIdDescriptor(long_descriptor);
}
}
// If descriptor exists then the id was encoded through descriptor mtf.
const uint32_t descriptor = uint32_t(decoded_value);
assert(descriptor == decoded_value);
assert(descriptor);
if (mtf == kMtfNone) {
if (model_->id_fallback_strategy() !=
MarkvModel::IdFallbackStrategy::kShortDescriptor) {
return SPV_UNSUPPORTED;
}
uint64_t decoded_value = 0;
if (!reader_.ReadBits(&decoded_value, kShortDescriptorNumBits))
return Diag(SPV_ERROR_INTERNAL) << "Failed to read short descriptor";
const uint32_t short_descriptor = uint32_t(decoded_value);
if (short_descriptor == 0) {
// Forward declared id.
return SPV_UNSUPPORTED;
}
mtf = GetMtfShortIdDescriptor(short_descriptor);
}
const uint64_t mtf = GetMtfIdDescriptor(descriptor);
return DecodeExistingId(mtf, id);
}
@ -1735,29 +1819,43 @@ spv_result_t MarkvEncoder::EncodeRefId(uint32_t id) {
// If can't be done continue with other methods.
}
// Encode using rule-based mtf.
uint64_t mtf = GetRuleBasedMtf();
const bool can_forward_declare = spvOperandCanBeForwardDeclaredFunction(
SpvOp(inst_.opcode))(operand_index_);
if (mtf != kMtfNone && !can_forward_declare) {
assert(multi_mtf_.HasValue(kMtfAll, id));
return EncodeExistingId(mtf, id);
}
if (mtf == kMtfNone) mtf = kMtfAll;
uint32_t rank = 0;
if (!multi_mtf_.RankFromValue(mtf, id, &rank)) {
// This is the first occurrence of a forward declared id.
multi_mtf_.Insert(kMtfAll, id);
multi_mtf_.Insert(kMtfForwardDeclared, id);
if (mtf != kMtfAll) multi_mtf_.Insert(mtf, id);
rank = 0;
}
if (model_->id_fallback_strategy() ==
MarkvModel::IdFallbackStrategy::kRuleBased) {
// Encode using rule-based mtf.
uint64_t mtf = GetRuleBasedMtf();
return EncodeMtfRankHuffman(rank, mtf, kMtfAll);
if (mtf != kMtfNone && !can_forward_declare) {
assert(multi_mtf_.HasValue(kMtfAll, id));
return EncodeExistingId(mtf, id);
}
if (mtf == kMtfNone) mtf = kMtfAll;
if (!multi_mtf_.RankFromValue(mtf, id, &rank)) {
// This is the first occurrence of a forward declared id.
multi_mtf_.Insert(kMtfAll, id);
multi_mtf_.Insert(kMtfForwardDeclared, id);
if (mtf != kMtfAll) multi_mtf_.Insert(mtf, id);
rank = 0;
}
return EncodeMtfRankHuffman(rank, mtf, kMtfAll);
} else {
assert(can_forward_declare);
if (!multi_mtf_.RankFromValue(kMtfForwardDeclared, id, &rank)) {
// This is the first occurrence of a forward declared id.
multi_mtf_.Insert(kMtfForwardDeclared, id);
rank = 0;
}
writer_.WriteVariableWidthU32(rank, model_->mtf_rank_chunk_length());
return SPV_SUCCESS;
}
}
spv_result_t MarkvDecoder::DecodeRefId(uint32_t* id) {
@ -1766,37 +1864,52 @@ spv_result_t MarkvDecoder::DecodeRefId(uint32_t* id) {
if (result != SPV_UNSUPPORTED) return result;
}
uint64_t mtf = GetRuleBasedMtf();
const bool can_forward_declare = spvOperandCanBeForwardDeclaredFunction(
SpvOp(inst_.opcode))(operand_index_);
if (mtf != kMtfNone && !can_forward_declare) {
return DecodeExistingId(mtf, id);
}
if (mtf == kMtfNone) mtf = kMtfAll;
uint32_t rank = 0;
*id = 0;
uint32_t rank = 0;
if (model_->id_fallback_strategy() ==
MarkvModel::IdFallbackStrategy::kRuleBased) {
uint64_t mtf = GetRuleBasedMtf();
if (mtf != kMtfNone && !can_forward_declare) {
return DecodeExistingId(mtf, id);
}
{
const spv_result_t result = DecodeMtfRankHuffman(mtf, kMtfAll, &rank);
if (result != SPV_SUCCESS) return result;
}
if (mtf == kMtfNone) mtf = kMtfAll;
{
const spv_result_t result = DecodeMtfRankHuffman(mtf, kMtfAll, &rank);
if (result != SPV_SUCCESS) return result;
}
if (rank == 0) {
// This is the first occurrence of a forward declared id.
*id = GetIdBound();
SetIdBound(*id + 1);
multi_mtf_.Insert(kMtfAll, *id);
multi_mtf_.Insert(kMtfForwardDeclared, *id);
if (mtf != kMtfAll) multi_mtf_.Insert(mtf, *id);
if (rank == 0) {
// This is the first occurrence of a forward declared id.
*id = GetIdBound();
SetIdBound(*id + 1);
multi_mtf_.Insert(kMtfAll, *id);
multi_mtf_.Insert(kMtfForwardDeclared, *id);
if (mtf != kMtfAll) multi_mtf_.Insert(mtf, *id);
} else {
if (!multi_mtf_.ValueFromRank(mtf, rank, id))
return Diag(SPV_ERROR_INTERNAL) << "MTF rank out of bounds";
}
} else {
if (!multi_mtf_.ValueFromRank(mtf, rank, id))
return Diag(SPV_ERROR_INTERNAL) << "MTF rank out of bounds";
}
assert(can_forward_declare);
if (!reader_.ReadVariableWidthU32(&rank, model_->mtf_rank_chunk_length()))
return Diag(SPV_ERROR_INTERNAL)
<< "Failed to decode MTF rank with varint";
if (rank == 0) {
// This is the first occurrence of a forward declared id.
*id = GetIdBound();
SetIdBound(*id + 1);
multi_mtf_.Insert(kMtfForwardDeclared, *id);
} else {
if (!multi_mtf_.ValueFromRank(kMtfForwardDeclared, rank, id))
return Diag(SPV_ERROR_INTERNAL) << "MTF rank out of bounds";
}
}
assert(*id);
return SPV_SUCCESS;
}
@ -1816,6 +1929,9 @@ spv_result_t MarkvEncoder::EncodeTypeId() {
// If can't be done continue with other methods.
}
assert(model_->id_fallback_strategy() ==
MarkvModel::IdFallbackStrategy::kRuleBased);
uint64_t mtf = GetRuleBasedMtf();
assert(!spvOperandCanBeForwardDeclaredFunction(SpvOp(inst_.opcode))(
operand_index_));
@ -1842,6 +1958,9 @@ spv_result_t MarkvDecoder::DecodeTypeId() {
if (result != SPV_UNSUPPORTED) return result;
}
assert(model_->id_fallback_strategy() ==
MarkvModel::IdFallbackStrategy::kRuleBased);
uint64_t mtf = GetRuleBasedMtf();
assert(!spvOperandCanBeForwardDeclaredFunction(SpvOp(inst_.opcode))(
operand_index_));
@ -1878,8 +1997,11 @@ spv_result_t MarkvEncoder::EncodeResultId() {
}
}
if (!rank) {
multi_mtf_.Insert(kMtfAll, inst_.result_id);
if (model_->id_fallback_strategy() ==
MarkvModel::IdFallbackStrategy::kRuleBased) {
if (!rank) {
multi_mtf_.Insert(kMtfAll, inst_.result_id);
}
}
return SPV_SUCCESS;
@ -1924,8 +2046,11 @@ spv_result_t MarkvDecoder::DecodeResultId() {
SetIdBound(inst_.result_id + 1);
}
if (!rank) {
multi_mtf_.Insert(kMtfAll, inst_.result_id);
if (model_->id_fallback_strategy() ==
MarkvModel::IdFallbackStrategy::kRuleBased) {
if (!rank) {
multi_mtf_.Insert(kMtfAll, inst_.result_id);
}
}
return SPV_SUCCESS;
@ -2078,7 +2203,7 @@ spv_result_t MarkvEncoder::EncodeInstruction(
if (result != SPV_SUCCESS) return result;
}
multi_mtf_.Promote(id);
PromoteIfNeeded(id);
break;
}
@ -2248,7 +2373,7 @@ spv_result_t MarkvDecoder::DecodeOperand(
inst_words_.push_back(inst_.result_id);
SetIdBound(std::max(GetIdBound(), inst_.result_id + 1));
multi_mtf_.Promote(inst_.result_id);
PromoteIfNeeded(inst_.result_id);
break;
}
@ -2258,7 +2383,7 @@ spv_result_t MarkvDecoder::DecodeOperand(
inst_words_.push_back(inst_.type_id);
SetIdBound(std::max(GetIdBound(), inst_.type_id + 1));
multi_mtf_.Promote(inst_.type_id);
PromoteIfNeeded(inst_.type_id);
break;
}
@ -2291,7 +2416,7 @@ spv_result_t MarkvDecoder::DecodeOperand(
inst_words_.push_back(id);
SetIdBound(std::max(GetIdBound(), id + 1));
multi_mtf_.Promote(id);
PromoteIfNeeded(id);
break;
}
@ -2793,4 +2918,4 @@ spv_result_t MarkvToSpirv(
return SPV_SUCCESS;
}
} // namespave spvtools
} // namespace spvtools

View File

@ -32,7 +32,46 @@ class MarkvModel {
public:
MarkvModel()
: operand_chunk_lengths_(
static_cast<size_t>(SPV_OPERAND_TYPE_NUM_OPERAND_TYPES), 0) {}
static_cast<size_t>(SPV_OPERAND_TYPE_NUM_OPERAND_TYPES), 0) {
// Set default values.
operand_chunk_lengths_[SPV_OPERAND_TYPE_TYPE_ID] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_RESULT_ID] = 8;
operand_chunk_lengths_[SPV_OPERAND_TYPE_ID] = 8;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SCOPE_ID] = 8;
operand_chunk_lengths_[SPV_OPERAND_TYPE_MEMORY_SEMANTICS_ID] = 8;
operand_chunk_lengths_[SPV_OPERAND_TYPE_LITERAL_INTEGER] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_LITERAL_INTEGER] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_CAPABILITY] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SOURCE_LANGUAGE] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_EXECUTION_MODEL] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_ADDRESSING_MODEL] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_MEMORY_MODEL] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_EXECUTION_MODE] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_STORAGE_CLASS] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_DIMENSIONALITY] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SAMPLER_ADDRESSING_MODE] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SAMPLER_FILTER_MODE] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SAMPLER_IMAGE_FORMAT] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_FP_ROUNDING_MODE] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_LINKAGE_TYPE] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_ACCESS_QUALIFIER] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_ACCESS_QUALIFIER] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_FUNCTION_PARAMETER_ATTRIBUTE] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_DECORATION] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_BUILT_IN] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_GROUP_OPERATION] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_KERNEL_ENQ_FLAGS] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_KERNEL_PROFILING_INFO] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_FP_FAST_MATH_MODE] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_FUNCTION_CONTROL] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_LOOP_CONTROL] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_IMAGE] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_IMAGE] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_MEMORY_ACCESS] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SELECTION_CONTROL] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_EXTENSION_INSTRUCTION_NUMBER] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_TYPED_LITERAL_NUMBER] = 6;
}
uint32_t model_type() const { return model_type_; }
uint32_t model_version() const { return model_version_; }
@ -47,6 +86,15 @@ class MarkvModel {
uint32_t s64_chunk_length() const { return s64_chunk_length_; }
uint32_t s64_block_exponent() const { return s64_block_exponent_; }
enum class IdFallbackStrategy {
kRuleBased = 0,
kShortDescriptor,
};
IdFallbackStrategy id_fallback_strategy() const {
return id_fallback_strategy_;
}
// Returns a codec for common opcode_and_num_operands words for the given
// previous opcode. May return nullptr if the codec doesn't exist.
const spvutils::HuffmanCodec<uint64_t>*
@ -99,6 +147,11 @@ class MarkvModel {
return descriptors_with_coding_scheme_.count(descriptor);
}
// Checks if any descriptor has a coding scheme.
bool AnyDescriptorHasCodingScheme() const {
return !descriptors_with_coding_scheme_.empty();
}
// Returns chunk length used for variable length encoding of spirv operand
// words.
uint32_t GetOperandVariableWidthChunkLength(spv_operand_type_t type) const {
@ -167,6 +220,8 @@ class MarkvModel {
uint32_t s64_chunk_length_ = 8;
uint32_t s64_block_exponent_ = 10;
IdFallbackStrategy id_fallback_strategy_ = IdFallbackStrategy::kShortDescriptor;
uint32_t model_type_ = 0;
uint32_t model_version_ = 0;
};

View File

@ -63,7 +63,10 @@ uint32_t IdDescriptorCollection::ProcessInstruction(
}
}
const uint32_t descriptor = HashU32Array(words_);
uint32_t descriptor = custom_hash_func_ ?
custom_hash_func_(words_) : HashU32Array(words_);
if (descriptor == 0)
descriptor = 1;
assert(descriptor);
words_.clear();

View File

@ -22,13 +22,19 @@
namespace libspirv {
using CustomHashFunc = std::function<uint32_t(const std::vector<uint32_t>&)>;
// Computes and stores id descriptors.
//
// Descriptors are computed as hash of all words in the instruction where ids
// were substituted with previously computed descriptors.
class IdDescriptorCollection {
public:
IdDescriptorCollection() { words_.reserve(16); }
explicit IdDescriptorCollection(
CustomHashFunc custom_hash_func = CustomHashFunc())
: custom_hash_func_(custom_hash_func) {
words_.reserve(16);
}
// Computes descriptor for the result id of the given instruction and
// registers it in id_to_descriptor_. Returns the computed descriptor.
@ -46,6 +52,8 @@ class IdDescriptorCollection {
private:
std::unordered_map<uint32_t, uint32_t> id_to_descriptor_;
std::function<uint32_t(const std::vector<uint32_t>&)> custom_hash_func_;
// Scratch buffer used for hashing. Class member to optimize on allocation.
std::vector<uint32_t> words_;
};

View File

@ -22,7 +22,7 @@ if(SPIRV_BUILD_COMPRESSION)
SRCS
markv_codec_test.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../tools/comp/markv_model_factory.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../tools/comp/markv_model_shader_default.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../tools/comp/markv_model_shader.cpp
${VAL_TEST_COMMON_SRCS}
LIBS SPIRV-Tools-comp ${SPIRV_TOOLS}
)

View File

@ -26,7 +26,9 @@
namespace {
using spvtools::MarkvModelType;
using spvtest::ScopedContext;
using MarkvTest = ::testing::TestWithParam<MarkvModelType>;
void DiagnosticsMessageHandler(spv_message_level_t level, const char*,
const spv_position_t& position,
@ -85,10 +87,11 @@ void Disassemble(const std::vector<uint32_t>& words,
// Encodes/decodes |original|, assembles/dissasembles |original|, then compares
// the results of the two operations.
void TestEncodeDecode(const std::string& original_text) {
void TestEncodeDecode(MarkvModelType model_type,
const std::string& original_text) {
ScopedContext ctx(SPV_ENV_UNIVERSAL_1_2);
std::unique_ptr<spvtools::MarkvModel> model =
spvtools::CreateMarkvModel(spvtools::kMarkvModelShaderDefault);
spvtools::CreateMarkvModel(model_type);
spvtools::MarkvCodecOptions options;
std::vector<uint32_t> expected_binary;
@ -133,7 +136,8 @@ void TestEncodeDecode(const std::string& original_text) {
EXPECT_EQ(expected_text, decoded_text) << encoder_comments.str();
}
void TestEncodeDecodeShaderMainBody(const std::string& body) {
void TestEncodeDecodeShaderMainBody(MarkvModelType model_type,
const std::string& body) {
const std::string prefix =
R"(
OpCapability Shader
@ -216,11 +220,11 @@ R"(
OpReturn
OpFunctionEnd)";
TestEncodeDecode(prefix + body + suffix);
TestEncodeDecode(model_type, prefix + body + suffix);
}
TEST(Markv, U32Literal) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, U32Literal) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
OpCapability Linkage
OpMemoryModel Logical GLSL450
@ -231,8 +235,8 @@ OpMemoryModel Logical GLSL450
)");
}
TEST(Markv, S32Literal) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, S32Literal) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
OpCapability Linkage
OpMemoryModel Logical GLSL450
@ -245,8 +249,8 @@ OpMemoryModel Logical GLSL450
)");
}
TEST(Markv, U64Literal) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, U64Literal) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
OpCapability Linkage
OpCapability Int64
@ -258,8 +262,8 @@ OpMemoryModel Logical GLSL450
)");
}
TEST(Markv, S64Literal) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, S64Literal) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
OpCapability Linkage
OpCapability Int64
@ -273,8 +277,8 @@ OpMemoryModel Logical GLSL450
)");
}
TEST(Markv, U16Literal) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, U16Literal) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
OpCapability Linkage
OpCapability Int16
@ -286,8 +290,8 @@ OpMemoryModel Logical GLSL450
)");
}
TEST(Markv, S16Literal) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, S16Literal) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
OpCapability Linkage
OpCapability Int16
@ -301,8 +305,8 @@ OpMemoryModel Logical GLSL450
)");
}
TEST(Markv, F32Literal) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, F32Literal) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
OpCapability Linkage
OpMemoryModel Logical GLSL450
@ -314,8 +318,8 @@ OpMemoryModel Logical GLSL450
)");
}
TEST(Markv, F64Literal) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, F64Literal) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
OpCapability Linkage
OpCapability Float64
@ -328,8 +332,8 @@ OpMemoryModel Logical GLSL450
)");
}
TEST(Markv, F16Literal) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, F16Literal) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
OpCapability Linkage
OpCapability Float16
@ -342,8 +346,8 @@ OpMemoryModel Logical GLSL450
)");
}
TEST(Markv, StringLiteral) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, StringLiteral) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
OpCapability Linkage
OpExtension "SPV_KHR_16bit_storage"
@ -354,8 +358,8 @@ OpMemoryModel Logical GLSL450
)");
}
TEST(Markv, WithFunction) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, WithFunction) {
TestEncodeDecode(GetParam(), R"(
OpCapability Addresses
OpCapability Kernel
OpCapability GenericPointer
@ -376,8 +380,8 @@ OpFunctionEnd
)");
}
TEST(Markv, WithMultipleFunctions) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, WithMultipleFunctions) {
TestEncodeDecode(GetParam(), R"(
OpCapability Addresses
OpCapability Kernel
OpCapability GenericPointer
@ -409,8 +413,8 @@ OpFunctionEnd
)");
}
TEST(Markv, ForwardDeclaredId) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, ForwardDeclaredId) {
TestEncodeDecode(GetParam(), R"(
OpCapability Addresses
OpCapability Kernel
OpCapability GenericPointer
@ -430,8 +434,8 @@ OpFunctionEnd
)");
}
TEST(Markv, WithSwitch) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, WithSwitch) {
TestEncodeDecode(GetParam(), R"(
OpCapability Addresses
OpCapability Kernel
OpCapability GenericPointer
@ -460,8 +464,8 @@ OpFunctionEnd
)");
}
TEST(Markv, WithLoop) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, WithLoop) {
TestEncodeDecode(GetParam(), R"(
OpCapability Addresses
OpCapability Kernel
OpCapability GenericPointer
@ -485,8 +489,8 @@ OpFunctionEnd
)");
}
TEST(Markv, WithDecorate) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, WithDecorate) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
OpCapability Linkage
OpMemoryModel Logical GLSL450
@ -497,8 +501,8 @@ OpDecorate %1 Uniform
)");
}
TEST(Markv, WithExtInst) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, WithExtInst) {
TestEncodeDecode(GetParam(), R"(
OpCapability Addresses
OpCapability Kernel
OpCapability GenericPointer
@ -517,8 +521,8 @@ OpFunctionEnd
)");
}
TEST(Markv, F32Mul) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, F32Mul) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%val1 = OpFMul %f32 %f32_0 %f32_1
%val2 = OpFMul %f32 %f32_2 %f32_0
%val3 = OpFMul %f32 %f32_pi %f32_2
@ -526,8 +530,8 @@ TEST(Markv, F32Mul) {
)");
}
TEST(Markv, U32Mul) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, U32Mul) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%val1 = OpIMul %u32 %u32_0 %u32_1
%val2 = OpIMul %u32 %u32_2 %u32_0
%val3 = OpIMul %u32 %u32_3 %u32_2
@ -535,8 +539,8 @@ TEST(Markv, U32Mul) {
)");
}
TEST(Markv, S32Mul) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, S32Mul) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%val1 = OpIMul %s32 %s32_0 %s32_1
%val2 = OpIMul %s32 %s32_2 %s32_0
%val3 = OpIMul %s32 %s32_m1 %s32_2
@ -544,8 +548,8 @@ TEST(Markv, S32Mul) {
)");
}
TEST(Markv, F32Add) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, F32Add) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%val1 = OpFAdd %f32 %f32_0 %f32_1
%val2 = OpFAdd %f32 %f32_2 %f32_0
%val3 = OpFAdd %f32 %f32_pi %f32_2
@ -553,8 +557,8 @@ TEST(Markv, F32Add) {
)");
}
TEST(Markv, U32Add) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, U32Add) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%val1 = OpIAdd %u32 %u32_0 %u32_1
%val2 = OpIAdd %u32 %u32_2 %u32_0
%val3 = OpIAdd %u32 %u32_3 %u32_2
@ -562,8 +566,8 @@ TEST(Markv, U32Add) {
)");
}
TEST(Markv, S32Add) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, S32Add) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%val1 = OpIAdd %s32 %s32_0 %s32_1
%val2 = OpIAdd %s32 %s32_2 %s32_0
%val3 = OpIAdd %s32 %s32_m1 %s32_2
@ -571,8 +575,8 @@ TEST(Markv, S32Add) {
)");
}
TEST(Markv, F32Dot) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, F32Dot) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%dot2_1 = OpDot %f32 %f32vec2_01 %f32vec2_12
%dot2_2 = OpDot %f32 %f32vec2_01 %f32vec2_01
%dot2_3 = OpDot %f32 %f32vec2_12 %f32vec2_12
@ -585,8 +589,8 @@ TEST(Markv, F32Dot) {
)");
}
TEST(Markv, F32VectorCompositeConstruct) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, F32VectorCompositeConstruct) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%cc1 = OpCompositeConstruct %f32vec4 %f32vec2_01 %f32vec2_12
%cc2 = OpCompositeConstruct %f32vec3 %f32vec2_01 %f32_2
%cc3 = OpCompositeConstruct %f32vec2 %f32_1 %f32_2
@ -594,8 +598,8 @@ TEST(Markv, F32VectorCompositeConstruct) {
)");
}
TEST(Markv, U32VectorCompositeConstruct) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, U32VectorCompositeConstruct) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%cc1 = OpCompositeConstruct %u32vec4 %u32vec2_01 %u32vec2_12
%cc2 = OpCompositeConstruct %u32vec3 %u32vec2_01 %u32_2
%cc3 = OpCompositeConstruct %u32vec2 %u32_1 %u32_2
@ -603,8 +607,8 @@ TEST(Markv, U32VectorCompositeConstruct) {
)");
}
TEST(Markv, S32VectorCompositeConstruct) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, S32VectorCompositeConstruct) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%cc1 = OpCompositeConstruct %u32vec4 %u32vec2_01 %u32vec2_12
%cc2 = OpCompositeConstruct %u32vec3 %u32vec2_01 %u32_2
%cc3 = OpCompositeConstruct %u32vec2 %u32_1 %u32_2
@ -612,15 +616,15 @@ TEST(Markv, S32VectorCompositeConstruct) {
)");
}
TEST(Markv, F32VectorCompositeExtract) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, F32VectorCompositeExtract) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%f32vec4_3210 = OpCompositeConstruct %f32vec4 %f32_3 %f32_2 %f32_1 %f32_0
%f32vec3_013 = OpCompositeExtract %f32vec3 %f32vec4_0123 0 1 3
)");
}
TEST(Markv, F32VectorComparison) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, F32VectorComparison) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%f32vec4_3210 = OpCompositeConstruct %f32vec4 %f32_3 %f32_2 %f32_1 %f32_0
%c1 = OpFOrdEqual %boolvec4 %f32vec4_0123 %f32vec4_3210
%c2 = OpFUnordEqual %boolvec4 %f32vec4_0123 %f32vec4_3210
@ -637,24 +641,24 @@ TEST(Markv, F32VectorComparison) {
)");
}
TEST(Markv, VectorShuffle) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, VectorShuffle) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%f32vec4_3210 = OpCompositeConstruct %f32vec4 %f32_3 %f32_2 %f32_1 %f32_0
%sh1 = OpVectorShuffle %f32vec2 %f32vec4_0123 %f32vec4_3210 3 6
%sh2 = OpVectorShuffle %f32vec3 %f32vec2_01 %f32vec4_3210 0 3 4
)");
}
TEST(Markv, VectorTimesScalar) {
TestEncodeDecodeShaderMainBody(R"(
TEST_P(MarkvTest, VectorTimesScalar) {
TestEncodeDecodeShaderMainBody(GetParam(), R"(
%f32vec4_3210 = OpCompositeConstruct %f32vec4 %f32_3 %f32_2 %f32_1 %f32_0
%res1 = OpVectorTimesScalar %f32vec4 %f32vec4_0123 %f32_2
%res2 = OpVectorTimesScalar %f32vec4 %f32vec4_3210 %f32_2
)");
}
TEST(Markv, SpirvSpecSample) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, SpirvSpecSample) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
%1 = OpExtInstImport "GLSL.std.450"
OpMemoryModel Logical GLSL450
@ -775,8 +779,8 @@ TEST(Markv, SpirvSpecSample) {
)");
}
TEST(Markv, SampleFromDeadBranchEliminationTest) {
TestEncodeDecode(R"(
TEST_P(MarkvTest, SampleFromDeadBranchEliminationTest) {
TestEncodeDecode(GetParam(), R"(
OpCapability Shader
%1 = OpExtInstImport "GLSL.std.450"
OpMemoryModel Logical GLSL450
@ -815,4 +819,12 @@ OpFunctionEnd
)");
}
INSTANTIATE_TEST_CASE_P(
AllMarkvModels, MarkvTest,
::testing::ValuesIn(std::vector<MarkvModelType>{
spvtools::kMarkvModelShaderLite,
spvtools::kMarkvModelShaderMid,
spvtools::kMarkvModelShaderMax,
}),);
} // namespace

View File

@ -64,7 +64,7 @@ if (NOT ${SPIRV_SKIP_EXECUTABLES})
add_spvtools_tool(TARGET spirv-markv
SRCS comp/markv.cpp
comp/markv_model_factory.cpp
comp/markv_model_shader_default.cpp
comp/markv_model_shader.cpp
LIBS SPIRV-Tools-comp SPIRV-Tools-opt ${SPIRV_TOOLS})
target_include_directories(spirv-markv PRIVATE ${spirv-tools_SOURCE_DIR}
${SPIRV_HEADER_INCLUDE_DIR})

View File

@ -72,6 +72,12 @@ Options:
--comments Write codec comments to stderr.
--version Display MARK-V codec version.
--validate Validate SPIR-V while encoding or decoding.
--model=<model-name>
Compression model, possible values:
shader_lite - fast, poor compression ratio
shader_mid - balanced
shader_max - best compression ratio
Default: shader_lite
-o <filename> Set the output filename.
Output goes to standard output if this option is
@ -132,6 +138,8 @@ int main(int argc, char** argv) {
bool want_comments = false;
bool validate_spirv_binary = false;
spvtools::MarkvModelType model_type = spvtools::kMarkvModelUnknown;
for (int argi = 2; argi < argc; ++argi) {
if ('-' == argv[argi][0]) {
switch (argv[argi][1]) {
@ -158,6 +166,18 @@ int main(int argc, char** argv) {
return 1;
} else if (0 == strcmp(argv[argi], "--validate")) {
validate_spirv_binary = true;
} else if (0 == strcmp(argv[argi], "--model=shader_lite")) {
if (model_type != spvtools::kMarkvModelUnknown)
fprintf(stderr, "error: More than one model specified\n");
model_type = spvtools::kMarkvModelShaderLite;
} else if (0 == strcmp(argv[argi], "--model=shader_mid")) {
if (model_type != spvtools::kMarkvModelUnknown)
fprintf(stderr, "error: More than one model specified\n");
model_type = spvtools::kMarkvModelShaderMid;
} else if (0 == strcmp(argv[argi], "--model=shader_max")) {
if (model_type != spvtools::kMarkvModelUnknown)
fprintf(stderr, "error: More than one model specified\n");
model_type = spvtools::kMarkvModelShaderMax;
} else {
print_usage(argv[0]);
return 1;
@ -186,6 +206,9 @@ int main(int argc, char** argv) {
}
}
if (model_type == spvtools::kMarkvModelUnknown)
model_type = spvtools::kMarkvModelShaderLite;
const auto no_comments = spvtools::MarkvLogConsumer();
const auto output_to_stderr = [](const std::string& str) {
std::cerr << str;
@ -194,7 +217,7 @@ int main(int argc, char** argv) {
ScopedContext ctx(kSpvEnv);
std::unique_ptr<spvtools::MarkvModel> model =
spvtools::CreateMarkvModel(spvtools::kMarkvModelShaderDefault);
spvtools::CreateMarkvModel(model_type);
std::vector<uint32_t> spirv;
std::vector<uint8_t> markv;

View File

@ -13,17 +13,30 @@
// limitations under the License.
#include "markv_model_factory.h"
#include "markv_model_shader_default.h"
#include "markv_model_shader.h"
namespace spvtools {
std::unique_ptr<MarkvModel> CreateMarkvModel(MarkvModelType type) {
std::unique_ptr<MarkvModel> model;
switch (type) {
case kMarkvModelShaderDefault: {
model.reset(new MarkvModelShaderDefault());
case kMarkvModelShaderLite: {
model.reset(new MarkvModelShaderLite());
break;
}
case kMarkvModelShaderMid: {
model.reset(new MarkvModelShaderMid());
break;
}
case kMarkvModelShaderMax: {
model.reset(new MarkvModelShaderMax());
break;
}
case kMarkvModelUnknown: {
assert(0 && "kMarkvModelUnknown supplied to CreateMarkvModel");
return model;
}
}
model->SetModelType(static_cast<uint32_t>(type));

View File

@ -22,7 +22,10 @@
namespace spvtools {
enum MarkvModelType {
kMarkvModelShaderDefault = 1,
kMarkvModelUnknown = 0,
kMarkvModelShaderLite,
kMarkvModelShaderMid,
kMarkvModelShaderMax,
};
std::unique_ptr<MarkvModel> CreateMarkvModel(MarkvModelType type);

View File

@ -0,0 +1,100 @@
// Copyright (c) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "markv_model_shader.h"
#include <algorithm>
#include <map>
#include <memory>
#include <unordered_map>
#include <unordered_set>
#include <vector>
using spvutils::HuffmanCodec;
namespace spvtools {
namespace {
// Signals that the value is not in the coding scheme and a fallback method
// needs to be used.
const uint64_t kMarkvNoneOfTheAbove = MarkvModel::GetMarkvNoneOfTheAbove();
inline uint32_t CombineOpcodeAndNumOperands(uint32_t opcode,
uint32_t num_operands) {
return opcode | (num_operands << 16);
}
// The following file contains autogenerated statistical coding rules.
// Can be generated by running spirv-stats on representative corpus of shaders
// with flags:
// --codegen_opcode_and_num_operands_hist
// --codegen_opcode_and_num_operands_markov_huffman_codecs
// --codegen_literal_string_huffman_codecs
// --codegen_non_id_word_huffman_codecs
// --codegen_id_descriptor_huffman_codecs
//
// Example:
// find <SHADER_CORPUS_DIR> -type f -print0 | xargs -0 -s 2000000
// ~/SPIRV-Tools/build/tools/spirv-stats -v
// --codegen_opcode_and_num_operands_hist
// --codegen_opcode_and_num_operands_markov_huffman_codecs
// --codegen_literal_string_huffman_codecs --codegen_non_id_word_huffman_codecs
// --codegen_id_descriptor_huffman_codecs -o
// ~/SPIRV-Tools/source/comp/markv_autogen.inc
#include "markv_model_shader_default_autogen.inc"
} // namespace
MarkvModelShaderLite::MarkvModelShaderLite() {
const uint16_t kVersionNumber = 1;
SetModelVersion(kVersionNumber);
opcode_and_num_operands_huffman_codec_.reset(
new HuffmanCodec<uint64_t>(GetOpcodeAndNumOperandsHist()));
id_fallback_strategy_ = IdFallbackStrategy::kShortDescriptor;
}
MarkvModelShaderMid::MarkvModelShaderMid() {
const uint16_t kVersionNumber = 1;
SetModelVersion(kVersionNumber);
opcode_and_num_operands_huffman_codec_.reset(
new HuffmanCodec<uint64_t>(GetOpcodeAndNumOperandsHist()));
non_id_word_huffman_codecs_ = GetNonIdWordHuffmanCodecs();
id_descriptor_huffman_codecs_ = GetIdDescriptorHuffmanCodecs();
descriptors_with_coding_scheme_ = GetDescriptorsWithCodingScheme();
literal_string_huffman_codecs_ = GetLiteralStringHuffmanCodecs();
id_fallback_strategy_ = IdFallbackStrategy::kShortDescriptor;
}
MarkvModelShaderMax::MarkvModelShaderMax() {
const uint16_t kVersionNumber = 1;
SetModelVersion(kVersionNumber);
opcode_and_num_operands_huffman_codec_.reset(
new HuffmanCodec<uint64_t>(GetOpcodeAndNumOperandsHist()));
opcode_and_num_operands_markov_huffman_codecs_ =
GetOpcodeAndNumOperandsMarkovHuffmanCodecs();
non_id_word_huffman_codecs_ = GetNonIdWordHuffmanCodecs();
id_descriptor_huffman_codecs_ = GetIdDescriptorHuffmanCodecs();
descriptors_with_coding_scheme_ = GetDescriptorsWithCodingScheme();
literal_string_huffman_codecs_ = GetLiteralStringHuffmanCodecs();
id_fallback_strategy_ = IdFallbackStrategy::kRuleBased;
}
} // namespace spvtools

View File

@ -0,0 +1,45 @@
// Copyright (c) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef SPIRV_TOOLS_MARKV_MODEL_SHADER_H_
#define SPIRV_TOOLS_MARKV_MODEL_SHADER_H_
#include "source/comp/markv_model.h"
namespace spvtools {
// MARK-V shader compression model, which only uses fast and lightweight
// algorithms, which do not require training and are not heavily dependent on
// SPIR-V grammar. Compression ratio is worse than by other models.
class MarkvModelShaderLite : public MarkvModel {
public:
MarkvModelShaderLite();
};
// MARK-V shader compression model with balanced compression ratio and runtime
// performance.
class MarkvModelShaderMid : public MarkvModel {
public:
MarkvModelShaderMid();
};
// MARK-V shader compression model designed for maximum compression.
class MarkvModelShaderMax : public MarkvModel {
public:
MarkvModelShaderMax();
};
} // namespace spvtools
#endif // SPIRV_TOOLS_MARKV_MODEL_SHADER_H_

View File

@ -1,112 +0,0 @@
// Copyright (c) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "markv_model_shader_default.h"
#include <algorithm>
#include <map>
#include <memory>
#include <unordered_map>
#include <unordered_set>
#include <vector>
using spvutils::HuffmanCodec;
namespace spvtools {
namespace {
// Signals that the value is not in the coding scheme and a fallback method
// needs to be used.
const uint64_t kMarkvNoneOfTheAbove = MarkvModel::GetMarkvNoneOfTheAbove();
inline uint32_t CombineOpcodeAndNumOperands(uint32_t opcode,
uint32_t num_operands) {
return opcode | (num_operands << 16);
}
// The following file contains autogenerated statistical coding rules.
// Generated by running spirv-stats on representative corpus of shaders with
// flags:
// --codegen_opcode_and_num_operands_hist
// --codegen_opcode_and_num_operands_markov_huffman_codecs
// --codegen_literal_string_huffman_codecs
// --codegen_non_id_word_huffman_codecs
// --codegen_id_descriptor_huffman_codecs
//
// Example:
// find <SHADER_CORPUS_DIR> -type f -print0 | xargs -0 -s 2000000
// ~/SPIRV-Tools/build/tools/spirv-stats -v
// --codegen_opcode_and_num_operands_hist
// --codegen_opcode_and_num_operands_markov_huffman_codecs
// --codegen_literal_string_huffman_codecs --codegen_non_id_word_huffman_codecs
// --codegen_id_descriptor_huffman_codecs -o
// ~/SPIRV-Tools/source/comp/markv_autogen.inc
#include "markv_model_shader_default_autogen.inc"
} // namespace
MarkvModelShaderDefault::MarkvModelShaderDefault() {
const uint16_t kVersionNumber = 0;
SetModelVersion(kVersionNumber);
opcode_and_num_operands_huffman_codec_.reset(
new HuffmanCodec<uint64_t>(GetOpcodeAndNumOperandsHist()));
opcode_and_num_operands_markov_huffman_codecs_ =
GetOpcodeAndNumOperandsMarkovHuffmanCodecs();
non_id_word_huffman_codecs_ = GetNonIdWordHuffmanCodecs();
id_descriptor_huffman_codecs_ = GetIdDescriptorHuffmanCodecs();
descriptors_with_coding_scheme_ = GetDescriptorsWithCodingScheme();
literal_string_huffman_codecs_ = GetLiteralStringHuffmanCodecs();
operand_chunk_lengths_[SPV_OPERAND_TYPE_TYPE_ID] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_RESULT_ID] = 8;
operand_chunk_lengths_[SPV_OPERAND_TYPE_ID] = 8;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SCOPE_ID] = 8;
operand_chunk_lengths_[SPV_OPERAND_TYPE_MEMORY_SEMANTICS_ID] = 8;
operand_chunk_lengths_[SPV_OPERAND_TYPE_LITERAL_INTEGER] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_LITERAL_INTEGER] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_CAPABILITY] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SOURCE_LANGUAGE] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_EXECUTION_MODEL] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_ADDRESSING_MODEL] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_MEMORY_MODEL] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_EXECUTION_MODE] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_STORAGE_CLASS] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_DIMENSIONALITY] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SAMPLER_ADDRESSING_MODE] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SAMPLER_FILTER_MODE] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SAMPLER_IMAGE_FORMAT] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_FP_ROUNDING_MODE] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_LINKAGE_TYPE] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_ACCESS_QUALIFIER] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_ACCESS_QUALIFIER] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_FUNCTION_PARAMETER_ATTRIBUTE] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_DECORATION] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_BUILT_IN] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_GROUP_OPERATION] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_KERNEL_ENQ_FLAGS] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_KERNEL_PROFILING_INFO] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_FP_FAST_MATH_MODE] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_FUNCTION_CONTROL] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_LOOP_CONTROL] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_IMAGE] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_IMAGE] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_MEMORY_ACCESS] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SELECTION_CONTROL] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_EXTENSION_INSTRUCTION_NUMBER] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_TYPED_LITERAL_NUMBER] = 6;
}
} // namespace spvtools

View File

@ -1,30 +0,0 @@
// Copyright (c) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef SPIRV_TOOLS_COMP_DEFAULT_SHADER_MARKV_MODEL_H_
#define SPIRV_TOOLS_COMP_DEFAULT_SHADER_MARKV_MODEL_H_
#include "source/comp/markv_model.h"
namespace spvtools {
// MARK-V model designed to be a default model for shader compression.
class MarkvModelShaderDefault : public MarkvModel {
public:
MarkvModelShaderDefault();
};
} // namespace spvtools
#endif // SPIRV_TOOLS_COMP_DEFAULT_SHADER_MARKV_MODEL_H_