AuroraMiddleware/SPIRV-Tools
1
0
Fork 0
mirror of https://github.com/KhronosGroup/SPIRV-Tools synced 2024-12-24 16:51:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactored MARK-V API

Browse Source 
- switched from C to C++
- moved MARK-V model creation from backend to frontend
- The same MARK-V model object can be used to encode/decode multiple
files
- Added MARK-V model factory (currently only one option)
- Added --validate option to spirv-markv (run validation while
encoding/decoding)
This commit is contained in:
Andrey Tuganov 2017-10-03 17:36:37 -04:00 committed by David Neto
parent b54997e6eb
commit 2401fc0a72
17 changed files with 616 additions and 624 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
include/spirv-tools/markv.h
View File

@ -1,91 +0,0 @@
// Copyright (c) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// MARK-V is a compression format for SPIR-V binaries. It strips away
// non-essential information (such as result ids which can be regenerated) and
// uses various bit reduction techiniques to reduce the size of the binary.
//
// WIP: MARK-V codec is in early stages of development. At the moment it only
// can encode and decode some SPIR-V files and only if exacly the same build of
// software is used (is doesn't write or handle version numbers yet).
#ifndef SPIRV_TOOLS_MARKV_H_
#define SPIRV_TOOLS_MARKV_H_
#include "libspirv.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct spv_markv_binary_t {
uint8_t* data;
size_t length;
} spv_markv_binary_t;
typedef spv_markv_binary_t* spv_markv_binary;
typedef const spv_markv_binary_t* const_spv_markv_binary;
typedef struct spv_markv_encoder_options_t spv_markv_encoder_options_t;
typedef spv_markv_encoder_options_t* spv_markv_encoder_options;
typedef const spv_markv_encoder_options_t* spv_const_markv_encoder_options;
typedef struct spv_markv_decoder_options_t spv_markv_decoder_options_t;
typedef spv_markv_decoder_options_t* spv_markv_decoder_options;
typedef const spv_markv_decoder_options_t* spv_const_markv_decoder_options;
// Creates spv_markv_encoder_options with default options. Returns a valid
// options object. The object remains valid until it is passed into
// spvMarkvEncoderOptionsDestroy.
spv_markv_encoder_options spvMarkvEncoderOptionsCreate();
// Destroys the given spv_markv_encoder_options object.
void spvMarkvEncoderOptionsDestroy(spv_markv_encoder_options options);
// Creates spv_markv_decoder_options with default options. Returns a valid
// options object. The object remains valid until it is passed into
// spvMarkvDecoderOptionsDestroy.
spv_markv_decoder_options spvMarkvDecoderOptionsCreate();
// Destroys the given spv_markv_decoder_options object.
void spvMarkvDecoderOptionsDestroy(spv_markv_decoder_options options);
// Encodes the given SPIR-V binary to MARK-V binary.
// If |comments| is not nullptr, it would contain a textual description of
// how encoding was done (with snippets of disassembly and bit sequences).
spv_result_t spvSpirvToMarkv(spv_const_context context,
const uint32_t* spirv_words,
size_t spirv_num_words,
spv_const_markv_encoder_options options,
spv_markv_binary* markv_binary,
spv_text* comments, spv_diagnostic* diagnostic);
// Decodes a SPIR-V binary from the given MARK-V binary.
// If |comments| is not nullptr, it would contain a textual description of
// how decoding was done (with snippets of disassembly and bit sequences).
spv_result_t spvMarkvToSpirv(spv_const_context context,
const uint8_t* markv_data,
size_t markv_size_bytes,
spv_const_markv_decoder_options options,
spv_binary* spirv_binary,
spv_text* comments, spv_diagnostic* diagnostic);
// Destroys MARK-V binary created by spvSpirvToMarkv().
void spvMarkvBinaryDestroy(spv_markv_binary binary);
#ifdef __cplusplus
}
#endif
#endif // SPIRV_TOOLS_MARKV_H_

2
source/comp/CMakeLists.txt
View File

@ -13,7 +13,7 @@
# limitations under the License.
if(SPIRV_BUILD_COMPRESSION)
add_library(SPIRV-Tools-comp markv_codec.cpp markv_autogen.cpp)
add_library(SPIRV-Tools-comp markv_codec.cpp)
spvtools_default_compile_options(SPIRV-Tools-comp)
target_include_directories(SPIRV-Tools-comp

64
source/comp/markv.h Normal file
View File

@ -0,0 +1,64 @@
// Copyright (c) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// MARK-V is a compression format for SPIR-V binaries. It strips away
// non-essential information (such as result ids which can be regenerated) and
// uses various bit reduction techiniques to reduce the size of the binary and
// make it more similar to other compressed SPIR-V files to further improve
// compression of the dataset.
#ifndef SPIRV_TOOLS_MARKV_HPP_
#define SPIRV_TOOLS_MARKV_HPP_
#include <string>
#include <vector>
#include "markv_model.h"
#include "spirv-tools/libspirv.hpp"
namespace spvtools {
struct MarkvEncoderOptions {
bool validate_spirv_binary = false;
};
struct MarkvDecoderOptions {
bool validate_spirv_binary = false;
};
// Encodes the given SPIR-V binary to MARK-V binary.
// If |comments| is not nullptr, it would contain a textual description of
// how encoding was done (with snippets of disassembly and bit sequences).
spv_result_t SpirvToMarkv(spv_const_context context,
const std::vector<uint32_t>& spirv,
const MarkvEncoderOptions& options,
const MarkvModel& markv_model,
MessageConsumer message_consumer,
std::vector<uint8_t>* markv,
std::string* comments);
// Decodes a SPIR-V binary from the given MARK-V binary.
// If |comments| is not nullptr, it would contain a textual description of
// how decoding was done (with snippets of disassembly and bit sequences).
spv_result_t MarkvToSpirv(spv_const_context context,
const std::vector<uint8_t>& markv,
const MarkvDecoderOptions& options,
const MarkvModel& markv_model,
MessageConsumer message_consumer,
std::vector<uint32_t>* spirv,
std::string* comments);
} // namespace spvtools
#endif // SPIRV_TOOLS_MARKV_HPP_

58
source/comp/markv_autogen.cpp
View File

@ -1,58 +0,0 @@
// Copyright (c) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "markv_autogen.h"
#include <algorithm>
#include <map>
#include <memory>
#include <vector>
#include <unordered_map>
#include <unordered_set>
#include "spirv/1.2/spirv.h"
using spvutils::HuffmanCodec;
namespace {
// Signals that the value is not in the coding scheme and a fallback method
// needs to be used.
const uint64_t kMarkvNoneOfTheAbove = GetMarkvNonOfTheAbove();
inline uint32_t CombineOpcodeAndNumOperands(uint32_t opcode,
uint32_t num_operands) {
return opcode | (num_operands << 16);
}
} // namespace
// The following file contains autogenerated statistical coding rules.
// Generated by running spirv-stats on representative corpus of shaders with
// flags:
// --codegen_opcode_and_num_operands_hist
// --codegen_opcode_and_num_operands_markov_huffman_codecs
// --codegen_literal_string_huffman_codecs
// --codegen_non_id_word_huffman_codecs
// --codegen_id_descriptor_huffman_codecs
//
// Example:
// find <SHADER_CORPUS_DIR> -type f -print0 | xargs -0 -s 2000000
// ~/SPIRV-Tools/build/tools/spirv-stats -v
// --codegen_opcode_and_num_operands_hist
// --codegen_opcode_and_num_operands_markov_huffman_codecs
// --codegen_literal_string_huffman_codecs --codegen_non_id_word_huffman_codecs
// --codegen_id_descriptor_huffman_codecs -o
// ~/SPIRV-Tools/source/comp/markv_autogen.inc
#include "markv_autogen.inc"

60
source/comp/markv_autogen.h
View File

@ -1,60 +0,0 @@
// Copyright (c) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef LIBSPIRV_COMP_MARKV_AUTOGEN_H_
#define LIBSPIRV_COMP_MARKV_AUTOGEN_H_
#include <map>
#include <memory>
#include <numeric>
#include <unordered_set>
#include "util/huffman_codec.h"
inline uint64_t GetMarkvNonOfTheAbove() {
// Magic number.
return 1111111111111111111;
}
// Returns of histogram of CombineOpcodeAndNumOperands(opcode, num_operands).
std::map<uint64_t, uint32_t> GetOpcodeAndNumOperandsHist();
// Returns Huffman codecs based on a Markov chain of histograms of
// CombineOpcodeAndNumOperands(opcode, num_operands).
// Map prev_opcode -> codec.
std::map<uint32_t, std::unique_ptr<spvutils::HuffmanCodec<uint64_t>>>
GetOpcodeAndNumOperandsMarkovHuffmanCodecs();
// Returns Huffman codecs for literal strings.
// Map opcode -> codec.
std::map<uint32_t, std::unique_ptr<spvutils::HuffmanCodec<std::string>>>
GetLiteralStringHuffmanCodecs();
// Returns Huffman codecs for single-word non-id operand slots.
// Map <opcode, operand_index> -> codec.
std::map<std::pair<uint32_t, uint32_t>,
std::unique_ptr<spvutils::HuffmanCodec<uint64_t>>>
GetNonIdWordHuffmanCodecs();
// Returns Huffman codecs for id descriptors used by common operand slots.
// Map <opcode, operand_index> -> codec.
std::map<std::pair<uint32_t, uint32_t>,
std::unique_ptr<spvutils::HuffmanCodec<uint64_t>>>
GetIdDescriptorHuffmanCodecs();
// Returns a set of all descriptors which are encodable by at least one codec
// returned by GetIdDescriptorHuffmanCodecs().
std::unordered_set<uint32_t> GetDescriptorsWithCodingScheme();
#endif // LIBSPIRV_COMP_MARKV_AUTOGEN_H_

424
source/comp/markv_codec.cpp
View File

@ -19,9 +19,6 @@
// MARK-V is a compression format for SPIR-V binaries. It strips away
// non-essential information (such as result ids which can be regenerated) and
// uses various bit reduction techiniques to reduce the size of the binary.
//
// MarkvModel is a flatbuffers object containing a set of rules defining how
// compression/decompression is done (coding schemes, dictionaries).
#include <algorithm>
#include <cassert>
@ -48,11 +45,11 @@
#include "ext_inst.h"
#include "id_descriptor.h"
#include "instruction.h"
#include "markv_autogen.h"
#include "markv.h"
#include "markv_model.h"
#include "opcode.h"
#include "operand.h"
#include "spirv-tools/libspirv.h"
#include "spirv-tools/markv.h"
#include "spirv_endian.h"
#include "spirv_validator_options.h"
#include "util/bit_stream.h"
@ -74,13 +71,7 @@ using spvutils::HuffmanCodec;
using MoveToFront = spvutils::MoveToFront<uint32_t>;
using MultiMoveToFront = spvutils::MultiMoveToFront<uint32_t>;
struct spv_markv_encoder_options_t {
bool validate_spirv_binary = false;
};
struct spv_markv_decoder_options_t {
bool validate_spirv_binary = false;
};
namespace spvtools {
namespace {
@ -155,7 +146,7 @@ const uint32_t kMarkvMaxPresumedAccessIndex = 31;
// Signals that the value is not in the coding scheme and a fallback method
// needs to be used.
const uint64_t kMarkvNoneOfTheAbove = GetMarkvNonOfTheAbove();
const uint64_t kMarkvNoneOfTheAbove = MarkvModel::GetMarkvNoneOfTheAbove();
// Mtf ranks smaller than this are encoded with Huffman coding.
const uint32_t kMtfSmallestRankEncodedByValue = 10;
@ -208,208 +199,6 @@ GetMtfHuffmanCodecs() {
return codecs;
}
// Encoding/decoding model containing various constants and codecs.
class MarkvModel {
public:
MarkvModel()
: mtf_huffman_codecs_(GetMtfHuffmanCodecs()),
opcode_and_num_operands_huffman_codec_(GetOpcodeAndNumOperandsHist()),
opcode_and_num_operands_markov_huffman_codecs_(
GetOpcodeAndNumOperandsMarkovHuffmanCodecs()),
non_id_word_huffman_codecs_(GetNonIdWordHuffmanCodecs()),
id_descriptor_huffman_codecs_(GetIdDescriptorHuffmanCodecs()),
descriptors_with_coding_scheme_(GetDescriptorsWithCodingScheme()),
literal_string_huffman_codecs_(GetLiteralStringHuffmanCodecs()) {}
size_t opcode_chunk_length() const { return 7; }
size_t num_operands_chunk_length() const { return 3; }
size_t mtf_rank_chunk_length() const { return 5; }
size_t u16_chunk_length() const { return 4; }
size_t s16_chunk_length() const { return 4; }
size_t s16_block_exponent() const { return 6; }
size_t u32_chunk_length() const { return 8; }
size_t s32_chunk_length() const { return 8; }
size_t s32_block_exponent() const { return 10; }
size_t u64_chunk_length() const { return 8; }
size_t s64_chunk_length() const { return 8; }
size_t s64_block_exponent() const { return 10; }
// Returns Huffman codec for ranks of the mtf with given |handle|.
// Different mtfs can use different rank distributions.
// May return nullptr if the codec doesn't exist.
const HuffmanCodec<uint32_t>* GetMtfHuffmanCodec(uint64_t handle) const {
const auto it = mtf_huffman_codecs_.find(handle);
if (it == mtf_huffman_codecs_.end())
return nullptr;
return it->second.get();
}
// Returns a codec for common opcode_and_num_operands words for the given
// previous opcode. May return nullptr if the codec doesn't exist.
const HuffmanCodec<uint64_t>* GetOpcodeAndNumOperandsMarkovHuffmanCodec(
uint32_t prev_opcode) const {
if (prev_opcode == SpvOpNop)
return &opcode_and_num_operands_huffman_codec_;
const auto it =
opcode_and_num_operands_markov_huffman_codecs_.find(prev_opcode);
if (it == opcode_and_num_operands_markov_huffman_codecs_.end())
return nullptr;
return it->second.get();
}
// Returns a codec for common non-id words used for given operand slot.
// Operand slot is defined by the opcode and the operand index.
// May return nullptr if the codec doesn't exist.
const HuffmanCodec<uint64_t>* GetNonIdWordHuffmanCodec(
uint32_t opcode, uint32_t operand_index) const {
const auto it = non_id_word_huffman_codecs_.find(
std::pair<uint32_t, uint32_t>(opcode, operand_index));
if (it == non_id_word_huffman_codecs_.end())
return nullptr;
return it->second.get();
}
// Returns a codec for common id descriptos used for given operand slot.
// Operand slot is defined by the opcode and the operand index.
// May return nullptr if the codec doesn't exist.
const HuffmanCodec<uint64_t>* GetIdDescriptorHuffmanCodec(
uint32_t opcode, uint32_t operand_index) const {
const auto it = id_descriptor_huffman_codecs_.find(
std::pair<uint32_t, uint32_t>(opcode, operand_index));
if (it == id_descriptor_huffman_codecs_.end())
return nullptr;
return it->second.get();
}
// Returns a codec for common strings used by the given opcode.
// Operand slot is defined by the opcode and the operand index.
// May return nullptr if the codec doesn't exist.
const HuffmanCodec<std::string>* GetLiteralStringHuffmanCodec(
uint32_t opcode) const {
const auto it = literal_string_huffman_codecs_.find(opcode);
if (it == literal_string_huffman_codecs_.end())
return nullptr;
return it->second.get();
}
bool DescriptorHasCodingScheme(uint32_t descriptor) const {
return descriptors_with_coding_scheme_.count(descriptor);
}
private:
// Huffman codecs for move-to-front ranks. The map key is mtf handle. Doesn't
// need to contain a different codec for every handle as most use one and the
// same.
std::map<uint64_t, std::unique_ptr<HuffmanCodec<uint32_t>>>
mtf_huffman_codecs_;
// Huffman codec for base-rate of opcode_and_num_operands.
HuffmanCodec<uint64_t> opcode_and_num_operands_huffman_codec_;
// Huffman codecs for opcode_and_num_operands. The map key is previous opcode.
std::map<uint32_t, std::unique_ptr<HuffmanCodec<uint64_t>>>
opcode_and_num_operands_markov_huffman_codecs_;
// Huffman codecs for non-id single-word operand values.
// The map key is pair <opcode, operand_index>.
std::map<std::pair<uint32_t, uint32_t>,
std::unique_ptr<HuffmanCodec<uint64_t>>>
non_id_word_huffman_codecs_;
// Huffman codecs for id descriptors. The map key is pair
// <opcode, operand_index>.
std::map<std::pair<uint32_t, uint32_t>,
std::unique_ptr<HuffmanCodec<uint64_t>>>
id_descriptor_huffman_codecs_;
std::unordered_set<uint32_t> descriptors_with_coding_scheme_;
// Huffman codecs for literal strings. The map key is the opcode of the
// current instruction. This assumes, that there is no more than one literal
// string operand per instruction, but would still work even if this is not
// the case. Names and debug information strings are not collected.
std::map<uint32_t, std::unique_ptr<HuffmanCodec<std::string>>>
literal_string_huffman_codecs_;
};
const MarkvModel* GetDefaultModel() {
static MarkvModel model;
return &model;
}
// Returns chunk length used for variable length encoding of spirv operand
// words. Returns zero if operand type corresponds to potentially multiple
// words or a word which is not expected to profit from variable width encoding.
// Chunk length is selected based on the size of expected value.
// Most of these values will later be encoded with probability-based coding,
// but variable width integer coding is a good quick solution.
// TODO(atgoo@github.com): Put this in MarkvModel flatbuffer.
size_t GetOperandVariableWidthChunkLength(spv_operand_type_t type) {
switch (type) {
case SPV_OPERAND_TYPE_TYPE_ID:
return 4;
case SPV_OPERAND_TYPE_RESULT_ID:
case SPV_OPERAND_TYPE_ID:
case SPV_OPERAND_TYPE_SCOPE_ID:
case SPV_OPERAND_TYPE_MEMORY_SEMANTICS_ID:
return 8;
case SPV_OPERAND_TYPE_LITERAL_INTEGER:
case SPV_OPERAND_TYPE_OPTIONAL_LITERAL_INTEGER:
return 6;
case SPV_OPERAND_TYPE_CAPABILITY:
return 6;
case SPV_OPERAND_TYPE_SOURCE_LANGUAGE:
case SPV_OPERAND_TYPE_EXECUTION_MODEL:
return 3;
case SPV_OPERAND_TYPE_ADDRESSING_MODEL:
case SPV_OPERAND_TYPE_MEMORY_MODEL:
return 2;
case SPV_OPERAND_TYPE_EXECUTION_MODE:
return 6;
case SPV_OPERAND_TYPE_STORAGE_CLASS:
return 4;
case SPV_OPERAND_TYPE_DIMENSIONALITY:
case SPV_OPERAND_TYPE_SAMPLER_ADDRESSING_MODE:
return 3;
case SPV_OPERAND_TYPE_SAMPLER_FILTER_MODE:
return 2;
case SPV_OPERAND_TYPE_SAMPLER_IMAGE_FORMAT:
return 6;
case SPV_OPERAND_TYPE_FP_ROUNDING_MODE:
case SPV_OPERAND_TYPE_LINKAGE_TYPE:
case SPV_OPERAND_TYPE_ACCESS_QUALIFIER:
case SPV_OPERAND_TYPE_OPTIONAL_ACCESS_QUALIFIER:
return 2;
case SPV_OPERAND_TYPE_FUNCTION_PARAMETER_ATTRIBUTE:
return 3;
case SPV_OPERAND_TYPE_DECORATION:
case SPV_OPERAND_TYPE_BUILT_IN:
return 6;
case SPV_OPERAND_TYPE_GROUP_OPERATION:
case SPV_OPERAND_TYPE_KERNEL_ENQ_FLAGS:
case SPV_OPERAND_TYPE_KERNEL_PROFILING_INFO:
return 2;
case SPV_OPERAND_TYPE_FP_FAST_MATH_MODE:
case SPV_OPERAND_TYPE_FUNCTION_CONTROL:
case SPV_OPERAND_TYPE_LOOP_CONTROL:
case SPV_OPERAND_TYPE_IMAGE:
case SPV_OPERAND_TYPE_OPTIONAL_IMAGE:
case SPV_OPERAND_TYPE_OPTIONAL_MEMORY_ACCESS:
case SPV_OPERAND_TYPE_SELECTION_CONTROL:
return 4;
case SPV_OPERAND_TYPE_EXTENSION_INSTRUCTION_NUMBER:
case SPV_OPERAND_TYPE_TYPED_LITERAL_NUMBER:
return 6;
default:
return 0;
}
return 0;
}
// Returns true if the opcode has a fixed number of operands. May return a
// false negative.
bool OpcodeHasFixedNumberOfOperands(SpvOp opcode) {
@ -534,21 +323,6 @@ class CommentLogger {
bool use_delimiter_ = false;
};
// Creates spv_text object containing text from |str|.
// The returned value is owned by the caller and needs to be destroyed with
// spvTextDestroy.
spv_text CreateSpvText(const std::string& str) {
spv_text out = new spv_text_t();
assert(out);
char* cstr = new char[str.length() + 1];
assert(cstr);
std::strncpy(cstr, str.c_str(), str.length());
cstr[str.length()] = '\0';
out->str = cstr;
out->length = str.length();
return out;
}
// Base class for MARK-V encoder and decoder. Contains common functionality
// such as:
// - Validator connection and validation state.
@ -561,10 +335,6 @@ class MarkvCodecBase {
MarkvCodecBase() = delete;
void SetModel(const MarkvModel* model) {
model_ = model;
}
protected:
struct MarkvHeader {
MarkvHeader() {
@ -585,10 +355,14 @@ class MarkvCodecBase {
uint32_t spirv_generator;
};
// |model| is owned by the caller, must be not null and valid during the
// lifetime of the codec.
explicit MarkvCodecBase(spv_const_context context,
spv_validator_options validator_options)
spv_validator_options validator_options,
const MarkvModel* model)
: validator_options_(validator_options), grammar_(context),
model_(GetDefaultModel()), context_(context),
model_(model), mtf_huffman_codecs_(GetMtfHuffmanCodecs()),
context_(context),
vstate_(validator_options ?
new ValidationState_t(context, validator_options_) : nullptr) {}
@ -713,9 +487,21 @@ class MarkvCodecBase {
vstate_->setIdBound(id_bound);
}
// Returns Huffman codec for ranks of the mtf with given |handle|.
// Different mtfs can use different rank distributions.
// May return nullptr if the codec doesn't exist.
const spvutils::HuffmanCodec<uint32_t>* GetMtfHuffmanCodec(uint64_t handle) const {
const auto it = mtf_huffman_codecs_.find(handle);
if (it == mtf_huffman_codecs_.end())
return nullptr;
return it->second.get();
}
spv_validator_options validator_options_ = nullptr;
const libspirv::AssemblyGrammar grammar_;
MarkvHeader header_;
// MARK-V model, not owned.
const MarkvModel* model_ = nullptr;
// Current instruction, current operand and current operand index.
@ -755,6 +541,12 @@ class MarkvCodecBase {
// Container/computer for id descriptors.
IdDescriptorCollection id_descriptors_;
// Huffman codecs for move-to-front ranks. The map key is mtf handle. Doesn't
// need to contain a different codec for every handle as most use one and the
// same.
std::map<uint64_t, std::unique_ptr<HuffmanCodec<uint32_t>>>
mtf_huffman_codecs_;
private:
spv_const_context context_ = nullptr;
@ -777,9 +569,12 @@ class MarkvCodecBase {
// on how encoding was done, which can later be accessed with GetComments().
class MarkvEncoder : public MarkvCodecBase {
public:
// |model| is owned by the caller, must be not null and valid during the
// lifetime of MarkvEncoder.
MarkvEncoder(spv_const_context context,
spv_const_markv_encoder_options options)
: MarkvCodecBase(context, GetValidatorOptions(options)),
const MarkvEncoderOptions& options,
const MarkvModel* model)
: MarkvCodecBase(context, GetValidatorOptions(options), model),
options_(options) {
(void) options_;
}
@ -804,19 +599,20 @@ class MarkvEncoder : public MarkvCodecBase {
// into a single buffer and returns it as spv_markv_binary. The returned
// value is owned by the caller and needs to be destroyed with
// spvMarkvBinaryDestroy().
spv_markv_binary GetMarkvBinary() {
std::vector<uint8_t> GetMarkvBinary() {
header_.markv_length_in_bits =
static_cast<uint32_t>(sizeof(header_) * 8 + writer_.GetNumBits());
const size_t num_bytes = sizeof(header_) + writer_.GetDataSizeBytes();
header_.markv_model =
(model_->model_type() << 16) | model_->model_version();
const size_t num_bytes = sizeof(header_) + writer_.GetDataSizeBytes();
std::vector<uint8_t> markv(num_bytes);
spv_markv_binary markv_binary = new spv_markv_binary_t();
markv_binary->data = new uint8_t[num_bytes];
markv_binary->length = num_bytes;
assert(writer_.GetData());
std::memcpy(markv_binary->data, &header_, sizeof(header_));
std::memcpy(markv_binary->data + sizeof(header_),
std::memcpy(markv.data(), &header_, sizeof(header_));
std::memcpy(markv.data() + sizeof(header_),
writer_.GetData(), writer_.GetDataSizeBytes());
return markv_binary;
return markv;
}
// Creates an internal logger which writes comments on the encoding process.
@ -854,8 +650,8 @@ class MarkvEncoder : public MarkvCodecBase {
private:
// Creates and returns validator options. Returned value owned by the caller.
static spv_validator_options GetValidatorOptions(
spv_const_markv_encoder_options options) {
return options->validate_spirv_binary ?
const MarkvEncoderOptions& options) {
return options.validate_spirv_binary ?
spvValidatorOptionsCreate() : nullptr;
}
@ -896,7 +692,7 @@ class MarkvEncoder : public MarkvCodecBase {
// Encodes a literal number operand and writes it to the bit stream.
spv_result_t EncodeLiteralNumber(const spv_parsed_operand_t& operand);
spv_const_markv_encoder_options options_;
MarkvEncoderOptions options_;
// Bit stream where encoded instructions are written.
BitWriterWord64 writer_;
@ -912,12 +708,14 @@ class MarkvEncoder : public MarkvCodecBase {
// Decodes MARK-V buffers written by MarkvEncoder.
class MarkvDecoder : public MarkvCodecBase {
public:
// |model| is owned by the caller, must be not null and valid during the
// lifetime of MarkvEncoder.
MarkvDecoder(spv_const_context context,
const uint8_t* markv_data,
size_t markv_size_bytes,
spv_const_markv_decoder_options options)
: MarkvCodecBase(context, GetValidatorOptions(options)),
options_(options), reader_(markv_data, markv_size_bytes) {
const std::vector<uint8_t>& markv,
const MarkvDecoderOptions& options,
const MarkvModel* model)
: MarkvCodecBase(context, GetValidatorOptions(options), model),
options_(options), reader_(markv) {
(void) options_;
SetIdBound(1);
parsed_operands_.reserve(25);
@ -938,8 +736,8 @@ class MarkvDecoder : public MarkvCodecBase {
// Creates and returns validator options. Returned value owned by the caller.
static spv_validator_options GetValidatorOptions(
spv_const_markv_decoder_options options) {
return options->validate_spirv_binary ?
const MarkvDecoderOptions& options) {
return options.validate_spirv_binary ?
spvValidatorOptionsCreate() : nullptr;
}
@ -1024,7 +822,7 @@ class MarkvDecoder : public MarkvCodecBase {
// kind SPV_NUMBER_NONE.
void RecordNumberType();
spv_const_markv_decoder_options options_;
MarkvDecoderOptions options_;
// Temporary sink where decoded SPIR-V words are written. Once it contains the
// entire module, the container is moved and returned.
@ -1690,7 +1488,8 @@ spv_result_t MarkvEncoder::EncodeNonIdWord(uint32_t word) {
}
// Fallback encoding.
const size_t chunk_length = GetOperandVariableWidthChunkLength(operand_.type);
const size_t chunk_length =
model_->GetOperandVariableWidthChunkLength(operand_.type);
if (chunk_length) {
writer_.WriteVariableWidthU32(word, chunk_length);
} else {
@ -1718,7 +1517,8 @@ spv_result_t MarkvDecoder::DecodeNonIdWord(uint32_t* word) {
// Received kMarkvNoneOfTheAbove signal, use fallback decoding.
}
const size_t chunk_length = GetOperandVariableWidthChunkLength(operand_.type);
const size_t chunk_length =
model_->GetOperandVariableWidthChunkLength(operand_.type);
if (chunk_length) {
if (!reader_.ReadVariableWidthU32(word, chunk_length))
return Diag(SPV_ERROR_INVALID_BINARY)
@ -1817,10 +1617,10 @@ spv_result_t MarkvDecoder::DecodeOpcodeAndNumberOfOperands(
spv_result_t MarkvEncoder::EncodeMtfRankHuffman(uint32_t rank, uint64_t mtf,
uint64_t fallback_method) {
const auto* codec = model_->GetMtfHuffmanCodec(mtf);
const auto* codec = GetMtfHuffmanCodec(mtf);
if (!codec) {
assert(fallback_method != kMtfNone);
codec = model_->GetMtfHuffmanCodec(fallback_method);
codec = GetMtfHuffmanCodec(fallback_method);
}
if (!codec)
@ -1850,10 +1650,10 @@ spv_result_t MarkvEncoder::EncodeMtfRankHuffman(uint32_t rank, uint64_t mtf,
spv_result_t MarkvDecoder::DecodeMtfRankHuffman(
uint64_t mtf, uint32_t fallback_method, uint32_t* rank) {
const auto* codec = model_->GetMtfHuffmanCodec(mtf);
const auto* codec = GetMtfHuffmanCodec(mtf);
if (!codec) {
assert(fallback_method != kMtfNone);
codec = model_->GetMtfHuffmanCodec(fallback_method);
codec = GetMtfHuffmanCodec(fallback_method);
}
if (!codec)
@ -2493,6 +2293,17 @@ spv_result_t MarkvDecoder::DecodeModule(std::vector<uint32_t>* spirv_binary) {
return Diag(SPV_ERROR_INVALID_BINARY)
<< "MARK-V binary and the codec have different versions";
const uint32_t model_type = header_.markv_model >> 16;
const uint32_t model_version = header_.markv_model & 0xFFFF;
if (model_type != model_->model_type())
return Diag(SPV_ERROR_INVALID_BINARY)
<< "MARK-V binary and the codec use different MARK-V models";
if (model_version != model_->model_version())
return Diag(SPV_ERROR_INVALID_BINARY)
<< "MARK-V binary and the codec use different versions if the same "
<< "MARK-V model";
spirv_.reserve(header_.markv_length_in_bits / 2); // Heuristic.
spirv_.resize(5, 0);
spirv_[0] = kSpirvMagicNumber;
@ -3026,19 +2837,17 @@ spv_result_t EncodeInstruction(
} // namespace
spv_result_t spvSpirvToMarkv(spv_const_context context,
const uint32_t* spirv_words,
const size_t spirv_num_words,
spv_const_markv_encoder_options options,
spv_markv_binary* markv_binary,
spv_text* comments, spv_diagnostic* diagnostic) {
spv_result_t SpirvToMarkv(spv_const_context context,
const std::vector<uint32_t>& spirv,
const MarkvEncoderOptions& options,
const MarkvModel& markv_model,
MessageConsumer message_consumer,
std::vector<uint8_t>* markv,
std::string* comments) {
spv_context_t hijack_context = *context;
if (diagnostic) {
*diagnostic = nullptr;
libspirv::UseDiagnosticAsMessageConsumer(&hijack_context, diagnostic);
}
SetContextMessageConsumer(&hijack_context, message_consumer);
spv_const_binary_t spirv_binary = {spirv_words, spirv_num_words};
spv_const_binary_t spirv_binary = {spirv.data(), spirv.size()};
spv_endianness_t endian;
spv_position_t position = {};
@ -3055,13 +2864,13 @@ spv_result_t spvSpirvToMarkv(spv_const_context context,
<< "Invalid SPIR-V header.";
}
MarkvEncoder encoder(&hijack_context, options);
MarkvEncoder encoder(&hijack_context, options, &markv_model);
if (comments) {
encoder.CreateCommentsLogger();
spv_text text = nullptr;
if (spvBinaryToText(&hijack_context, spirv_words, spirv_num_words,
if (spvBinaryToText(&hijack_context, spirv.data(), spirv.size(),
SPV_BINARY_TO_TEXT_OPTION_NO_HEADER, &text, nullptr)
!= SPV_SUCCESS) {
return DiagnosticStream(position, hijack_context.consumer,
@ -3074,72 +2883,41 @@ spv_result_t spvSpirvToMarkv(spv_const_context context,
}
if (spvBinaryParse(
&hijack_context, &encoder, spirv_words, spirv_num_words, EncodeHeader,
EncodeInstruction, diagnostic) != SPV_SUCCESS) {
&hijack_context, &encoder, spirv.data(), spirv.size(), EncodeHeader,
EncodeInstruction, nullptr) != SPV_SUCCESS) {
return DiagnosticStream(position, hijack_context.consumer,
SPV_ERROR_INVALID_BINARY)
<< "Unable to encode to MARK-V.";
}
if (comments)
*comments = CreateSpvText(encoder.GetComments());
*comments = encoder.GetComments();
*markv_binary = encoder.GetMarkvBinary();
*markv = encoder.GetMarkvBinary();
return SPV_SUCCESS;
}
spv_result_t spvMarkvToSpirv(spv_const_context context,
const uint8_t* markv_data,
size_t markv_size_bytes,
spv_const_markv_decoder_options options,
spv_binary* spirv_binary,
spv_text* /* comments */,
spv_diagnostic* diagnostic) {
spv_result_t MarkvToSpirv(spv_const_context context,
const std::vector<uint8_t>& markv,
const MarkvDecoderOptions& options,
const MarkvModel& markv_model,
MessageConsumer message_consumer,
std::vector<uint32_t>* spirv,
std::string* /* comments */) {
spv_position_t position = {};
spv_context_t hijack_context = *context;
if (diagnostic) {
*diagnostic = nullptr;
libspirv::UseDiagnosticAsMessageConsumer(&hijack_context, diagnostic);
}
SetContextMessageConsumer(&hijack_context, message_consumer);
MarkvDecoder decoder(&hijack_context, markv_data, markv_size_bytes, options);
MarkvDecoder decoder(&hijack_context, markv, options, &markv_model);
std::vector<uint32_t> words;
if (decoder.DecodeModule(&words) != SPV_SUCCESS) {
if (decoder.DecodeModule(spirv) != SPV_SUCCESS) {
return DiagnosticStream(position, hijack_context.consumer,
SPV_ERROR_INVALID_BINARY)
<< "Unable to decode MARK-V.";
}
assert(!words.empty());
*spirv_binary = new spv_binary_t();
(*spirv_binary)->code = new uint32_t[words.size()];
(*spirv_binary)->wordCount = words.size();
std::memcpy((*spirv_binary)->code, words.data(), 4 * words.size());
assert(!spirv->empty());
return SPV_SUCCESS;
}
void spvMarkvBinaryDestroy(spv_markv_binary binary) {
if (!binary) return;
delete[] binary->data;
delete binary;
}
spv_markv_encoder_options spvMarkvEncoderOptionsCreate() {
return new spv_markv_encoder_options_t;
}
void spvMarkvEncoderOptionsDestroy(spv_markv_encoder_options options) {
delete options;
}
spv_markv_decoder_options spvMarkvDecoderOptionsCreate() {
return new spv_markv_decoder_options_t;
}
void spvMarkvDecoderOptionsDestroy(spv_markv_decoder_options options) {
delete options;
}
} // namespave spvtools

176
source/comp/markv_model.h Normal file
View File

@ -0,0 +1,176 @@
// Copyright (c) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef LIBSPIRV_COMP_MARKV_MODEL_H_
#define LIBSPIRV_COMP_MARKV_MODEL_H_
#include <map>
#include <unordered_set>
#include <vector>
#include "spirv/1.2/spirv.h"
#include "spirv-tools/libspirv.h"
#include "util/huffman_codec.h"
namespace spvtools {
// Base class for MARK-V models.
// The class contains encoding/decoding model with various constants and
// codecs used by the compression algorithm.
class MarkvModel {
public:
MarkvModel() : operand_chunk_lengths_(
static_cast<size_t>(SPV_OPERAND_TYPE_NUM_OPERAND_TYPES), 0) {}
uint32_t model_type() const { return model_type_; }
uint32_t model_version() const { return model_version_; }
uint32_t opcode_chunk_length() const { return opcode_chunk_length_; }
uint32_t num_operands_chunk_length() const { return num_operands_chunk_length_; }
uint32_t mtf_rank_chunk_length() const { return mtf_rank_chunk_length_; }
uint32_t u64_chunk_length() const { return u64_chunk_length_; }
uint32_t s64_chunk_length() const { return s64_chunk_length_; }
uint32_t s64_block_exponent() const { return s64_block_exponent_; }
// Returns a codec for common opcode_and_num_operands words for the given
// previous opcode. May return nullptr if the codec doesn't exist.
const spvutils::HuffmanCodec<uint64_t>* GetOpcodeAndNumOperandsMarkovHuffmanCodec(
uint32_t prev_opcode) const {
if (prev_opcode == SpvOpNop)
return opcode_and_num_operands_huffman_codec_.get();
const auto it =
opcode_and_num_operands_markov_huffman_codecs_.find(prev_opcode);
if (it == opcode_and_num_operands_markov_huffman_codecs_.end())
return nullptr;
return it->second.get();
}
// Returns a codec for common non-id words used for given operand slot.
// Operand slot is defined by the opcode and the operand index.
// May return nullptr if the codec doesn't exist.
const spvutils::HuffmanCodec<uint64_t>* GetNonIdWordHuffmanCodec(
uint32_t opcode, uint32_t operand_index) const {
const auto it = non_id_word_huffman_codecs_.find(
std::pair<uint32_t, uint32_t>(opcode, operand_index));
if (it == non_id_word_huffman_codecs_.end())
return nullptr;
return it->second.get();
}
// Returns a codec for common id descriptos used for given operand slot.
// Operand slot is defined by the opcode and the operand index.
// May return nullptr if the codec doesn't exist.
const spvutils::HuffmanCodec<uint64_t>* GetIdDescriptorHuffmanCodec(
uint32_t opcode, uint32_t operand_index) const {
const auto it = id_descriptor_huffman_codecs_.find(
std::pair<uint32_t, uint32_t>(opcode, operand_index));
if (it == id_descriptor_huffman_codecs_.end())
return nullptr;
return it->second.get();
}
// Returns a codec for common strings used by the given opcode.
// Operand slot is defined by the opcode and the operand index.
// May return nullptr if the codec doesn't exist.
const spvutils::HuffmanCodec<std::string>* GetLiteralStringHuffmanCodec(
uint32_t opcode) const {
const auto it = literal_string_huffman_codecs_.find(opcode);
if (it == literal_string_huffman_codecs_.end())
return nullptr;
return it->second.get();
}
// Checks if |descriptor| has a coding scheme in any of
// id_descriptor_huffman_codecs_.
bool DescriptorHasCodingScheme(uint32_t descriptor) const {
return descriptors_with_coding_scheme_.count(descriptor);
}
// Returns chunk length used for variable length encoding of spirv operand
// words.
uint32_t GetOperandVariableWidthChunkLength(spv_operand_type_t type) const {
return operand_chunk_lengths_.at(static_cast<size_t>(type));
}
// Sets model type.
void SetModelType(uint32_t in_model_type) {
model_type_ = in_model_type;
}
// Sets model version.
void SetModelVersion(uint32_t in_model_version) {
model_version_ = in_model_version;
}
// Returns value used by Huffman codecs as a signal that a value is not in the
// coding table.
static uint64_t GetMarkvNoneOfTheAbove() {
// Magic number.
return 1111111111111111111;
}
MarkvModel(const MarkvModel&) = delete;
const MarkvModel& operator=(const MarkvModel&) = delete;
protected:
// Huffman codec for base-rate of opcode_and_num_operands.
std::unique_ptr<spvutils::HuffmanCodec<uint64_t>>
opcode_and_num_operands_huffman_codec_;
// Huffman codecs for opcode_and_num_operands. The map key is previous opcode.
std::map<uint32_t, std::unique_ptr<spvutils::HuffmanCodec<uint64_t>>>
opcode_and_num_operands_markov_huffman_codecs_;
// Huffman codecs for non-id single-word operand values.
// The map key is pair <opcode, operand_index>.
std::map<std::pair<uint32_t, uint32_t>,
std::unique_ptr<spvutils::HuffmanCodec<uint64_t>>> non_id_word_huffman_codecs_;
// Huffman codecs for id descriptors. The map key is pair
// <opcode, operand_index>.
std::map<std::pair<uint32_t, uint32_t>,
std::unique_ptr<spvutils::HuffmanCodec<uint64_t>>> id_descriptor_huffman_codecs_;
// Set of all descriptors which have a coding scheme in any of
// id_descriptor_huffman_codecs_.
std::unordered_set<uint32_t> descriptors_with_coding_scheme_;
// Huffman codecs for literal strings. The map key is the opcode of the
// current instruction. This assumes, that there is no more than one literal
// string operand per instruction, but would still work even if this is not
// the case. Names and debug information strings are not collected.
std::map<uint32_t, std::unique_ptr<spvutils::HuffmanCodec<std::string>>>
literal_string_huffman_codecs_;
// Chunk lengths used for variable width encoding of operands (index is
// spv_operand_type of the operand).
std::vector<uint32_t> operand_chunk_lengths_;
uint32_t opcode_chunk_length_ = 7;
uint32_t num_operands_chunk_length_ = 3;
uint32_t mtf_rank_chunk_length_ = 5;
uint32_t u64_chunk_length_ = 8;
uint32_t s64_chunk_length_ = 8;
uint32_t s64_block_exponent_ = 10;
uint32_t model_type_ = 0;
uint32_t model_version_ = 0;
};
} // namespace spvtools
#endif // LIBSPIRV_COMP_MARKV_MODEL_H_

6
test/comp/CMakeLists.txt
View File

@ -19,7 +19,11 @@ set(VAL_TEST_COMMON_SRCS
if(SPIRV_BUILD_COMPRESSION)
add_spvtools_unittest(TARGET markv_codec
SRCS markv_codec_test.cpp ${VAL_TEST_COMMON_SRCS}
SRCS
markv_codec_test.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../tools/comp/markv_model_factory.cpp
${CMAKE_CURRENT_SOURCE_DIR}/../../tools/comp/markv_model_shader_default.cpp
${VAL_TEST_COMMON_SRCS}
LIBS SPIRV-Tools-comp ${SPIRV_TOOLS}
)
endif(SPIRV_BUILD_COMPRESSION)

70
test/comp/markv_codec_test.cpp
View File

@ -19,8 +19,9 @@
#include <string>
#include "gmock/gmock.h"
#include "spirv-tools/markv.h"
#include "source/comp/markv.h"
#include "test_fixture.h"
#include "tools/comp/markv_model_factory.h"
#include "unit_spirv.h"
namespace {
@ -82,52 +83,15 @@ void Disassemble(const std::vector<uint32_t>& words,
spvTextDestroy(text);
}
// Encodes SPIR-V |words| to |markv_binary|. |comments| context snippets of
// disassembly and bit sequences for debugging.
void Encode(const std::vector<uint32_t>& words,
spv_markv_binary* markv_binary,
std::string* comments,
spv_target_env env = SPV_ENV_UNIVERSAL_1_2) {
ScopedContext ctx(env);
SetContextMessageConsumer(ctx.context, DiagnosticsMessageHandler);
std::unique_ptr<spv_markv_encoder_options_t,
std::function<void(spv_markv_encoder_options_t*)>> options(
spvMarkvEncoderOptionsCreate(), &spvMarkvEncoderOptionsDestroy);
spv_text spv_text_comments;
ASSERT_EQ(SPV_SUCCESS, spvSpirvToMarkv(ctx.context, words.data(),
words.size(), options.get(),
markv_binary, &spv_text_comments,
nullptr));
*comments = std::string(spv_text_comments->str, spv_text_comments->length);
spvTextDestroy(spv_text_comments);
}
// Decodes |markv_binary| to SPIR-V |words|.
void Decode(const spv_markv_binary markv_binary,
std::vector<uint32_t>* words,
spv_target_env env = SPV_ENV_UNIVERSAL_1_2) {
ScopedContext ctx(env);
SetContextMessageConsumer(ctx.context, DiagnosticsMessageHandler);
spv_binary spirv_binary = nullptr;
std::unique_ptr<spv_markv_decoder_options_t,
std::function<void(spv_markv_decoder_options_t*)>> options(
spvMarkvDecoderOptionsCreate(), &spvMarkvDecoderOptionsDestroy);
ASSERT_EQ(SPV_SUCCESS, spvMarkvToSpirv(ctx.context, markv_binary->data,
markv_binary->length, options.get(),
&spirv_binary, nullptr, nullptr));
*words = std::vector<uint32_t>(
spirv_binary->code, spirv_binary->code + spirv_binary->wordCount);
spvBinaryDestroy(spirv_binary);
}
// Encodes/decodes |original|, assembles/dissasembles |original|, then compares
// the results of the two operations.
void TestEncodeDecode(const std::string& original_text) {
ScopedContext ctx(SPV_ENV_UNIVERSAL_1_2);
std::unique_ptr<spvtools::MarkvModel> model =
spvtools::CreateMarkvModel(spvtools::kMarkvModelShaderDefault);
spvtools::MarkvEncoderOptions encoder_options;
spvtools::MarkvDecoderOptions decoder_options;
std::vector<uint32_t> expected_binary;
Compile(original_text, &expected_binary);
ASSERT_FALSE(expected_binary.empty());
@ -141,17 +105,17 @@ void TestEncodeDecode(const std::string& original_text) {
SPV_TEXT_TO_BINARY_OPTION_PRESERVE_NUMERIC_IDS);
ASSERT_FALSE(binary_to_encode.empty());
spv_markv_binary markv_binary = nullptr;
std::vector<uint8_t> markv;
std::string encoder_comments;
Encode(binary_to_encode, &markv_binary, &encoder_comments);
ASSERT_NE(nullptr, markv_binary);
// std::cerr << encoder_comments << std::endl;
// std::cerr << "SPIR-V size: " << expected_binary.size() * 4 << std::endl;
// std::cerr << "MARK-V size: " << markv_binary->length << std::endl;
ASSERT_EQ(SPV_SUCCESS, spvtools::SpirvToMarkv(
ctx.context, binary_to_encode, encoder_options, *model,
DiagnosticsMessageHandler, &markv, &encoder_comments));
ASSERT_FALSE(markv.empty());
std::vector<uint32_t> decoded_binary;
Decode(markv_binary, &decoded_binary);
ASSERT_EQ(SPV_SUCCESS, spvtools::MarkvToSpirv(
ctx.context, markv, decoder_options, *model,
DiagnosticsMessageHandler, &decoded_binary, nullptr));
ASSERT_FALSE(decoded_binary.empty());
EXPECT_EQ(expected_binary, decoded_binary) << encoder_comments;
@ -161,8 +125,6 @@ void TestEncodeDecode(const std::string& original_text) {
ASSERT_FALSE(decoded_text.empty());
EXPECT_EQ(expected_text, decoded_text) << encoder_comments;
spvMarkvBinaryDestroy(markv_binary);
}
void TestEncodeDecodeShaderMainBody(const std::string& body) {

5
tools/CMakeLists.txt
View File

@ -61,7 +61,10 @@ if (NOT ${SPIRV_SKIP_EXECUTABLES})
spirv-cfg spirv-link)
if(SPIRV_BUILD_COMPRESSION)
add_spvtools_tool(TARGET spirv-markv SRCS comp/markv.cpp
add_spvtools_tool(TARGET spirv-markv
SRCS comp/markv.cpp
comp/markv_model_factory.cpp
comp/markv_model_shader_default.cpp
LIBS SPIRV-Tools-comp ${SPIRV_TOOLS})
target_include_directories(spirv-markv PRIVATE ${spirv-tools_SOURCE_DIR}
${SPIRV_HEADER_INCLUDE_DIR})

71
tools/comp/markv.cpp
View File

@ -20,9 +20,10 @@
#include <memory>
#include <vector>
#include "markv_model_factory.h"
#include "source/comp/markv.h"
#include "source/spirv_target_env.h"
#include "source/table.h"
#include "spirv-tools/markv.h"
#include "tools/io.h"
namespace {
@ -63,6 +64,7 @@ Options:
-h, --help Print this help.
--comments Write codec comments to stdout.
--version Display MARK-V codec version.
--validate Validate SPIR-V while encoding or decoding.
-o <filename> Set the output filename.
Output goes to standard output if this option is
@ -119,6 +121,7 @@ int main(int argc, char** argv) {
}
bool want_comments = false;
bool validate_spirv_binary = false;
for (int argi = 2; argi < argc; ++argi) {
if ('-' == argv[argi][0]) {
@ -143,6 +146,8 @@ int main(int argc, char** argv) {
} else if (0 == strcmp(argv[argi], "--version")) {
fprintf(stderr, "error: Not implemented\n");
return 1;
} else if (0 == strcmp(argv[argi], "--validate")) {
validate_spirv_binary = true;
} else {
print_usage(argv[0]);
return 1;
@ -179,69 +184,69 @@ int main(int argc, char** argv) {
const bool write_to_stdout = output_filename == nullptr ||
0 == strcmp(output_filename, "-");
spv_text comments = nullptr;
spv_text* comments_ptr = want_comments ? &comments : nullptr;
std::string comments;
std::string* comments_ptr = want_comments ? &comments : nullptr;
ScopedContext ctx(SPV_ENV_UNIVERSAL_1_2);
SetContextMessageConsumer(ctx.context, DiagnosticsMessageHandler);
std::unique_ptr<spvtools::MarkvModel> model =
spvtools::CreateMarkvModel(spvtools::kMarkvModelShaderDefault);
if (task == kEncode) {
std::vector<uint32_t> contents;
if (!ReadFile<uint32_t>(input_filename, "rb", &contents)) return 1;
std::vector<uint32_t> spirv;
if (!ReadFile<uint32_t>(input_filename, "rb", &spirv)) return 1;
std::unique_ptr<spv_markv_encoder_options_t,
std::function<void(spv_markv_encoder_options_t*)>> options(
spvMarkvEncoderOptionsCreate(), &spvMarkvEncoderOptionsDestroy);
spv_markv_binary markv_binary = nullptr;
spvtools::MarkvEncoderOptions options;
options.validate_spirv_binary = validate_spirv_binary;
if (SPV_SUCCESS !=
spvSpirvToMarkv(ctx.context, contents.data(), contents.size(),
options.get(), &markv_binary, comments_ptr, nullptr)) {
std::vector<uint8_t> markv;
if (SPV_SUCCESS != spvtools::SpirvToMarkv(
ctx.context, spirv, options, *model, DiagnosticsMessageHandler,
&markv, comments_ptr)) {
std::cerr << "error: Failed to encode " << input_filename << " to MARK-V "
<< std::endl;
return 1;
}
if (want_comments) {
if (!WriteFile<char>(nullptr, "w", comments->str,
comments->length)) return 1;
if (!WriteFile<char>(nullptr, "w", comments.c_str(),
comments.length())) return 1;
}
if (!want_comments || !write_to_stdout) {
if (!WriteFile<uint8_t>(output_filename, "wb", markv_binary->data,
markv_binary->length)) return 1;
if (!WriteFile<uint8_t>(output_filename, "wb", markv.data(),
markv.size())) return 1;
}
} else if (task == kDecode) {
std::vector<uint8_t> contents;
if (!ReadFile<uint8_t>(input_filename, "rb", &contents)) return 1;
std::vector<uint8_t> markv;
if (!ReadFile<uint8_t>(input_filename, "rb", &markv)) return 1;
std::unique_ptr<spv_markv_decoder_options_t,
std::function<void(spv_markv_decoder_options_t*)>> options(
spvMarkvDecoderOptionsCreate(), &spvMarkvDecoderOptionsDestroy);
spv_binary spirv_binary = nullptr;
spvtools::MarkvDecoderOptions options;
options.validate_spirv_binary = validate_spirv_binary;
if (SPV_SUCCESS !=
spvMarkvToSpirv(ctx.context, contents.data(), contents.size(),
options.get(), &spirv_binary, comments_ptr, nullptr)) {
std::cerr << "error: Failed to encode " << input_filename << " to MARK-V "
std::vector<uint32_t> spirv;
if (SPV_SUCCESS != spvtools::MarkvToSpirv(
ctx.context, markv, options, *model, DiagnosticsMessageHandler,
&spirv, comments_ptr)) {
std::cerr << "error: Failed to decode " << input_filename << " to SPIR-V "
<< std::endl;
return 1;
}
if (want_comments) {
if (!WriteFile<char>(nullptr, "w", comments->str,
comments->length)) return 1;
if (!WriteFile<char>(nullptr, "w", comments.c_str(),
comments.length())) return 1;
}
if (!want_comments || !write_to_stdout) {
if (!WriteFile<uint32_t>(output_filename, "wb", spirv_binary->code,
spirv_binary->wordCount)) return 1;
if (!WriteFile<uint32_t>(output_filename, "wb", spirv.data(),
spirv.size())) return 1;
}
} else {
assert(false && "Unknown task");
}
spvTextDestroy(comments);
return 0;
}

34
tools/comp/markv_model_factory.cpp Normal file
View File

@ -0,0 +1,34 @@
// Copyright (c) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "markv_model_factory.h"
#include "markv_model_shader_default.h"
namespace spvtools {
std::unique_ptr<MarkvModel> CreateMarkvModel(MarkvModelType type) {
std::unique_ptr<MarkvModel> model;
switch (type) {
case kMarkvModelShaderDefault: {
model.reset(new MarkvModelShaderDefault());
break;
}
}
model->SetModelType(static_cast<uint32_t>(type));
return model;
}
} // namespace spvtools

32
tools/comp/markv_model_factory.h Normal file
View File

@ -0,0 +1,32 @@
// Copyright (c) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef SPIRV_TOOLS_COMP_MARKV_MODEL_FACTORY_H_
#define SPIRV_TOOLS_COMP_MARKV_MODEL_FACTORY_H_
#include <memory>
#include "source/comp/markv_model.h"
namespace spvtools {
enum MarkvModelType {
kMarkvModelShaderDefault = 1,
};
std::unique_ptr<MarkvModel> CreateMarkvModel(MarkvModelType type);
} // namespace spvtools
#endif // SPIRV_TOOLS_COMP_MARKV_MODEL_FACTORY_H_

112
tools/comp/markv_model_shader_default.cpp Normal file
View File

@ -0,0 +1,112 @@
// Copyright (c) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "markv_model_shader_default.h"
#include <algorithm>
#include <map>
#include <memory>
#include <vector>
#include <unordered_map>
#include <unordered_set>
using spvutils::HuffmanCodec;
namespace spvtools {
namespace {
// Signals that the value is not in the coding scheme and a fallback method
// needs to be used.
const uint64_t kMarkvNoneOfTheAbove = MarkvModel::GetMarkvNoneOfTheAbove();
inline uint32_t CombineOpcodeAndNumOperands(uint32_t opcode,
uint32_t num_operands) {
return opcode | (num_operands << 16);
}
// The following file contains autogenerated statistical coding rules.
// Generated by running spirv-stats on representative corpus of shaders with
// flags:
// --codegen_opcode_and_num_operands_hist
// --codegen_opcode_and_num_operands_markov_huffman_codecs
// --codegen_literal_string_huffman_codecs
// --codegen_non_id_word_huffman_codecs
// --codegen_id_descriptor_huffman_codecs
//
// Example:
// find <SHADER_CORPUS_DIR> -type f -print0 | xargs -0 -s 2000000
// ~/SPIRV-Tools/build/tools/spirv-stats -v
// --codegen_opcode_and_num_operands_hist
// --codegen_opcode_and_num_operands_markov_huffman_codecs
// --codegen_literal_string_huffman_codecs --codegen_non_id_word_huffman_codecs
// --codegen_id_descriptor_huffman_codecs -o
// ~/SPIRV-Tools/source/comp/markv_autogen.inc
#include "markv_model_shader_default_autogen.inc"
} // namespace
MarkvModelShaderDefault::MarkvModelShaderDefault() {
const uint16_t kVersionNumber = 0;
SetModelVersion(kVersionNumber);
opcode_and_num_operands_huffman_codec_.reset(
new HuffmanCodec<uint64_t>(GetOpcodeAndNumOperandsHist()));
opcode_and_num_operands_markov_huffman_codecs_ =
GetOpcodeAndNumOperandsMarkovHuffmanCodecs();
non_id_word_huffman_codecs_ = GetNonIdWordHuffmanCodecs();
id_descriptor_huffman_codecs_ = GetIdDescriptorHuffmanCodecs();
descriptors_with_coding_scheme_ = GetDescriptorsWithCodingScheme();
literal_string_huffman_codecs_ = GetLiteralStringHuffmanCodecs();
operand_chunk_lengths_[SPV_OPERAND_TYPE_TYPE_ID] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_RESULT_ID] = 8;
operand_chunk_lengths_[SPV_OPERAND_TYPE_ID] = 8;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SCOPE_ID] = 8;
operand_chunk_lengths_[SPV_OPERAND_TYPE_MEMORY_SEMANTICS_ID] = 8;
operand_chunk_lengths_[SPV_OPERAND_TYPE_LITERAL_INTEGER] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_LITERAL_INTEGER] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_CAPABILITY] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SOURCE_LANGUAGE] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_EXECUTION_MODEL] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_ADDRESSING_MODEL] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_MEMORY_MODEL] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_EXECUTION_MODE] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_STORAGE_CLASS] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_DIMENSIONALITY] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SAMPLER_ADDRESSING_MODE] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SAMPLER_FILTER_MODE] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SAMPLER_IMAGE_FORMAT] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_FP_ROUNDING_MODE] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_LINKAGE_TYPE] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_ACCESS_QUALIFIER] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_ACCESS_QUALIFIER] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_FUNCTION_PARAMETER_ATTRIBUTE] = 3;
operand_chunk_lengths_[SPV_OPERAND_TYPE_DECORATION] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_BUILT_IN] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_GROUP_OPERATION] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_KERNEL_ENQ_FLAGS] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_KERNEL_PROFILING_INFO] = 2;
operand_chunk_lengths_[SPV_OPERAND_TYPE_FP_FAST_MATH_MODE] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_FUNCTION_CONTROL] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_LOOP_CONTROL] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_IMAGE] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_IMAGE] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_MEMORY_ACCESS] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_SELECTION_CONTROL] = 4;
operand_chunk_lengths_[SPV_OPERAND_TYPE_EXTENSION_INSTRUCTION_NUMBER] = 6;
operand_chunk_lengths_[SPV_OPERAND_TYPE_TYPED_LITERAL_NUMBER] = 6;
}
} // namespace spvtools

30
tools/comp/markv_model_shader_default.h Normal file
View File

@ -0,0 +1,30 @@
// Copyright (c) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef SPIRV_TOOLS_COMP_DEFAULT_SHADER_MARKV_MODEL_H_
#define SPIRV_TOOLS_COMP_DEFAULT_SHADER_MARKV_MODEL_H_
#include "source/comp/markv_model.h"
namespace spvtools {
// MARK-V model designed to be a default model for shader compression.
class MarkvModelShaderDefault : public MarkvModel {
public:
MarkvModelShaderDefault();
};
} // namespace spvtools
#endif // SPIRV_TOOLS_COMP_DEFAULT_SHADER_MARKV_MODEL_H_

0
source/comp/markv_autogen.inc → tools/comp/markv_model_shader_default_autogen.inc
View File

5
tools/stats/stats_analyzer.cpp
View File

@ -25,7 +25,7 @@
#include "spirv/1.2/spirv.h"
#include "source/enum_string_mapping.h"
#include "source/comp/markv_autogen.h"
#include "source/comp/markv_model.h"
#include "source/opcode.h"
#include "source/operand.h"
#include "source/spirv_constant.h"
@ -38,7 +38,8 @@ namespace {
// Signals that the value is not in the coding scheme and a fallback method
// needs to be used.
const uint64_t kMarkvNoneOfTheAbove = GetMarkvNonOfTheAbove();
const uint64_t kMarkvNoneOfTheAbove =
spvtools::MarkvModel::GetMarkvNoneOfTheAbove();
inline uint32_t CombineOpcodeAndNumOperands(uint32_t opcode,
uint32_t num_operands) {