From cfd95f3d5a732e9be9f193d35b04ae633940cfd7 Mon Sep 17 00:00:00 2001 From: Andrey Tuganov Date: Tue, 17 Oct 2017 17:38:04 -0400 Subject: [PATCH] Refactored compression debugger Markv codec now receives two optional callbacks: LogConsumer for internal codec logging DebugConsumer for testing if encoding->decoding produces the original results. --- source/comp/markv.h | 44 +++++--- source/comp/markv_codec.cpp | 131 +++++++++++++++------- source/util/bit_stream.cpp | 2 + source/util/bit_stream.h | 17 +++ test/comp/markv_codec_test.cpp | 24 ++-- tools/CMakeLists.txt | 2 +- tools/comp/markv.cpp | 197 +++++++++++++++++++++++++-------- 7 files changed, 303 insertions(+), 114 deletions(-) diff --git a/source/comp/markv.h b/source/comp/markv.h index 063c210ef..b3e27bbd2 100644 --- a/source/comp/markv.h +++ b/source/comp/markv.h @@ -29,35 +29,51 @@ namespace spvtools { -struct MarkvEncoderOptions { +struct MarkvCodecOptions { bool validate_spirv_binary = false; }; -struct MarkvDecoderOptions { - bool validate_spirv_binary = false; -}; +// Debug callback. Called once per instruction. +// |words| is instruction SPIR-V words. +// |bits| is a textual representation of the MARK-V bit sequence used to encode +// the instruction (char '0' for 0, char '1' for 1). +// |comment| contains all logs generated while processing the instruction. +using MarkvDebugConsumer = std::function& words, const std::string& bits, + const std::string& comment)>; + +// Logging callback. Called often (if decoder reads a single bit, the log +// consumer will receive 1 character string with that bit). +// This callback is more suitable for continous output than MarkvDebugConsumer, +// for example if the codec crashes it would allow to pinpoint on which operand +// or bit the crash happened. +// |snippet| could be any atomic fragment of text logged by the codec. It can +// contain a paragraph of text with newlines, or can be just one character. +using MarkvLogConsumer = std::function; // Encodes the given SPIR-V binary to MARK-V binary. -// If |comments| is not nullptr, it would contain a textual description of -// how encoding was done (with snippets of disassembly and bit sequences). +// |log_consumer| is optional (pass MarkvLogConsumer() to disable). +// |debug_consumer| is optional (pass MarkvDebugConsumer() to disable). spv_result_t SpirvToMarkv(spv_const_context context, const std::vector& spirv, - const MarkvEncoderOptions& options, + const MarkvCodecOptions& options, const MarkvModel& markv_model, MessageConsumer message_consumer, - std::vector* markv, - std::string* comments); + MarkvLogConsumer log_consumer, + MarkvDebugConsumer debug_consumer, + std::vector* markv); // Decodes a SPIR-V binary from the given MARK-V binary. -// If |comments| is not nullptr, it would contain a textual description of -// how decoding was done (with snippets of disassembly and bit sequences). +// |log_consumer| is optional (pass MarkvLogConsumer() to disable). +// |debug_consumer| is optional (pass MarkvDebugConsumer() to disable). spv_result_t MarkvToSpirv(spv_const_context context, const std::vector& markv, - const MarkvDecoderOptions& options, + const MarkvCodecOptions& options, const MarkvModel& markv_model, MessageConsumer message_consumer, - std::vector* spirv, - std::string* comments); + MarkvLogConsumer log_consumer, + MarkvDebugConsumer debug_consumer, + std::vector* spirv); } // namespace spvtools diff --git a/source/comp/markv_codec.cpp b/source/comp/markv_codec.cpp index 9ab4ef86b..46fb09b5e 100644 --- a/source/comp/markv_codec.cpp +++ b/source/comp/markv_codec.cpp @@ -64,7 +64,6 @@ using libspirv::IdDescriptorCollection; using libspirv::Instruction; using libspirv::ValidationState_t; using libspirv::DiagnosticStream; -using spvtools::ValidateInstructionAndUpdateValidationState; using spvutils::BitReaderWord64; using spvutils::BitWriterWord64; using spvutils::HuffmanCodec; @@ -276,8 +275,11 @@ uint32_t GetMarkvVersion() { return kVersionMinor | (kVersionMajor << 16); } -class CommentLogger { +class MarkvLogger { public: + MarkvLogger(MarkvLogConsumer log_consumer, MarkvDebugConsumer debug_consumer) + : log_consumer_(log_consumer), debug_consumer_(debug_consumer) {} + void AppendText(const std::string& str) { Append(str); use_delimiter_ = false; @@ -290,6 +292,8 @@ class CommentLogger { } void AppendBitSequence(const std::string& str) { + if (debug_consumer_) + instruction_bits_ << str; if (use_delimiter_) Append("-"); Append(str); @@ -306,17 +310,36 @@ class CommentLogger { use_delimiter_ = false; } - std::string GetText() const { - return ss_.str(); + bool DebugInstruction(const spv_parsed_instruction_t& inst) { + bool result = true; + if (debug_consumer_) { + result = debug_consumer_( + std::vector(inst.words, inst.words + inst.num_words), + instruction_bits_.str(), instruction_comment_.str()); + instruction_bits_.str(std::string()); + instruction_comment_.str(std::string()); + } + return result; } private: + MarkvLogger(const MarkvLogger&) = delete; + MarkvLogger(MarkvLogger&&) = delete; + MarkvLogger& operator=(const MarkvLogger&) = delete; + MarkvLogger& operator=(MarkvLogger&&) = delete; + void Append(const std::string& str) { - ss_ << str; - // std::cerr << str; + if (log_consumer_) + log_consumer_(str); + if (debug_consumer_) + instruction_comment_ << str; } - std::stringstream ss_; + MarkvLogConsumer log_consumer_; + MarkvDebugConsumer debug_consumer_; + + std::stringstream instruction_bits_; + std::stringstream instruction_comment_; // If true a delimiter will be appended before the next bit sequence. // Used to generate outputs like: 1100-0 1110-1-1100-1-1111-0 110-0. @@ -547,6 +570,9 @@ class MarkvCodecBase { std::map>> mtf_huffman_codecs_; + // If not nullptr, codec will log comments on the compression process. + std::unique_ptr logger_; + private: spv_const_context context_ = nullptr; @@ -572,7 +598,7 @@ class MarkvEncoder : public MarkvCodecBase { // |model| is owned by the caller, must be not null and valid during the // lifetime of MarkvEncoder. MarkvEncoder(spv_const_context context, - const MarkvEncoderOptions& options, + const MarkvCodecOptions& options, const MarkvModel* model) : MarkvCodecBase(context, GetValidatorOptions(options), model), options_(options) { @@ -590,6 +616,15 @@ class MarkvEncoder : public MarkvCodecBase { return SPV_SUCCESS; } + // Creates an internal logger which writes comments on the encoding process. + void CreateLogger(MarkvLogConsumer log_consumer, + MarkvDebugConsumer debug_consumer) { + logger_.reset(new MarkvLogger(log_consumer, debug_consumer)); + writer_.SetCallback([this](const std::string& str){ + logger_->AppendBitSequence(str); + }); + } + // Encodes SPIR-V instruction to MARK-V and writes to bit stream. // Operation can fail if the instruction fails to pass the validator or if // the encoder stubmles on something unexpected. @@ -615,15 +650,6 @@ class MarkvEncoder : public MarkvCodecBase { return markv; } - // Creates an internal logger which writes comments on the encoding process. - // Output can later be accessed with GetComments(). - void CreateCommentsLogger() { - logger_.reset(new CommentLogger()); - writer_.SetCallback([this](const std::string& str){ - logger_->AppendBitSequence(str); - }); - } - // Optionally adds disassembly to the comments. // Disassembly should contain all instructions in the module separated by // \n, and no header. @@ -640,17 +666,10 @@ class MarkvEncoder : public MarkvCodecBase { } } - // Extracts the text from the comment logger. - std::string GetComments() const { - if (!logger_) - return ""; - return logger_->GetText(); - } - private: // Creates and returns validator options. Returned value owned by the caller. static spv_validator_options GetValidatorOptions( - const MarkvEncoderOptions& options) { + const MarkvCodecOptions& options) { return options.validate_spirv_binary ? spvValidatorOptionsCreate() : nullptr; } @@ -692,14 +711,11 @@ class MarkvEncoder : public MarkvCodecBase { // Encodes a literal number operand and writes it to the bit stream. spv_result_t EncodeLiteralNumber(const spv_parsed_operand_t& operand); - MarkvEncoderOptions options_; + MarkvCodecOptions options_; // Bit stream where encoded instructions are written. BitWriterWord64 writer_; - // If not nullptr, encoder will write comments. - std::unique_ptr logger_; - // If not nullptr, disassembled instruction lines will be written to comments. // Format: \n separated instruction lines, no header. std::unique_ptr disassembly_; @@ -712,7 +728,7 @@ class MarkvDecoder : public MarkvCodecBase { // lifetime of MarkvEncoder. MarkvDecoder(spv_const_context context, const std::vector& markv, - const MarkvDecoderOptions& options, + const MarkvCodecOptions& options, const MarkvModel* model) : MarkvCodecBase(context, GetValidatorOptions(options), model), options_(options), reader_(markv) { @@ -722,6 +738,12 @@ class MarkvDecoder : public MarkvCodecBase { inst_words_.reserve(25); } + // Creates an internal logger which writes comments on the decoding process. + void CreateLogger(MarkvLogConsumer log_consumer, + MarkvDebugConsumer debug_consumer) { + logger_.reset(new MarkvLogger(log_consumer, debug_consumer)); + } + // Decodes SPIR-V from MARK-V and stores the words in |spirv_binary|. // Can be called only once. Fails if data of wrong format or ends prematurely, // of if validation fails. @@ -736,7 +758,7 @@ class MarkvDecoder : public MarkvCodecBase { // Creates and returns validator options. Returned value owned by the caller. static spv_validator_options GetValidatorOptions( - const MarkvDecoderOptions& options) { + const MarkvCodecOptions& options) { return options.validate_spirv_binary ? spvValidatorOptionsCreate() : nullptr; } @@ -822,7 +844,7 @@ class MarkvDecoder : public MarkvCodecBase { // kind SPV_NUMBER_NONE. void RecordNumberType(); - MarkvDecoderOptions options_; + MarkvCodecOptions options_; // Temporary sink where decoded SPIR-V words are written. Once it contains the // entire module, the container is moved and returned. @@ -2260,6 +2282,8 @@ spv_result_t MarkvEncoder::EncodeInstruction( if (logger_) { logger_->NewLine(); logger_->NewLine(); + if (!logger_->DebugInstruction(inst_)) + return SPV_REQUESTED_TERMINATION; } ProcessCurInstruction(); @@ -2310,6 +2334,12 @@ spv_result_t MarkvDecoder::DecodeModule(std::vector* spirv_binary) { spirv_[1] = header_.spirv_version; spirv_[2] = header_.spirv_generator; + if (logger_) { + reader_.SetCallback([this](const std::string& str){ + logger_->AppendBitSequence(str); + }); + } + while (reader_.GetNumReadBits() < header_.markv_length_in_bits) { inst_ = {}; const spv_result_t decode_result = DecodeInstruction(); @@ -2777,6 +2807,19 @@ spv_result_t MarkvDecoder::DecodeInstruction() { return Diag(SPV_ERROR_INVALID_BINARY) << "Failed to read to byte break"; + if (logger_) { + logger_->NewLine(); + std::stringstream ss; + ss << spvOpcodeString(opcode) << " "; + for (size_t index = 1; index < inst_words_.size(); ++index) + ss << inst_words_[index] << " "; + logger_->AppendText(ss.str()); + logger_->NewLine(); + logger_->NewLine(); + if (!logger_->DebugInstruction(inst_)) + return SPV_REQUESTED_TERMINATION; + } + return SPV_SUCCESS; } @@ -2839,11 +2882,12 @@ spv_result_t EncodeInstruction( spv_result_t SpirvToMarkv(spv_const_context context, const std::vector& spirv, - const MarkvEncoderOptions& options, + const MarkvCodecOptions& options, const MarkvModel& markv_model, MessageConsumer message_consumer, - std::vector* markv, - std::string* comments) { + MarkvLogConsumer log_consumer, + MarkvDebugConsumer debug_consumer, + std::vector* markv) { spv_context_t hijack_context = *context; SetContextMessageConsumer(&hijack_context, message_consumer); @@ -2866,8 +2910,8 @@ spv_result_t SpirvToMarkv(spv_const_context context, MarkvEncoder encoder(&hijack_context, options, &markv_model); - if (comments) { - encoder.CreateCommentsLogger(); + if (log_consumer || debug_consumer) { + encoder.CreateLogger(log_consumer, debug_consumer); spv_text text = nullptr; if (spvBinaryToText(&hijack_context, spirv.data(), spirv.size(), @@ -2890,26 +2934,27 @@ spv_result_t SpirvToMarkv(spv_const_context context, << "Unable to encode to MARK-V."; } - if (comments) - *comments = encoder.GetComments(); - *markv = encoder.GetMarkvBinary(); return SPV_SUCCESS; } spv_result_t MarkvToSpirv(spv_const_context context, const std::vector& markv, - const MarkvDecoderOptions& options, + const MarkvCodecOptions& options, const MarkvModel& markv_model, MessageConsumer message_consumer, - std::vector* spirv, - std::string* /* comments */) { + MarkvLogConsumer log_consumer, + MarkvDebugConsumer debug_consumer, + std::vector* spirv) { spv_position_t position = {}; spv_context_t hijack_context = *context; SetContextMessageConsumer(&hijack_context, message_consumer); MarkvDecoder decoder(&hijack_context, markv, options, &markv_model); + if (log_consumer || debug_consumer) + decoder.CreateLogger(log_consumer, debug_consumer); + if (decoder.DecodeModule(spirv) != SPV_SUCCESS) { return DiagnosticStream(position, hijack_context.consumer, SPV_ERROR_INVALID_BINARY) diff --git a/source/util/bit_stream.cpp b/source/util/bit_stream.cpp index 56f370054..d66f13dc9 100644 --- a/source/util/bit_stream.cpp +++ b/source/util/bit_stream.cpp @@ -414,6 +414,7 @@ size_t BitReaderWord64::ReadBits(uint64_t* bits, size_t num_bits) { if (pos_ >= buffer_.size() * 64) { // Reached end of buffer_. + EmitSequence(*bits, num_read_from_first_word); return num_read_from_first_word; } @@ -426,6 +427,7 @@ size_t BitReaderWord64::ReadBits(uint64_t* bits, size_t num_bits) { // We likely have written more bits than requested. Clear excessive bits. *bits = GetLowerBits(*bits, num_bits); + EmitSequence(*bits, num_bits); return num_bits; } diff --git a/source/util/bit_stream.h b/source/util/bit_stream.h index b626d2c7c..f98b74bf8 100644 --- a/source/util/bit_stream.h +++ b/source/util/bit_stream.h @@ -436,9 +436,26 @@ class BitReaderWord64 : public BitReaderInterface { bool OnlyZeroesLeft() const override; BitReaderWord64() = delete; + + // Sets callback to emit bit sequences after every read. + void SetCallback(std::function callback) { + callback_ = callback; + } + + protected: + // Sends string generated from arguments to callback_ if defined. + void EmitSequence(uint64_t bits, size_t num_bits) const { + if (callback_) + callback_(BitsToStream(bits, num_bits)); + } + private: const std::vector buffer_; size_t pos_; + + // If not null, the reader will use the callback to emit the read bit + // sequence as a string of '0' and '1'. + std::function callback_; }; } // namespace spvutils diff --git a/test/comp/markv_codec_test.cpp b/test/comp/markv_codec_test.cpp index 09d39a8c0..46dde8528 100644 --- a/test/comp/markv_codec_test.cpp +++ b/test/comp/markv_codec_test.cpp @@ -89,8 +89,7 @@ void TestEncodeDecode(const std::string& original_text) { ScopedContext ctx(SPV_ENV_UNIVERSAL_1_2); std::unique_ptr model = spvtools::CreateMarkvModel(spvtools::kMarkvModelShaderDefault); - spvtools::MarkvEncoderOptions encoder_options; - spvtools::MarkvDecoderOptions decoder_options; + spvtools::MarkvCodecOptions options; std::vector expected_binary; Compile(original_text, &expected_binary); @@ -105,26 +104,33 @@ void TestEncodeDecode(const std::string& original_text) { SPV_TEXT_TO_BINARY_OPTION_PRESERVE_NUMERIC_IDS); ASSERT_FALSE(binary_to_encode.empty()); + std::stringstream encoder_comments; + const auto output_to_string_stream = + [&encoder_comments](const std::string& str) { + encoder_comments << str; + }; + std::vector markv; - std::string encoder_comments; ASSERT_EQ(SPV_SUCCESS, spvtools::SpirvToMarkv( - ctx.context, binary_to_encode, encoder_options, *model, - DiagnosticsMessageHandler, &markv, &encoder_comments)); + ctx.context, binary_to_encode, options, *model, + DiagnosticsMessageHandler, output_to_string_stream, + spvtools::MarkvDebugConsumer(), &markv)); ASSERT_FALSE(markv.empty()); std::vector decoded_binary; ASSERT_EQ(SPV_SUCCESS, spvtools::MarkvToSpirv( - ctx.context, markv, decoder_options, *model, - DiagnosticsMessageHandler, &decoded_binary, nullptr)); + ctx.context, markv, options, *model, + DiagnosticsMessageHandler, spvtools::MarkvLogConsumer(), + spvtools::MarkvDebugConsumer(), &decoded_binary)); ASSERT_FALSE(decoded_binary.empty()); - EXPECT_EQ(expected_binary, decoded_binary) << encoder_comments; + EXPECT_EQ(expected_binary, decoded_binary) << encoder_comments.str(); std::string decoded_text; Disassemble(decoded_binary, &decoded_text); ASSERT_FALSE(decoded_text.empty()); - EXPECT_EQ(expected_text, decoded_text) << encoder_comments; + EXPECT_EQ(expected_text, decoded_text) << encoder_comments.str(); } void TestEncodeDecodeShaderMainBody(const std::string& body) { diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 1219720f5..f650b3f3c 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -65,7 +65,7 @@ if (NOT ${SPIRV_SKIP_EXECUTABLES}) SRCS comp/markv.cpp comp/markv_model_factory.cpp comp/markv_model_shader_default.cpp - LIBS SPIRV-Tools-comp ${SPIRV_TOOLS}) + LIBS SPIRV-Tools-comp SPIRV-Tools-opt ${SPIRV_TOOLS}) target_include_directories(spirv-markv PRIVATE ${spirv-tools_SOURCE_DIR} ${SPIRV_HEADER_INCLUDE_DIR}) set(SPIRV_INSTALL_TARGETS ${SPIRV_INSTALL_TARGETS} spirv-markv) diff --git a/tools/comp/markv.cpp b/tools/comp/markv.cpp index b107cdd24..23585e5b9 100644 --- a/tools/comp/markv.cpp +++ b/tools/comp/markv.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -24,14 +25,18 @@ #include "source/comp/markv.h" #include "source/spirv_target_env.h" #include "source/table.h" +#include "spirv-tools/optimizer.hpp" #include "tools/io.h" namespace { +const auto kSpvEnv = SPV_ENV_UNIVERSAL_1_2; + enum Task { kNoTask = 0, kEncode, kDecode, + kTest, }; struct ScopedContext { @@ -44,7 +49,7 @@ void print_usage(char* argv0) { printf( R"(%s - Encodes or decodes a SPIR-V binary to or from a MARK-V binary. -USAGE: %s [e|d] [options] [] +USAGE: %s [e|d|t] [options] [] The input binary is read from . If no file is specified, or if the filename is "-", then the binary is read from standard input. @@ -59,16 +64,19 @@ software is used (is doesn't write or handle version numbers yet). Tasks: e Encode SPIR-V to MARK-V. d Decode MARK-V to SPIR-V. + t Test the codec by first encoding the given SPIR-V file to + MARK-V, then decoding it back to SPIR-V and comparing results. Options: -h, --help Print this help. - --comments Write codec comments to stdout. + --comments Write codec comments to stderr. --version Display MARK-V codec version. --validate Validate SPIR-V while encoding or decoding. -o Set the output filename. Output goes to standard output if this option is not specified, or if the filename is "-". + Not needed for 't' task (testing). )", argv0, argv0); } @@ -84,11 +92,11 @@ void DiagnosticsMessageHandler(spv_message_level_t level, const char*, << std::endl; break; case SPV_MSG_WARNING: - std::cout << "warning: " << position.index << ": " << message + std::cerr << "warning: " << position.index << ": " << message << std::endl; break; case SPV_MSG_INFO: - std::cout << "info: " << position.index << ": " << message << std::endl; + std::cerr << "info: " << position.index << ": " << message << std::endl; break; default: break; @@ -113,6 +121,8 @@ int main(int argc, char** argv) { task = kEncode; } else if (0 == strcmp("d", task_char)) { task = kDecode; + } else if (0 == strcmp("t", task_char)) { + task = kTest; } if (task == kNoTask) { @@ -130,7 +140,8 @@ int main(int argc, char** argv) { print_usage(argv[0]); return 0; case 'o': { - if (!output_filename && argi + 1 < argc) { + if (!output_filename && argi + 1 < argc && + (task == kEncode || task == kDecode)) { output_filename = argv[++argi]; } else { print_usage(argv[0]); @@ -176,76 +187,168 @@ int main(int argc, char** argv) { } } - if (task == kDecode && want_comments) { - fprintf(stderr, "warning: Decoder comments not yet implemented\n"); - want_comments = false; - } + const auto no_comments = spvtools::MarkvLogConsumer(); + const auto output_to_stderr = [](const std::string& str) { + std::cerr << str; + }; - const bool write_to_stdout = output_filename == nullptr || - 0 == strcmp(output_filename, "-"); - - std::string comments; - std::string* comments_ptr = want_comments ? &comments : nullptr; - - ScopedContext ctx(SPV_ENV_UNIVERSAL_1_2); + ScopedContext ctx(kSpvEnv); std::unique_ptr model = spvtools::CreateMarkvModel(spvtools::kMarkvModelShaderDefault); + std::vector spirv; + std::vector markv; + + spvtools::MarkvCodecOptions options; + options.validate_spirv_binary = validate_spirv_binary; + if (task == kEncode) { - std::vector spirv; if (!ReadFile(input_filename, "rb", &spirv)) return 1; - - spvtools::MarkvEncoderOptions options; - options.validate_spirv_binary = validate_spirv_binary; - - std::vector markv; + assert(!spirv.empty()); if (SPV_SUCCESS != spvtools::SpirvToMarkv( ctx.context, spirv, options, *model, DiagnosticsMessageHandler, - &markv, comments_ptr)) { + want_comments ? output_to_stderr : no_comments, + spvtools::MarkvDebugConsumer(), &markv)) { std::cerr << "error: Failed to encode " << input_filename << " to MARK-V " << std::endl; return 1; } - if (want_comments) { - if (!WriteFile(nullptr, "w", comments.c_str(), - comments.length())) return 1; - } - - if (!want_comments || !write_to_stdout) { - if (!WriteFile(output_filename, "wb", markv.data(), - markv.size())) return 1; - } + if (!WriteFile(output_filename, "wb", markv.data(), + markv.size())) return 1; } else if (task == kDecode) { - std::vector markv; if (!ReadFile(input_filename, "rb", &markv)) return 1; - - spvtools::MarkvDecoderOptions options; - options.validate_spirv_binary = validate_spirv_binary; - - std::vector spirv; + assert(!markv.empty()); if (SPV_SUCCESS != spvtools::MarkvToSpirv( ctx.context, markv, options, *model, DiagnosticsMessageHandler, - &spirv, comments_ptr)) { + want_comments ? output_to_stderr : no_comments, + spvtools::MarkvDebugConsumer(), &spirv)) { std::cerr << "error: Failed to decode " << input_filename << " to SPIR-V " << std::endl; return 1; } - if (want_comments) { - if (!WriteFile(nullptr, "w", comments.c_str(), - comments.length())) return 1; + if (!WriteFile(output_filename, "wb", spirv.data(), + spirv.size())) return 1; + } else if (task == kTest) { + if (!ReadFile(input_filename, "rb", &spirv)) return 1; + assert(!spirv.empty()); + + std::vector spirv_before; + spvtools::Optimizer optimizer(kSpvEnv); + optimizer.RegisterPass(spvtools::CreateCompactIdsPass()); + if (!optimizer.Run(spirv.data(), spirv.size(), &spirv_before)) { + std::cerr << "error: Optimizer failure on: " + << input_filename << std::endl; } - if (!want_comments || !write_to_stdout) { - if (!WriteFile(output_filename, "wb", spirv.data(), - spirv.size())) return 1; + std::vector encoder_instruction_bits; + std::vector encoder_instruction_comments; + std::vector> encoder_instruction_words; + std::vector decoder_instruction_bits; + std::vector decoder_instruction_comments; + std::vector> decoder_instruction_words; + + const auto encoder_debug_consumer = [&]( + const std::vector& words, const std::string& bits, + const std::string& comment) { + encoder_instruction_words.push_back(words); + encoder_instruction_bits.push_back(bits); + encoder_instruction_comments.push_back(comment); + return true; + }; + + if (SPV_SUCCESS != spvtools::SpirvToMarkv( + ctx.context, spirv_before, options, *model, DiagnosticsMessageHandler, + want_comments ? output_to_stderr : no_comments, + encoder_debug_consumer, &markv)) { + std::cerr << "error: Failed to encode " << input_filename << " to MARK-V " + << std::endl; + return 1; } - } else { - assert(false && "Unknown task"); + + const auto write_bug_report = [&]() { + for (size_t inst_index = 0; inst_index < decoder_instruction_words.size(); + ++inst_index) { + std::cerr << "\nInstruction #" << inst_index << std::endl; + std::cerr << "\nEncoder words: "; + for (uint32_t word : encoder_instruction_words[inst_index]) + std::cerr << word << " "; + std::cerr << "\nDecoder words: "; + for (uint32_t word : decoder_instruction_words[inst_index]) + std::cerr << word << " "; + std::cerr << std::endl; + + std::cerr << "\nEncoder bits: " << encoder_instruction_bits[inst_index]; + std::cerr << "\nDecoder bits: " << decoder_instruction_bits[inst_index]; + std::cerr << std::endl; + + std::cerr << "\nEncoder comments:\n" + << encoder_instruction_comments[inst_index]; + std::cerr << "Decoder comments:\n" + << decoder_instruction_comments[inst_index]; + std::cerr << std::endl; + } + }; + + const auto decoder_debug_consumer = [&]( + const std::vector& words, const std::string& bits, + const std::string& comment) { + const size_t inst_index = decoder_instruction_words.size(); + if (inst_index >= encoder_instruction_words.size()) { + write_bug_report(); + std::cerr << "error: Decoder has more instructions than encoder: " + << input_filename << std::endl; + return false; + } + + decoder_instruction_words.push_back(words); + decoder_instruction_bits.push_back(bits); + decoder_instruction_comments.push_back(comment); + + if (encoder_instruction_words[inst_index] != + decoder_instruction_words[inst_index]) { + write_bug_report(); + std::cerr << "error: Words of the last decoded instruction differ from " + "reference: " << input_filename << std::endl; + return false; + } + + if (encoder_instruction_bits[inst_index] != + decoder_instruction_bits[inst_index]) { + write_bug_report(); + std::cerr << "error: Bits of the last decoded instruction differ from " + "reference: " << input_filename << std::endl; + return false; + } + return true; + }; + + std::vector spirv_after; + const spv_result_t decoding_result = spvtools::MarkvToSpirv( + ctx.context, markv, options, *model, DiagnosticsMessageHandler, + want_comments ? output_to_stderr : no_comments, + decoder_debug_consumer, &spirv_after); + + if (decoding_result == SPV_REQUESTED_TERMINATION) { + std::cerr << "error: Decoding interrupted by the debugger: " + << input_filename << std::endl; + return 1; + } + + if (decoding_result != SPV_SUCCESS) { + std::cerr << "error: Failed to decode encoded " << input_filename + << " back to SPIR-V " << std::endl; + return 1; + } + + assert(spirv_before.size() == spirv_after.size()); + assert(std::mismatch(std::next(spirv_before.begin(), 5), spirv_before.end(), + std::next(spirv_after.begin(), 5)) == + std::make_pair(spirv_before.end(), spirv_after.end())); } return 0;