Updated markv_autogen

- now includes a table of all descriptors with coding scheme
(improves performance by 5% by allowing to avoid creation of
move-to-front sequences which will never be used)
- increased the size of markv_autogen.inc, clang doesn't seem
to have the long compilation time problem now
(probably was inadvertently fixed by using Huffman codec
serialization)

source/comp/markv_autogen.cpp

@@ -19,6 +19,7 @@
 #include <memory>
 #include <vector>
 #include <unordered_map>
+#include <unordered_set>
 
 #include "spirv/1.2/spirv.h"
 

source/comp/markv_autogen.h

@@ -18,6 +18,7 @@
 #include <map>
 #include <memory>
 #include <numeric>
+#include <unordered_set>
 
 #include "util/huffman_codec.h"
 
@@ -26,20 +27,34 @@ inline uint64_t GetMarkvNonOfTheAbove() {
   return 1111111111111111111;
 }
 
+// Returns of histogram of CombineOpcodeAndNumOperands(opcode, num_operands).
 std::map<uint64_t, uint32_t> GetOpcodeAndNumOperandsHist();
 
+// Returns Huffman codecs based on a Markov chain of histograms of
+// CombineOpcodeAndNumOperands(opcode, num_operands).
+// Map prev_opcode -> codec.
 std::map<uint32_t, std::unique_ptr<spvutils::HuffmanCodec<uint64_t>>>
-GetOpcodeAndNumOperandsMarkovHuffmanCodecs();
+    GetOpcodeAndNumOperandsMarkovHuffmanCodecs();
 
+// Returns Huffman codecs for literal strings.
+// Map opcode -> codec.
 std::map<uint32_t, std::unique_ptr<spvutils::HuffmanCodec<std::string>>>
-GetLiteralStringHuffmanCodecs();
+    GetLiteralStringHuffmanCodecs();
 
+// Returns Huffman codecs for single-word non-id operand slots.
+// Map <opcode, operand_index> -> codec.
 std::map<std::pair<uint32_t, uint32_t>,
     std::unique_ptr<spvutils::HuffmanCodec<uint64_t>>>
     GetNonIdWordHuffmanCodecs();
 
-    std::map<std::pair<uint32_t, uint32_t>,
+// Returns Huffman codecs for id descriptors used by common operand slots.
+// Map <opcode, operand_index> -> codec.
+std::map<std::pair<uint32_t, uint32_t>,
     std::unique_ptr<spvutils::HuffmanCodec<uint64_t>>>
     GetIdDescriptorHuffmanCodecs();
 
+// Returns a set of all descriptors which are encodable by at least one codec
+// returned by GetIdDescriptorHuffmanCodecs().
+std::unordered_set<uint32_t> GetDescriptorsWithCodingScheme();
+
 #endif  // LIBSPIRV_COMP_MARKV_AUTOGEN_H_

source/comp/markv_codec.cpp

@@ -34,6 +34,8 @@
 #include <numeric>
 #include <string>
 #include <vector>
+#include <unordered_map>
+#include <unordered_set>
 
 #include "spirv/1.2/GLSL.std.450.h"
 #include "spirv/1.2/OpenCL.std.h"
@@ -213,6 +215,7 @@ class MarkvModel {
             GetOpcodeAndNumOperandsMarkovHuffmanCodecs()),
         non_id_word_huffman_codecs_(GetNonIdWordHuffmanCodecs()),
         id_descriptor_huffman_codecs_(GetIdDescriptorHuffmanCodecs()),
+        descriptors_with_coding_scheme_(GetDescriptorsWithCodingScheme()),
         literal_string_huffman_codecs_(GetLiteralStringHuffmanCodecs()) {}
 
   size_t opcode_chunk_length() const { return 7; }
@@ -290,6 +293,10 @@ class MarkvModel {
     return it->second.get();
   }
 
+  bool DescriptorHasCodingScheme(uint32_t descriptor) const {
+    return descriptors_with_coding_scheme_.count(descriptor);
+  }
+
  private:
   // Huffman codecs for move-to-front ranks. The map key is mtf handle. Doesn't
   // need to contain a different codec for every handle as most use one and the
@@ -316,6 +323,8 @@ class MarkvModel {
       std::unique_ptr<HuffmanCodec<uint64_t>>>
       id_descriptor_huffman_codecs_;
 
+  std::unordered_set<uint32_t> descriptors_with_coding_scheme_;
+
   // Huffman codecs for literal strings. The map key is the opcode of the
   // current instruction. This assumes, that there is no more than one literal
   // string operand per instruction, but would still work even if this is not
@@ -471,7 +480,7 @@ size_t GetNumBitsToNextByte(size_t bit_pos) {
 // Defines and returns current MARK-V version.
 uint32_t GetMarkvVersion() {
   const uint32_t kVersionMajor = 1;
-  const uint32_t kVersionMinor = 1;
+  const uint32_t kVersionMinor = 2;
   return kVersionMinor | (kVersionMajor << 16);
 }
 
@@ -1196,8 +1205,8 @@ void MarkvCodecBase::ProcessCurInstruction() {
   }
 
   const uint32_t descriptor = id_descriptors_.ProcessInstruction(inst_);
-
-  multi_mtf_.Insert(GetMtfIdDescriptor(descriptor), inst_.result_id);
+  if (model_->DescriptorHasCodingScheme(descriptor))
+    multi_mtf_.Insert(GetMtfIdDescriptor(descriptor), inst_.result_id);
 }
 
 uint64_t MarkvCodecBase::GetRuleBasedMtf() {

tools/stats/stats_analyzer.cpp

@@ -19,6 +19,8 @@
 #include <cstring>
 #include <iostream>
 #include <sstream>
+#include <unordered_map>
+#include <unordered_set>
 #include <vector>
 
 #include "spirv/1.2/spirv.h"
@@ -647,7 +649,7 @@ void StatsAnalyzer::WriteCodegenOpcodeAndNumOperandsMarkovHuffmanCodecs(
 
   for (const auto& kv : stats_.opcode_and_num_operands_markov_hist) {
     const uint32_t prev_opcode = kv.first;
-    const double kFrequentEnoughToAnalyze = 0.01;
+    const double kFrequentEnoughToAnalyze = 0.001;
     if (opcode_freq_[prev_opcode] < kFrequentEnoughToAnalyze) continue;
 
     const std::unordered_map<uint32_t, uint32_t>& hist = kv.second;
@@ -783,7 +785,7 @@ void StatsAnalyzer::WriteCodegenNonIdWordHuffmanCodecs(std::ostream& out) {
       const uint32_t word = pair.first;
       const uint32_t count = pair.second;
       const double freq = double(count) / double(total);
-      const double kWordFrequentEnoughToAnalyze = 0.01;
+      const double kWordFrequentEnoughToAnalyze = 0.003;
       if (freq < kWordFrequentEnoughToAnalyze) {
         left_out += count;
         continue;
@@ -819,12 +821,14 @@ void StatsAnalyzer::WriteCodegenIdDescriptorHuffmanCodecs(
       << "  std::map<std::pair<uint32_t, uint32_t>, "
       << "std::unique_ptr<HuffmanCodec<uint64_t>>> codecs;\n";
 
+  std::unordered_set<uint32_t> descriptors_with_coding_scheme;
+
   for (const auto& kv : stats_.operand_slot_id_descriptor_hist) {
     const auto& opcode_and_index = kv.first;
     const uint32_t opcode = opcode_and_index.first;
     const uint32_t index = opcode_and_index.second;
 
-    const double kOpcodeFrequentEnoughToAnalyze = 0.01;
+    const double kOpcodeFrequentEnoughToAnalyze = 0.003;
     if (opcode_freq_[opcode] < kOpcodeFrequentEnoughToAnalyze) continue;
 
     const std::map<uint32_t, uint32_t>& hist = kv.second;
@@ -842,18 +846,20 @@ void StatsAnalyzer::WriteCodegenIdDescriptorHuffmanCodecs(
       const uint32_t descriptor = pair.first;
       const uint32_t count = pair.second;
       const double freq = double(count) / double(total);
-      const double kDescriptorFrequentEnoughToAnalyze = 0.01;
+      const double kDescriptorFrequentEnoughToAnalyze = 0.003;
       if (freq < kDescriptorFrequentEnoughToAnalyze) {
         left_out += count;
         continue;
       }
       processed_hist.emplace(descriptor, count);
+      descriptors_with_coding_scheme.insert(descriptor);
     }
 
     // Heuristic.
     processed_hist.emplace(kMarkvNoneOfTheAbove,
                            std::max(1, int(left_out + total * 0.01)));
 
+
     HuffmanCodec<uint64_t> codec(processed_hist);
 
     out << "  {\n";
@@ -868,4 +874,12 @@ void StatsAnalyzer::WriteCodegenIdDescriptorHuffmanCodecs(
   }
 
   out << "  return codecs;\n}\n";
+
+  out << "\nstd::unordered_set<uint32_t> GetDescriptorsWithCodingScheme() {\n"
+      << "  std::unordered_set<uint32_t> descriptors_with_coding_scheme = {\n";
+  for (uint32_t descriptor : descriptors_with_coding_scheme) {
+    out << "    " << descriptor << ",\n";
+  }
+  out << "  };\n";
+  out << "  return descriptors_with_coding_scheme;\n}\n";
 }