Move bit_stream, move_to_front and huffman_codec. (#1833)

bit_stream, move_to_front and huffman_codec are only used by
source/tools. Move into that directory to make the usage clearer.

Android.mk

@@ -24,7 +24,6 @@ SPVTOOLS_SRC_FILES := \
 		source/table.cpp \
 		source/text.cpp \
 		source/text_handler.cpp \
-		source/util/bit_stream.cpp \
 		source/util/bit_vector.cpp \
 		source/util/parse_number.cpp \
 		source/util/string_utils.cpp \

BUILD.gn

@@ -326,16 +326,12 @@ static_library("spvtools") {
     "source/text.h",
     "source/text_handler.cpp",
     "source/text_handler.h",
-    "source/util/bit_stream.cpp",
-    "source/util/bit_stream.h",
     "source/util/bit_vector.cpp",
     "source/util/bit_vector.h",
     "source/util/bitutils.h",
     "source/util/hex_float.h",
-    "source/util/hufman_codec.h",
     "source/util/ilist.h",
     "source/util/ilist_node.h",
-    "source/util/move_to_front.h",
     "source/util/parse_number.cpp",
     "source/util/parse_number.h",
     "source/util/small_vector.h",

source/CMakeLists.txt

@@ -217,7 +217,6 @@ set(SPIRV_SOURCES
   ${spirv-tools_SOURCE_DIR}/include/spirv-tools/libspirv.h
 
   ${CMAKE_CURRENT_SOURCE_DIR}/util/bitutils.h
-  ${CMAKE_CURRENT_SOURCE_DIR}/util/bit_stream.h
   ${CMAKE_CURRENT_SOURCE_DIR}/util/bit_vector.h
   ${CMAKE_CURRENT_SOURCE_DIR}/util/hex_float.h
   ${CMAKE_CURRENT_SOURCE_DIR}/util/parse_number.h
@@ -254,7 +253,6 @@ set(SPIRV_SOURCES
   ${CMAKE_CURRENT_SOURCE_DIR}/text_handler.h
   ${CMAKE_CURRENT_SOURCE_DIR}/val/validate.h
 
-  ${CMAKE_CURRENT_SOURCE_DIR}/util/bit_stream.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/util/bit_vector.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/util/parse_number.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/util/string_utils.cpp

source/comp/CMakeLists.txt

@@ -14,6 +14,9 @@
 
 if(SPIRV_BUILD_COMPRESSION)
   add_library(SPIRV-Tools-comp
+      bit_stream.cpp
+      bit_stream.h
+      huffman_codec.h
       markv_codec.cpp
       markv_codec.h
       markv.cpp
@@ -23,8 +26,8 @@ if(SPIRV_BUILD_COMPRESSION)
       markv_encoder.cpp
       markv_encoder.h
       markv_logger.h
-      ${CMAKE_CURRENT_SOURCE_DIR}/../util/move_to_front.h
-      ${CMAKE_CURRENT_SOURCE_DIR}/../util/move_to_front.cpp)
+      move_to_front.h
+      move_to_front.cpp)
 
   spvtools_default_compile_options(SPIRV-Tools-comp)
   target_include_directories(SPIRV-Tools-comp

source/comp/bit_stream.cpp

@@ -18,10 +18,10 @@
 #include <sstream>
 #include <type_traits>
 
-#include "source/util/bit_stream.h"
+#include "source/comp/bit_stream.h"
 
 namespace spvtools {
-namespace utils {
+namespace comp {
 namespace {
 
 // Returns if the system is little-endian. Unfortunately only works during
@@ -344,5 +344,5 @@ bool BitReaderWord64::OnlyZeroesLeft() const {
   return !remaining_bits;
 }
 
-}  // namespace utils
+}  // namespace comp
 }  // namespace spvtools

source/comp/bit_stream.h

@@ -14,11 +14,12 @@
 
 // Contains utils for reading, writing and debug printing bit streams.
 
-#ifndef SOURCE_UTIL_BIT_STREAM_H_
-#define SOURCE_UTIL_BIT_STREAM_H_
+#ifndef SOURCE_COMP_BIT_STREAM_H_
+#define SOURCE_COMP_BIT_STREAM_H_
 
 #include <algorithm>
 #include <bitset>
+#include <cassert>
 #include <cstdint>
 #include <functional>
 #include <sstream>
@@ -27,7 +28,7 @@
 #include <vector>
 
 namespace spvtools {
-namespace utils {
+namespace comp {
 
 // Terminology:
 // Bits - usually used for a uint64 word, first bit is the lowest.
@@ -272,7 +273,7 @@ class BitReaderWord64 : public BitReaderInterface {
   std::function<void(const std::string&)> callback_;
 };
 
-}  // namespace utils
+}  // namespace comp
 }  // namespace spvtools
 
-#endif  // SOURCE_UTIL_BIT_STREAM_H_
+#endif  // SOURCE_COMP_BIT_STREAM_H_

source/comp/huffman_codec.h

@@ -14,8 +14,8 @@
 
 // Contains utils for reading, writing and debug printing bit streams.
 
-#ifndef SOURCE_UTIL_HUFFMAN_CODEC_H_
-#define SOURCE_UTIL_HUFFMAN_CODEC_H_
+#ifndef SOURCE_COMP_HUFFMAN_CODEC_H_
+#define SOURCE_COMP_HUFFMAN_CODEC_H_
 
 #include <algorithm>
 #include <cassert>
@@ -34,7 +34,7 @@
 #include <vector>
 
 namespace spvtools {
-namespace utils {
+namespace comp {
 
 // Used to generate and apply a Huffman coding scheme.
 // |Val| is the type of variable being encoded (for example a string or a
@@ -383,7 +383,7 @@ class HuffmanCodec {
   uint32_t next_node_id_ = 1;
 };
 
-}  // namespace utils
+}  // namespace comp
 }  // namespace spvtools
 
-#endif  // SOURCE_UTIL_HUFFMAN_CODEC_H_
+#endif  // SOURCE_COMP_HUFFMAN_CODEC_H_

source/comp/markv_codec.cpp

@@ -41,13 +41,13 @@ uint32_t ShortHashU32Array(const std::vector<uint32_t>& words) {
 
 // Returns a set of mtf rank codecs based on a plausible hand-coded
 // distribution.
-std::map<uint64_t, std::unique_ptr<utils::HuffmanCodec<uint32_t>>>
+std::map<uint64_t, std::unique_ptr<HuffmanCodec<uint32_t>>>
 GetMtfHuffmanCodecs() {
-  std::map<uint64_t, std::unique_ptr<utils::HuffmanCodec<uint32_t>>> codecs;
+  std::map<uint64_t, std::unique_ptr<HuffmanCodec<uint32_t>>> codecs;
 
-  std::unique_ptr<utils::HuffmanCodec<uint32_t>> codec;
+  std::unique_ptr<HuffmanCodec<uint32_t>> codec;
 
-  codec.reset(new utils::HuffmanCodec<uint32_t>(std::map<uint32_t, uint32_t>({
+  codec.reset(new HuffmanCodec<uint32_t>(std::map<uint32_t, uint32_t>({
       {0, 5},
       {1, 40},
       {2, 10},
@@ -62,7 +62,7 @@ GetMtfHuffmanCodecs() {
   })));
   codecs.emplace(kMtfAll, std::move(codec));
 
-  codec.reset(new utils::HuffmanCodec<uint32_t>(std::map<uint32_t, uint32_t>({
+  codec.reset(new HuffmanCodec<uint32_t>(std::map<uint32_t, uint32_t>({
       {1, 50},
       {2, 20},
       {3, 5},

source/comp/markv_codec.h

@@ -21,11 +21,12 @@
 #include <vector>
 
 #include "source/assembly_grammar.h"
+#include "source/comp/huffman_codec.h"
 #include "source/comp/markv_model.h"
+#include "source/comp/move_to_front.h"
 #include "source/diagnostic.h"
 #include "source/id_descriptor.h"
-#include "source/util/huffman_codec.h"
-#include "source/util/move_to_front.h"
+
 #include "source/val/instruction.h"
 
 // Base class for MARK-V encoder and decoder. Contains common functionality
@@ -244,8 +245,7 @@ class MarkvCodec {
   // Returns Huffman codec for ranks of the mtf with given |handle|.
   // Different mtfs can use different rank distributions.
   // May return nullptr if the codec doesn't exist.
-  const utils::HuffmanCodec<uint32_t>* GetMtfHuffmanCodec(
-      uint64_t handle) const {
+  const HuffmanCodec<uint32_t>* GetMtfHuffmanCodec(uint64_t handle) const {
     const auto it = mtf_huffman_codecs_.find(handle);
     if (it == mtf_huffman_codecs_.end()) return nullptr;
     return it->second.get();
@@ -281,7 +281,7 @@ class MarkvCodec {
   std::unordered_map<uint32_t, uint32_t> id_to_type_id_;
 
   // Container for all move-to-front sequences.
-  utils::MultiMoveToFront multi_mtf_;
+  MultiMoveToFront multi_mtf_;
 
   // Id of the current function or zero if outside of function.
   uint32_t cur_function_id_ = 0;
@@ -316,7 +316,7 @@ class MarkvCodec {
   // Huffman codecs for move-to-front ranks. The map key is mtf handle. Doesn't
   // need to contain a different codec for every handle as most use one and the
   // same.
-  std::map<uint64_t, std::unique_ptr<utils::HuffmanCodec<uint32_t>>>
+  std::map<uint64_t, std::unique_ptr<HuffmanCodec<uint32_t>>>
       mtf_huffman_codecs_;
 
   // If not nullptr, codec will log comments on the compression process.

source/comp/markv_decoder.h

@@ -12,10 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "source/comp/bit_stream.h"
 #include "source/comp/markv.h"
 #include "source/comp/markv_codec.h"
 #include "source/comp/markv_logger.h"
-#include "source/util/bit_stream.h"
 
 #ifndef SOURCE_COMP_MARKV_DECODER_H_
 #define SOURCE_COMP_MARKV_DECODER_H_
@@ -151,7 +151,7 @@ class MarkvDecoder : public MarkvCodec {
   std::vector<uint32_t> spirv_;
 
   // Bit stream containing encoded data.
-  utils::BitReaderWord64 reader_;
+  BitReaderWord64 reader_;
 
   // Temporary storage for operands of the currently parsed instruction.
   // Valid until next DecodeInstruction call.

source/comp/markv_encoder.h

@@ -12,10 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "source/comp/bit_stream.h"
 #include "source/comp/markv.h"
 #include "source/comp/markv_codec.h"
 #include "source/comp/markv_logger.h"
-#include "source/util/bit_stream.h"
 
 #ifndef SOURCE_COMP_MARKV_ENCODER_H_
 #define SOURCE_COMP_MARKV_ENCODER_H_
@@ -152,7 +152,7 @@ class MarkvEncoder : public MarkvCodec {
   MarkvCodecOptions options_;
 
   // Bit stream where encoded instructions are written.
-  utils::BitWriterWord64 writer_;
+  BitWriterWord64 writer_;
 
   // If not nullptr, disassembled instruction lines will be written to comments.
   // Format: \n separated instruction lines, no header.

source/comp/markv_model.h

@@ -17,8 +17,8 @@
 
 #include <unordered_set>
 
+#include "source/comp/huffman_codec.h"
 #include "source/latest_version_spirv_header.h"
-#include "source/util/huffman_codec.h"
 #include "spirv-tools/libspirv.hpp"
 
 namespace spvtools {
@@ -96,8 +96,8 @@ class MarkvModel {
 
   // Returns a codec for common opcode_and_num_operands words for the given
   // previous opcode. May return nullptr if the codec doesn't exist.
-  const utils::HuffmanCodec<uint64_t>*
-  GetOpcodeAndNumOperandsMarkovHuffmanCodec(uint32_t prev_opcode) const {
+  const HuffmanCodec<uint64_t>* GetOpcodeAndNumOperandsMarkovHuffmanCodec(
+      uint32_t prev_opcode) const {
     if (prev_opcode == SpvOpNop)
       return opcode_and_num_operands_huffman_codec_.get();
 
@@ -111,7 +111,7 @@ class MarkvModel {
   // Returns a codec for common non-id words used for given operand slot.
   // Operand slot is defined by the opcode and the operand index.
   // May return nullptr if the codec doesn't exist.
-  const utils::HuffmanCodec<uint64_t>* GetNonIdWordHuffmanCodec(
+  const HuffmanCodec<uint64_t>* GetNonIdWordHuffmanCodec(
       uint32_t opcode, uint32_t operand_index) const {
     const auto it = non_id_word_huffman_codecs_.find(
         std::pair<uint32_t, uint32_t>(opcode, operand_index));
@@ -122,7 +122,7 @@ class MarkvModel {
   // Returns a codec for common id descriptos used for given operand slot.
   // Operand slot is defined by the opcode and the operand index.
   // May return nullptr if the codec doesn't exist.
-  const utils::HuffmanCodec<uint64_t>* GetIdDescriptorHuffmanCodec(
+  const HuffmanCodec<uint64_t>* GetIdDescriptorHuffmanCodec(
       uint32_t opcode, uint32_t operand_index) const {
     const auto it = id_descriptor_huffman_codecs_.find(
         std::pair<uint32_t, uint32_t>(opcode, operand_index));
@@ -133,7 +133,7 @@ class MarkvModel {
   // Returns a codec for common strings used by the given opcode.
   // Operand slot is defined by the opcode and the operand index.
   // May return nullptr if the codec doesn't exist.
-  const utils::HuffmanCodec<std::string>* GetLiteralStringHuffmanCodec(
+  const HuffmanCodec<std::string>* GetLiteralStringHuffmanCodec(
       uint32_t opcode) const {
     const auto it = literal_string_huffman_codecs_.find(opcode);
     if (it == literal_string_huffman_codecs_.end()) return nullptr;
@@ -177,23 +177,23 @@ class MarkvModel {
 
  protected:
   // Huffman codec for base-rate of opcode_and_num_operands.
-  std::unique_ptr<utils::HuffmanCodec<uint64_t>>
+  std::unique_ptr<HuffmanCodec<uint64_t>>
       opcode_and_num_operands_huffman_codec_;
 
   // Huffman codecs for opcode_and_num_operands. The map key is previous opcode.
-  std::map<uint32_t, std::unique_ptr<utils::HuffmanCodec<uint64_t>>>
+  std::map<uint32_t, std::unique_ptr<HuffmanCodec<uint64_t>>>
       opcode_and_num_operands_markov_huffman_codecs_;
 
   // Huffman codecs for non-id single-word operand values.
   // The map key is pair <opcode, operand_index>.
   std::map<std::pair<uint32_t, uint32_t>,
-           std::unique_ptr<utils::HuffmanCodec<uint64_t>>>
+           std::unique_ptr<HuffmanCodec<uint64_t>>>
       non_id_word_huffman_codecs_;
 
   // Huffman codecs for id descriptors. The map key is pair
   // <opcode, operand_index>.
   std::map<std::pair<uint32_t, uint32_t>,
-           std::unique_ptr<utils::HuffmanCodec<uint64_t>>>
+           std::unique_ptr<HuffmanCodec<uint64_t>>>
       id_descriptor_huffman_codecs_;
 
   // Set of all descriptors which have a coding scheme in any of
@@ -204,7 +204,7 @@ class MarkvModel {
   // current instruction. This assumes, that there is no more than one literal
   // string operand per instruction, but would still work even if this is not
   // the case. Names and debug information strings are not collected.
-  std::map<uint32_t, std::unique_ptr<utils::HuffmanCodec<std::string>>>
+  std::map<uint32_t, std::unique_ptr<HuffmanCodec<std::string>>>
       literal_string_huffman_codecs_;
 
   // Chunk lengths used for variable width encoding of operands (index is

source/comp/move_to_front.cpp

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "source/util/move_to_front.h"
+#include "source/comp/move_to_front.h"
 
 #include <algorithm>
 #include <iomanip>
@@ -23,7 +23,7 @@
 #include <utility>
 
 namespace spvtools {
-namespace utils {
+namespace comp {
 
 bool MoveToFront::Insert(uint32_t value) {
   auto it = value_to_node_.find(value);
@@ -452,5 +452,5 @@ void MoveToFront::UpdateNode(uint32_t node) {
       1 + std::max(HeightOf(LeftOf(node)), HeightOf(RightOf(node)));
 }
 
-}  // namespace utils
+}  // namespace comp
 }  // namespace spvtools

source/comp/move_to_front.h

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef SOURCE_UTIL_MOVE_TO_FRONT_H_
-#define SOURCE_UTIL_MOVE_TO_FRONT_H_
+#ifndef SOURCE_COMP_MOVE_TO_FRONT_H_
+#define SOURCE_COMP_MOVE_TO_FRONT_H_
 
 #include <cassert>
 #include <cstdint>
@@ -23,7 +23,7 @@
 #include <vector>
 
 namespace spvtools {
-namespace utils {
+namespace comp {
 
 // Log(n) move-to-front implementation. Implements the following functions:
 // Insert - pushes value to the front of the mtf sequence
@@ -378,7 +378,7 @@ class MultiMoveToFront {
   MoveToFront* cached_mtf_ = nullptr;
 };
 
-}  // namespace utils
+}  // namespace comp
 }  // namespace spvtools
 
-#endif  // SOURCE_UTIL_MOVE_TO_FRONT_H_
+#endif  // SOURCE_COMP_MOVE_TO_FRONT_H_

test/CMakeLists.txt

@@ -198,18 +198,23 @@ add_spvtools_unittest(
 add_spvtools_unittest(
   TARGET bit_stream
   SRCS bit_stream.cpp
+      ${CMAKE_CURRENT_SOURCE_DIR}/../source/comp/bit_stream.cpp
+      ${CMAKE_CURRENT_SOURCE_DIR}/../source/comp/bit_stream.h
   LIBS ${SPIRV_TOOLS})
 
 add_spvtools_unittest(
   TARGET huffman_codec
   SRCS huffman_codec.cpp
+      ${CMAKE_CURRENT_SOURCE_DIR}/../source/comp/bit_stream.cpp
+      ${CMAKE_CURRENT_SOURCE_DIR}/../source/comp/bit_stream.h
+      ${CMAKE_CURRENT_SOURCE_DIR}/../source/comp/huffman_codec.h
   LIBS ${SPIRV_TOOLS})
 
 add_spvtools_unittest(
   TARGET move_to_front
   SRCS move_to_front_test.cpp
-    ../source/util/move_to_front.h
-    ../source/util/move_to_front.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/../source/comp/move_to_front.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/../source/comp/move_to_front.cpp
   LIBS ${SPIRV_TOOLS})
 
 add_subdirectory(comp)

test/bit_stream.cpp

@@ -19,10 +19,10 @@
 #include <vector>
 
 #include "gmock/gmock.h"
-#include "source/util/bit_stream.h"
+#include "source/comp/bit_stream.h"
 
 namespace spvtools {
-namespace utils {
+namespace comp {
 namespace {
 
 // Converts |buffer| to a stream of '0' and '1'.
@@ -1021,5 +1021,5 @@ TEST(VariableWidthWriteRead, VariedNumbersChunkLength8) {
 }
 
 }  // namespace
-}  // namespace utils
+}  // namespace comp
 }  // namespace spvtools

test/huffman_codec.cpp

@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Contains utils for reading, writing and debug printing bit streams.
-
 #include <algorithm>
 #include <map>
 #include <sstream>
@@ -23,11 +21,11 @@
 #include <vector>
 
 #include "gmock/gmock.h"
-#include "source/util/bit_stream.h"
-#include "source/util/huffman_codec.h"
+#include "source/comp/bit_stream.h"
+#include "source/comp/huffman_codec.h"
 
 namespace spvtools {
-namespace utils {
+namespace comp {
 namespace {
 
 const std::map<std::string, uint32_t>& GetTestSet() {
@@ -315,5 +313,5 @@ TEST(Huffman, CreateFromTextU64) {
 }
 
 }  // namespace
-}  // namespace utils
+}  // namespace comp
 }  // namespace spvtools

test/move_to_front_test.cpp

@@ -19,10 +19,10 @@
 #include <vector>
 
 #include "gmock/gmock.h"
-#include "source/util/move_to_front.h"
+#include "source/comp/move_to_front.h"
 
 namespace spvtools {
-namespace utils {
+namespace comp {
 namespace {
 
 // Class used to test the inner workings of MoveToFront.
@@ -824,5 +824,5 @@ TEST(MoveToFront, LargerScale) {
 }
 
 }  // namespace
-}  // namespace utils
+}  // namespace comp
 }  // namespace spvtools

tools/comp/markv_model_shader.cpp

@@ -21,8 +21,6 @@
 #include <unordered_set>
 #include <vector>
 
-using spvtools::utils::HuffmanCodec;
-
 namespace spvtools {
 namespace comp {
 namespace {