Added bit stream utils

Also added generalized zigzag coding.
- Due to signed integers being mostly non-negative, improved zigzag coding
  to favor positive values.

source/CMakeLists.txt

@@ -190,6 +190,7 @@ set(SPIRV_SOURCES
   ${spirv-tools_SOURCE_DIR}/include/spirv-tools/libspirv.h
 
   ${CMAKE_CURRENT_SOURCE_DIR}/util/bitutils.h
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/bit_stream.h
   ${CMAKE_CURRENT_SOURCE_DIR}/util/hex_float.h
   ${CMAKE_CURRENT_SOURCE_DIR}/util/parse_number.h
   ${CMAKE_CURRENT_SOURCE_DIR}/util/string_utils.h
@@ -218,6 +219,7 @@ set(SPIRV_SOURCES
   ${CMAKE_CURRENT_SOURCE_DIR}/text_handler.h
   ${CMAKE_CURRENT_SOURCE_DIR}/validate.h
 
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/bit_stream.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/util/parse_number.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/util/string_utils.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/assembly_grammar.cpp

source/util/bit_stream.cpp

@@ -0,0 +1,387 @@
+// Copyright (c) 2017 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <algorithm>
+#include <cassert>
+#include <cstring>
+#include <sstream>
+#include <type_traits>
+
+#include "util/bit_stream.h"
+
+namespace spvutils {
+
+namespace {
+
+// Returns if the system is little-endian. Unfortunately only works during
+// runtime.
+bool IsLittleEndian() {
+  // This constant value allows the detection of the host machine's endianness.
+  // Accessing it as an array of bytes is valid due to C++11 section 3.10
+  // paragraph 10.
+  static const uint16_t kFF00 = 0xff00;
+  return reinterpret_cast<const unsigned char*>(&kFF00)[0] == 0;
+}
+
+// Copies uint8_t buffer to a uint64_t buffer.
+// Motivation: casting uint64_t* to uint8_t* is ok. Casting in the other
+// direction is only advisable if uint8_t* is aligned to 64-bit word boundary.
+std::vector<uint64_t> ToBuffer64(const std::vector<uint8_t>& in) {
+  std::vector<uint64_t> out;
+  out.resize((in.size() + 7) / 8, 0);
+  memcpy(out.data(), in.data(), in.size());
+  return out;
+}
+
+// Returns uint64_t containing the same bits as |val|.
+// Type size must be less than 8 bytes.
+template <typename T>
+uint64_t ToU64(T val) {
+  static_assert(sizeof(T) <= 8, "Type size too big");
+  uint64_t val64 = 0;
+  std::memcpy(&val64, &val, sizeof(T));
+  return val64;
+}
+
+// Returns value of type T containing the same bits as |val64|.
+// Type size must be less than 8 bytes. Upper (unused) bits of |val64| must be
+// zero (irrelevant, but is checked with assertion).
+template <typename T>
+T FromU64(uint64_t val64) {
+  assert(sizeof(T) == 8 || (val64 >> (sizeof(T) * 8)) == 0);
+  static_assert(sizeof(T) <= 8, "Type size too big");
+  T val = 0;
+  std::memcpy(&val, &val64, sizeof(T));
+  return val;
+}
+
+// Writes bits from |val| to |writer| in chunks of size |chunk_length|.
+// Signal bit is used to signal if the reader should expect another chunk:
+// 0 - no more chunks to follow
+// 1 - more chunks to follow
+// If number of written bits reaches |max_payload| last chunk is truncated.
+void WriteVariableWidthInternal(BitWriterInterface* writer, uint64_t val,
+                                size_t chunk_length, size_t max_payload) {
+  assert(chunk_length > 0);
+  assert(chunk_length < max_payload);
+  assert(max_payload == 64 || (val >> max_payload) == 0);
+
+  if (val == 0) {
+    writer->WriteBits(0, chunk_length + 1);
+    return;
+  }
+
+  size_t payload_written = 0;
+
+  while (val) {
+    if (payload_written + chunk_length >= max_payload) {
+      // This has to be the last chunk.
+      // There is no need for the signal bit and the chunk can be truncated.
+      const size_t left_to_write = max_payload - payload_written;
+      assert((val >> left_to_write) == 0);
+      writer->WriteBits(val, left_to_write);
+      break;
+    }
+
+    writer->WriteBits(val, chunk_length);
+    payload_written += chunk_length;
+    val = val >> chunk_length;
+
+    // Write a single bit to signal if there is more to come.
+    writer->WriteBits(val ? 1 : 0, 1);
+  }
+}
+
+// Reads data written with WriteVariableWidthInternal. |chunk_length| and
+// |max_payload| should be identical to those used to write the data.
+// Returns false if the stream ends prematurely.
+bool ReadVariableWidthInternal(BitReaderInterface* reader, uint64_t* val,
+                               size_t chunk_length, size_t max_payload) {
+  assert(chunk_length > 0);
+  assert(chunk_length <= max_payload);
+  size_t payload_read = 0;
+
+  while (payload_read + chunk_length < max_payload) {
+    uint64_t bits = 0;
+    if (reader->ReadBits(&bits, chunk_length) != chunk_length)
+      return false;
+
+    *val |= bits << payload_read;
+    payload_read += chunk_length;
+
+    uint64_t more_to_come = 0;
+    if (reader->ReadBits(&more_to_come, 1) != 1)
+      return false;
+
+    if (!more_to_come) {
+      return true;
+    }
+  }
+
+  // Need to read the last chunk which may be truncated. No signal bit follows.
+  uint64_t bits = 0;
+  const size_t left_to_read = max_payload - payload_read;
+  if (reader->ReadBits(&bits, left_to_read) != left_to_read)
+    return false;
+
+  *val |= bits << payload_read;
+  return true;
+}
+
+// Calls WriteVariableWidthInternal with the right max_payload argument.
+template <typename T>
+void WriteVariableWidthUnsigned(BitWriterInterface* writer, T val,
+                                size_t chunk_length) {
+  static_assert(std::is_unsigned<T>::value, "Type must be unsigned");
+  static_assert(std::is_integral<T>::value, "Type must be integral");
+  WriteVariableWidthInternal(writer, val, chunk_length, sizeof(T) * 8);
+}
+
+// Calls ReadVariableWidthInternal with the right max_payload argument.
+template <typename T>
+bool ReadVariableWidthUnsigned(BitReaderInterface* reader, T* val,
+                               size_t chunk_length) {
+  static_assert(std::is_unsigned<T>::value, "Type must be unsigned");
+  static_assert(std::is_integral<T>::value, "Type must be integral");
+  uint64_t val64 = 0;
+  if (!ReadVariableWidthInternal(reader, &val64, chunk_length, sizeof(T) * 8))
+    return false;
+  *val = static_cast<T>(val64);
+  assert(*val == val64);
+  return true;
+}
+
+// Encodes signed |val| to an unsigned value and calls
+// WriteVariableWidthInternal with the right max_payload argument.
+template <typename T>
+void WriteVariableWidthSigned(BitWriterInterface* writer, T val,
+                              size_t chunk_length, size_t zigzag_exponent) {
+  static_assert(std::is_signed<T>::value, "Type must be signed");
+  static_assert(std::is_integral<T>::value, "Type must be integral");
+  WriteVariableWidthInternal(writer, EncodeZigZag(val, zigzag_exponent),
+                             chunk_length, sizeof(T) * 8);
+}
+
+// Calls ReadVariableWidthInternal with the right max_payload argument
+// and decodes the value.
+template <typename T>
+bool ReadVariableWidthSigned(BitReaderInterface* reader, T* val,
+                             size_t chunk_length, size_t zigzag_exponent) {
+  static_assert(std::is_signed<T>::value, "Type must be signed");
+  static_assert(std::is_integral<T>::value, "Type must be integral");
+  uint64_t encoded = 0;
+  if (!ReadVariableWidthInternal(reader, &encoded, chunk_length, sizeof(T) * 8))
+    return false;
+
+  const int64_t decoded = DecodeZigZag(encoded, zigzag_exponent);
+
+  *val = static_cast<T>(decoded);
+  assert(*val == decoded);
+  return true;
+}
+
+}  // namespace
+
+void BitWriterInterface::WriteVariableWidthU64(uint64_t val,
+                                               size_t chunk_length) {
+  WriteVariableWidthUnsigned(this, val, chunk_length);
+}
+
+void BitWriterInterface::WriteVariableWidthU32(uint32_t val,
+                                               size_t chunk_length) {
+  WriteVariableWidthUnsigned(this, val, chunk_length);
+}
+
+void BitWriterInterface::WriteVariableWidthU16(uint16_t val,
+                                               size_t chunk_length) {
+  WriteVariableWidthUnsigned(this, val, chunk_length);
+}
+
+void BitWriterInterface::WriteVariableWidthU8(uint8_t val,
+                                              size_t chunk_length) {
+  WriteVariableWidthUnsigned(this, val, chunk_length);
+}
+
+void BitWriterInterface::WriteVariableWidthS64(int64_t val,
+                                               size_t chunk_length,
+                                               size_t zigzag_exponent) {
+  WriteVariableWidthSigned(this, val, chunk_length, zigzag_exponent);
+}
+
+void BitWriterInterface::WriteVariableWidthS32(int32_t val,
+                                               size_t chunk_length,
+                                               size_t zigzag_exponent) {
+  WriteVariableWidthSigned(this, val, chunk_length, zigzag_exponent);
+}
+
+void BitWriterInterface::WriteVariableWidthS16(int16_t val,
+                                               size_t chunk_length,
+                                               size_t zigzag_exponent) {
+  WriteVariableWidthSigned(this, val, chunk_length, zigzag_exponent);
+}
+
+void BitWriterInterface::WriteVariableWidthS8(int8_t val,
+                                              size_t chunk_length,
+                                              size_t zigzag_exponent) {
+  WriteVariableWidthSigned(this, val, chunk_length, zigzag_exponent);
+}
+
+BitWriterWord64::BitWriterWord64(size_t reserve_bits) : end_(0) {
+  buffer_.reserve(NumBitsToNumWords<64>(reserve_bits));
+}
+
+void BitWriterWord64::WriteBits(uint64_t bits, size_t num_bits) {
+  // Check that |bits| and |num_bits| are valid and consistent.
+  assert(num_bits <= 64);
+  const bool is_little_endian = IsLittleEndian();
+  assert(is_little_endian && "Big-endian architecture support not implemented");
+  if (!is_little_endian) return;
+
+  bits = GetLowerBits(bits, num_bits);
+
+  // Offset from the start of the current word.
+  const size_t offset = end_ % 64;
+
+  if (offset == 0) {
+    // If no offset, simply add |bits| as a new word to the buffer_.
+    buffer_.push_back(bits);
+  } else {
+    // Shift bits and add them to the current word after offset.
+    const uint64_t first_word = bits << offset;
+    buffer_.back() |= first_word;
+
+    // If we don't overflow to the next word, there is nothing more to do.
+
+    if (offset + num_bits > 64) {
+      // We overflow to the next word.
+      const uint64_t second_word = bits >> (64 - offset);
+      // Add remaining bits as a new word to buffer_.
+      buffer_.push_back(second_word);
+    }
+  }
+
+  // Move end_ into position for next write.
+  end_ += num_bits;
+  assert(buffer_.size() * 64 >= end_);
+}
+
+bool BitReaderInterface::ReadVariableWidthU64(uint64_t* val,
+                                              size_t chunk_length) {
+  return ReadVariableWidthUnsigned(this, val, chunk_length);
+}
+
+bool BitReaderInterface::ReadVariableWidthU32(uint32_t* val,
+                                              size_t chunk_length) {
+  return ReadVariableWidthUnsigned(this, val, chunk_length);
+}
+
+bool BitReaderInterface::ReadVariableWidthU16(uint16_t* val,
+                                              size_t chunk_length) {
+  return ReadVariableWidthUnsigned(this, val, chunk_length);
+}
+
+bool BitReaderInterface::ReadVariableWidthU8(uint8_t* val,
+                                             size_t chunk_length) {
+  return ReadVariableWidthUnsigned(this, val, chunk_length);
+}
+
+bool BitReaderInterface::ReadVariableWidthS64(int64_t* val,
+                                              size_t chunk_length,
+                                              size_t zigzag_exponent) {
+  return ReadVariableWidthSigned(this, val, chunk_length, zigzag_exponent);
+}
+
+bool BitReaderInterface::ReadVariableWidthS32(int32_t* val,
+                                              size_t chunk_length,
+                                              size_t zigzag_exponent) {
+  return ReadVariableWidthSigned(this, val, chunk_length, zigzag_exponent);
+}
+
+bool BitReaderInterface::ReadVariableWidthS16(int16_t* val,
+                                              size_t chunk_length,
+                                              size_t zigzag_exponent) {
+  return ReadVariableWidthSigned(this, val, chunk_length, zigzag_exponent);
+}
+
+bool BitReaderInterface::ReadVariableWidthS8(int8_t* val,
+                                             size_t chunk_length,
+                                             size_t zigzag_exponent) {
+  return ReadVariableWidthSigned(this, val, chunk_length, zigzag_exponent);
+}
+
+BitReaderWord64::BitReaderWord64(std::vector<uint64_t>&& buffer)
+    : buffer_(std::move(buffer)), pos_(0) {}
+
+BitReaderWord64::BitReaderWord64(const std::vector<uint8_t>& buffer)
+    : buffer_(ToBuffer64(buffer)), pos_(0) {}
+
+size_t BitReaderWord64::ReadBits(uint64_t* bits, size_t num_bits) {
+  assert(num_bits <= 64);
+  const bool is_little_endian = IsLittleEndian();
+  assert(is_little_endian && "Big-endian architecture support not implemented");
+  if (!is_little_endian) return 0;
+
+  if (ReachedEnd())
+    return 0;
+
+  // Index of the current word.
+  const size_t index = pos_ / 64;
+
+  // Bit position in the current word where we start reading.
+  const size_t offset = pos_ % 64;
+
+  // Read all bits from the current word (it might be too much, but
+  // excessive bits will be removed later).
+  *bits = buffer_[index] >> offset;
+
+  const size_t num_read_from_first_word = std::min(64 - offset, num_bits);
+  pos_ += num_read_from_first_word;
+
+  if (pos_ >= buffer_.size() * 64) {
+    // Reached end of buffer_.
+    return num_read_from_first_word;
+  }
+
+  if (offset + num_bits > 64) {
+    // Requested |num_bits| overflows to next word.
+    // Write all bits from the beginning of next word to *bits after offset.
+    *bits |= buffer_[index + 1] << (64 - offset);
+    pos_ += offset + num_bits - 64;
+  }
+
+  // We likely have written more bits than requested. Clear excessive bits.
+  *bits = GetLowerBits(*bits, num_bits);
+  return num_bits;
+}
+
+bool BitReaderWord64::ReachedEnd() const {
+  return pos_ >= buffer_.size() * 64;
+}
+
+bool BitReaderWord64::OnlyZeroesLeft() const {
+  if (ReachedEnd())
+    return true;
+
+  const size_t index = pos_ / 64;
+  if (index < buffer_.size() - 1)
+    return false;
+
+  assert(index == buffer_.size() - 1);
+
+  const size_t offset = pos_ % 64;
+  const uint64_t remaining_bits = buffer_[index] >> offset;
+  return !remaining_bits;
+}
+
+}  // namespace spvutils

source/util/bit_stream.h

@@ -0,0 +1,378 @@
+// Copyright (c) 2017 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Contains utils for reading, writing and debug printing bit streams.
+
+#ifndef LIBSPIRV_UTIL_BIT_STREAM_H_
+#define LIBSPIRV_UTIL_BIT_STREAM_H_
+
+#include <bitset>
+#include <cstdint>
+#include <string>
+#include <sstream>
+#include <vector>
+
+namespace spvutils {
+
+// Terminology:
+// Bits - usually used for a uint64 word, first bit is the lowest.
+// Stream - std::string of '0' and '1', read left-to-right,
+//          i.e. first bit is at the front and not at the end as in
+//          std::bitset::to_string().
+// Bitset - std::bitset corresponding to uint64 bits and to reverse(stream).
+
+// Converts number of bits to a respective number of chunks of size N.
+// For example NumBitsToNumWords<8> returns how many bytes are needed to store
+// |num_bits|.
+template <size_t N>
+inline size_t NumBitsToNumWords(size_t num_bits) {
+  return (num_bits + (N - 1)) / N;
+}
+
+// Returns value of the same type as |in|, where all but the first |num_bits|
+// are set to zero.
+template <typename T>
+inline T GetLowerBits(T in, size_t num_bits) {
+  return sizeof(T) * 8 == num_bits ? in : in & T((T(1) << num_bits) - T(1));
+}
+
+// Encodes signed integer as unsigned in zigzag order:
+//  0 -> 0
+// -1 -> 1
+//  1 -> 2
+// -2 -> 3
+//  2 -> 4
+// Motivation: -1 is 0xFF...FF what doesn't work very well with
+// WriteVariableWidth which prefers to have as many 0 bits as possible.
+inline uint64_t EncodeZigZag(int64_t val) {
+  return (val << 1) ^ (val >> 63);
+}
+
+// Decodes signed integer encoded with EncodeZigZag.
+inline int64_t DecodeZigZag(uint64_t val) {
+  if (val & 1) {
+    // Negative.
+    // 1 -> -1
+    // 3 -> -2
+    // 5 -> -3
+    return -1 - (val >> 1);
+  } else {
+    // Non-negative.
+    // 0 -> 0
+    // 2 -> 1
+    // 4 -> 2
+    return val >> 1;
+  }
+}
+
+// Encodes signed integer as unsigned. This is a generalized version of
+// EncodeZigZag, designed to favor small positive numbers.
+// Values are transformed in blocks of 2^|block_exponent|.
+// If |block_exponent| is zero, then this degenerates into normal EncodeZigZag.
+// Example when |block_exponent| is 1 (return value is the index):
+// 0, 1, -1, -2, 2, 3, -3, -4, 4, 5, -5, -6, 6, 7, -7, -8
+// Example when |block_exponent| is 2:
+// 0, 1, 2, 3, -1, -2, -3, -4, 4, 5, 6, 7, -5, -6, -7, -8
+inline uint64_t EncodeZigZag(int64_t val, size_t block_exponent) {
+  assert(block_exponent < 64);
+  const uint64_t uval = static_cast<uint64_t>(val >= 0 ? val : -val - 1);
+  const uint64_t block_num = ((uval >> block_exponent) << 1) + (val >= 0 ? 0 : 1);
+  const uint64_t pos = GetLowerBits(uval, block_exponent);
+  return (block_num << block_exponent) + pos;
+}
+
+// Decodes signed integer encoded with EncodeZigZag. |block_exponent| must be
+// the same.
+inline int64_t DecodeZigZag(uint64_t val, size_t block_exponent) {
+  assert(block_exponent < 64);
+  const uint64_t block_num = val >> block_exponent;
+  const uint64_t pos = GetLowerBits(val, block_exponent);
+  if (block_num & 1) {
+    // Negative.
+    return -1LL - ((block_num >> 1) << block_exponent) - pos;
+  } else {
+    // Positive.
+    return ((block_num >> 1) << block_exponent) + pos;
+  }
+}
+
+// Converts |buffer| to a stream of '0' and '1'.
+template <typename T>
+std::string BufferToStream(const std::vector<T>& buffer) {
+  std::stringstream ss;
+  for (auto it = buffer.begin(); it != buffer.end(); ++it) {
+    std::string str = std::bitset<sizeof(T) * 8>(*it).to_string();
+    // Strings generated by std::bitset::to_string are read right to left.
+    // Reversing to left to right.
+    std::reverse(str.begin(), str.end());
+    ss << str;
+  }
+  return ss.str();
+}
+
+// Converts a left-to-right input string of '0' and '1' to a buffer of |T|
+// words.
+template <typename T>
+std::vector<T> StreamToBuffer(std::string str) {
+  // The input string is left-to-right, the input argument of std::bitset needs
+  // to right-to-left. Instead of reversing tokens, reverse the entire string
+  // and iterate tokens from end to begin.
+  std::reverse(str.begin(), str.end());
+  const int word_size = static_cast<int>(sizeof(T) * 8);
+  const int str_length = static_cast<int>(str.length());
+  std::vector<T> buffer;
+  buffer.reserve(NumBitsToNumWords<sizeof(T)>(str.length()));
+  for (int index = str_length - word_size; index >= 0; index -= word_size) {
+    buffer.push_back(static_cast<T>(std::bitset<sizeof(T) * 8>(
+        str, index, word_size).to_ullong()));
+  }
+  const size_t suffix_length = str.length() % word_size;
+  if (suffix_length != 0) {
+    buffer.push_back(static_cast<T>(std::bitset<sizeof(T) * 8>(
+        str, 0, suffix_length).to_ullong()));
+  }
+  return buffer;
+}
+
+// Adds '0' chars at the end of the string until the size is a multiple of N.
+template <size_t N>
+inline std::string PadToWord(std::string&& str) {
+  const size_t tail_length = str.size() % N;
+  if (tail_length != 0)
+    str += std::string(N - tail_length, '0');
+  return str;
+}
+
+// Adds '0' chars at the end of the string until the size is a multiple of N.
+template <size_t N>
+inline std::string PadToWord(const std::string& str) {
+  return PadToWord<N>(std::string(str));
+}
+
+// Converts a left-to-right stream of bits to std::bitset.
+template <size_t N>
+inline std::bitset<N> StreamToBitset(std::string str) {
+  std::reverse(str.begin(), str.end());
+  return std::bitset<N>(str);
+}
+
+// Converts first |num_bits| of std::bitset to a left-to-right stream of bits.
+template <size_t N>
+inline std::string BitsetToStream(const std::bitset<N>& bits, size_t num_bits = N) {
+  std::string str = bits.to_string().substr(N - num_bits);
+  std::reverse(str.begin(), str.end());
+  return str;
+}
+
+// Converts a left-to-right stream of bits to uint64.
+inline uint64_t StreamToBits(std::string str) {
+  std::reverse(str.begin(), str.end());
+  return std::bitset<64>(str).to_ullong();
+}
+
+// Converts first |num_bits| stored in uint64 to a left-to-right stream of bits.
+inline std::string BitsToStream(uint64_t bits, size_t num_bits = 64) {
+  std::bitset<64> bitset(bits);
+  return BitsetToStream(bitset, num_bits);
+}
+
+// Base class for writing sequences of bits.
+class BitWriterInterface {
+ public:
+  BitWriterInterface() {}
+  virtual ~BitWriterInterface() {}
+
+  // Writes lower |num_bits| in |bits| to the stream.
+  // |num_bits| must be no greater than 64.
+  virtual void WriteBits(uint64_t bits, size_t num_bits) = 0;
+
+  // Writes left-to-right string of '0' and '1' to stream.
+  // String length must be no greater than 64.
+  // Note: "01" will be writen as 0x2, not 0x1. The string doesn't represent
+  // numbers but a stream of bits in the order they come from encoder.
+  virtual void WriteStream(const std::string& bits) {
+    WriteBits(StreamToBits(bits), bits.length());
+  }
+
+  // Writes lower |num_bits| in |bits| to the stream.
+  // |num_bits| must be no greater than 64.
+  template <size_t N>
+  void WriteBitset(const std::bitset<N>& bits, size_t num_bits = N) {
+    WriteBits(bits.to_ullong(), num_bits);
+  }
+
+  // Writes |val| in chunks of size |chunk_length| followed by a signal bit:
+  // 0 - no more chunks to follow
+  // 1 - more chunks to follow
+  // for example 255 is encoded into 1111 1 1111 0 for chunk length 4.
+  // The last chunk can be truncated and signal bit omitted, if the entire
+  // payload (for example 16 bit for uint16_t has already been written).
+  void WriteVariableWidthU64(uint64_t val, size_t chunk_length);
+  void WriteVariableWidthU32(uint32_t val, size_t chunk_length);
+  void WriteVariableWidthU16(uint16_t val, size_t chunk_length);
+  void WriteVariableWidthU8(uint8_t val, size_t chunk_length);
+  void WriteVariableWidthS64(
+      int64_t val, size_t chunk_length, size_t zigzag_exponent);
+  void WriteVariableWidthS32(
+      int32_t val, size_t chunk_length, size_t zigzag_exponent);
+  void WriteVariableWidthS16(
+      int16_t val, size_t chunk_length, size_t zigzag_exponent);
+  void WriteVariableWidthS8(
+      int8_t val, size_t chunk_length, size_t zigzag_exponent);
+
+  // Returns number of bits written.
+  virtual size_t GetNumBits() const = 0;
+
+  // Provides direct access to the buffer data if implemented.
+  virtual const uint8_t* GetData() const {
+    return nullptr;
+  }
+
+  // Returns buffer size in bytes.
+  size_t GetDataSizeBytes() const {
+    return NumBitsToNumWords<8>(GetNumBits());
+  }
+
+  // Generates and returns byte array containing written bits.
+  virtual std::vector<uint8_t> GetDataCopy() const = 0;
+
+  BitWriterInterface(const BitWriterInterface&) = delete;
+  BitWriterInterface& operator=(const BitWriterInterface&) = delete;
+};
+
+// This class is an implementation of BitWriterInterface, using
+// std::vector<uint64_t> to store written bits.
+class BitWriterWord64 : public BitWriterInterface {
+ public:
+  explicit BitWriterWord64(size_t reserve_bits = 64);
+
+  void WriteBits(uint64_t bits, size_t num_bits) override;
+
+  size_t GetNumBits() const override {
+    return end_;
+  }
+
+  const uint8_t* GetData() const override {
+    return reinterpret_cast<const uint8_t*>(buffer_.data());
+  }
+
+  std::vector<uint8_t> GetDataCopy() const override {
+    return std::vector<uint8_t>(GetData(), GetData() + GetDataSizeBytes());
+  }
+
+  // Returns written stream as std::string, padded with zeroes so that the
+  // length is a multiple of 64.
+  std::string GetStreamPadded64() const {
+    return BufferToStream(buffer_);
+  }
+
+ private:
+  std::vector<uint64_t> buffer_;
+  // Total number of bits written so far. Named 'end' as analogy to std::end().
+  size_t end_;
+};
+
+// Base class for reading sequences of bits.
+class BitReaderInterface {
+ public:
+  BitReaderInterface() {}
+  virtual ~BitReaderInterface() {}
+
+  // Reads |num_bits| from the stream, stores them in |bits|.
+  // Returns number of read bits. |num_bits| must be no greater than 64.
+  virtual size_t ReadBits(uint64_t* bits, size_t num_bits) = 0;
+
+  // Reads |num_bits| from the stream, stores them in |bits|.
+  // Returns number of read bits. |num_bits| must be no greater than 64.
+  template <size_t N>
+  size_t ReadBitset(std::bitset<N>* bits, size_t num_bits = N) {
+    uint64_t val = 0;
+    size_t num_read = ReadBits(&val, num_bits);
+    if (num_read) {
+      *bits = std::bitset<N>(val);
+    }
+    return num_read;
+  }
+
+  // Reads |num_bits| from the stream, returns string in left-to-right order.
+  // The length of the returned string may be less than |num_bits| if end was
+  // reached.
+  std::string ReadStream(size_t num_bits) {
+    uint64_t bits = 0;
+    size_t num_read = ReadBits(&bits, num_bits);
+    return BitsToStream(bits, num_read);
+  }
+
+  // These two functions define 'hard' and 'soft' EOF.
+  //
+  // Returns true if the end of the buffer was reached.
+  virtual bool ReachedEnd() const = 0;
+  // Returns true if we reached the end of the buffer or are nearing it and only
+  // zero bits are left to read. Implementations of this function are allowed to
+  // commit a "false negative" error if the end of the buffer was not reached,
+  // i.e. it can return false even if indeed only zeroes are left.
+  // It is assumed that the consumer expects that
+  // the buffer stream ends with padding zeroes, and would accept this as a
+  // 'soft' EOF. Implementations of this class do not necessarily need to
+  // implement this, default behavior can simply delegate to ReachedEnd().
+  virtual bool OnlyZeroesLeft() const {
+    return ReachedEnd();
+  }
+
+  // Reads value encoded with WriteVariableWidthXXX (see BitWriterInterface).
+  // Reader and writer must use the same |chunk_length| and variable type.
+  // Returns true on success, false if the bit stream ends prematurely.
+  bool ReadVariableWidthU64(uint64_t* val, size_t chunk_length);
+  bool ReadVariableWidthU32(uint32_t* val, size_t chunk_length);
+  bool ReadVariableWidthU16(uint16_t* val, size_t chunk_length);
+  bool ReadVariableWidthU8(uint8_t* val, size_t chunk_length);
+  bool ReadVariableWidthS64(
+      int64_t* val, size_t chunk_length, size_t zigzag_exponent);
+  bool ReadVariableWidthS32(
+      int32_t* val, size_t chunk_length, size_t zigzag_exponent);
+  bool ReadVariableWidthS16(
+      int16_t* val, size_t chunk_length, size_t zigzag_exponent);
+  bool ReadVariableWidthS8(
+      int8_t* val, size_t chunk_length, size_t zigzag_exponent);
+
+  BitReaderInterface(const BitReaderInterface&) = delete;
+  BitReaderInterface& operator=(const BitReaderInterface&) = delete;
+};
+
+// This class is an implementation of BitReaderInterface which accepts both
+// uint8_t and uint64_t buffers as input. uint64_t buffers are consumed and
+// owned. uint8_t buffers are copied.
+class BitReaderWord64 : public BitReaderInterface {
+ public:
+  // Consumes and owns the buffer.
+  explicit BitReaderWord64(std::vector<uint64_t>&& buffer);
+
+  // Copies the buffer and casts it to uint64.
+  // Consuming the original buffer and casting it to uint64 is difficult,
+  // as it would potentially cause data misalignment and poor performance.
+  explicit BitReaderWord64(const std::vector<uint8_t>& buffer);
+
+  size_t ReadBits(uint64_t* bits, size_t num_bits) override;
+  bool ReachedEnd() const override;
+  bool OnlyZeroesLeft() const override;
+
+  BitReaderWord64() = delete;
+ private:
+  const std::vector<uint64_t> buffer_;
+  size_t pos_;
+};
+
+}  // namespace spvutils
+
+#endif  // LIBSPIRV_UTIL_BIT_STREAM_H_

test/CMakeLists.txt

@@ -164,6 +164,11 @@ add_spvtools_unittest(
   SRCS preserve_numeric_ids_test.cpp
   LIBS ${SPIRV_TOOLS})
 
+add_spvtools_unittest(
+  TARGET bit_stream
+  SRCS bit_stream.cpp
+  LIBS ${SPIRV_TOOLS})
+
 add_subdirectory(opt)
 add_subdirectory(val)
 add_subdirectory(stats)