Merge pull request #6 from szabadka/master

Move the block switch storing functions to the brotli_bit_stream library...

enc/block_splitter.cc

@@ -301,7 +301,7 @@ void SplitByteVector(const std::vector<DataType>& data,
                      const double block_switch_cost,
                      BlockSplit* split) {
   if (data.empty()) {
-    split->num_types_ = 0;
+    split->num_types_ = 1;
     return;
   } else if (data.size() < kMinLengthForBlockSplitting) {
     split->num_types_ = 1;

enc/block_splitter.h

@@ -29,8 +29,7 @@ namespace brotli {
 
 struct BlockSplit {
   int num_types_;
-  std::vector<uint8_t> types_;
-  std::vector<int> type_codes_;
+  std::vector<int> types_;
   std::vector<int> lengths_;
 };
 

enc/brotli_bit_stream.cc

@@ -22,6 +22,7 @@
 
 #include "./entropy_encode.h"
 #include "./fast_log.h"
+#include "./prefix.h"
 #include "./write_bits.h"
 
 namespace brotli {
@@ -305,6 +306,69 @@ void BuildAndStoreHuffmanTree(const int *histogram,
   }
 }
 
+void StoreBlockSwitch(const BlockSplitCode& code,
+                      const int block_ix,
+                      int* storage_ix,
+                      uint8_t* storage) {
+  if (block_ix > 0) {
+    int typecode = code.type_code[block_ix];
+    WriteBits(code.type_depths[typecode], code.type_bits[typecode],
+              storage_ix, storage);
+  }
+  int lencode = code.length_prefix[block_ix];
+  WriteBits(code.length_depths[lencode], code.length_bits[lencode],
+            storage_ix, storage);
+  WriteBits(code.length_nextra[block_ix], code.length_extra[block_ix],
+            storage_ix, storage);
+}
+
+void BuildAndStoreBlockSplitCode(const std::vector<int>& types,
+                                 const std::vector<int>& lengths,
+                                 const int num_types,
+                                 const int quality,
+                                 BlockSplitCode* code,
+                                 int* storage_ix,
+                                 uint8_t* storage) {
+  const int num_blocks = types.size();
+  std::vector<int> type_histo(num_types + 2);
+  std::vector<int> length_histo(26);
+  int last_type = 1;
+  int second_last_type = 0;
+  code->type_code.resize(num_blocks);
+  code->length_prefix.resize(num_blocks);
+  code->length_nextra.resize(num_blocks);
+  code->length_extra.resize(num_blocks);
+  code->type_depths.resize(num_types + 2);
+  code->type_bits.resize(num_types + 2);
+  code->length_depths.resize(26);
+  code->length_bits.resize(26);
+  for (int i = 0; i < num_blocks; ++i) {
+    int type = types[i];
+    int type_code = (type == last_type + 1 ? 1 :
+                     type == second_last_type ? 0 :
+                     type + 2);
+    second_last_type = last_type;
+    last_type = type;
+    code->type_code[i] = type_code;
+    if (i > 0) ++type_histo[type_code];
+    GetBlockLengthPrefixCode(lengths[i],
+                             &code->length_prefix[i],
+                             &code->length_nextra[i],
+                             &code->length_extra[i]);
+    ++length_histo[code->length_prefix[i]];
+  }
+  StoreVarLenUint8(num_types - 1, storage_ix, storage);
+  if (num_types > 1) {
+    BuildAndStoreHuffmanTree(&type_histo[0], num_types + 2, quality,
+                             &code->type_depths[0], &code->type_bits[0],
+                             storage_ix, storage);
+    BuildAndStoreHuffmanTree(&length_histo[0], 26, quality,
+                             &code->length_depths[0], &code->length_bits[0],
+                             storage_ix, storage);
+    StoreBlockSwitch(*code, 0, storage_ix, storage);
+  }
+}
+
 void StoreTrivialContextMap(int num_types,
                             int context_bits,
                             int* storage_ix,

enc/brotli_bit_stream.h

@@ -26,6 +26,7 @@
 
 #include <stddef.h>
 #include <stdint.h>
+#include <vector>
 
 namespace brotli {
 
@@ -62,6 +63,35 @@ void BuildAndStoreHuffmanTree(const int *histogram,
                               int* storage_ix,
                               uint8_t* storage);
 
+// Data structure that stores everything that is needed to encode each block
+// switch command.
+struct BlockSplitCode {
+  std::vector<int> type_code;
+  std::vector<int> length_prefix;
+  std::vector<int> length_nextra;
+  std::vector<int> length_extra;
+  std::vector<uint8_t> type_depths;
+  std::vector<uint16_t> type_bits;
+  std::vector<uint8_t> length_depths;
+  std::vector<uint16_t> length_bits;
+};
+
+// Builds a BlockSplitCode data structure from the block split given by the
+// vector of block types and block lengths and stores it to the bit stream.
+void BuildAndStoreBlockSplitCode(const std::vector<int>& types,
+                                 const std::vector<int>& lengths,
+                                 const int num_types,
+                                 const int quality,
+                                 BlockSplitCode* code,
+                                 int* storage_ix,
+                                 uint8_t* storage);
+
+// Stores the block switch command with index block_ix to the bit stream.
+void StoreBlockSwitch(const BlockSplitCode& code,
+                      const int block_ix,
+                      int* storage_ix,
+                      uint8_t* storage);
+
 }  // namespace brotli
 
 #endif  // BROTLI_ENC_BROTLI_BIT_STREAM_H_

enc/encode.cc

@@ -435,74 +435,6 @@ void EncodeContextMap(const std::vector<int>& context_map,
   WriteBits(1, 1, storage_ix, storage);  // use move-to-front
 }
 
-struct BlockSplitCode {
-  EntropyCodeBlockType block_type_code;
-  EntropyCodeBlockLength block_len_code;
-};
-
-void EncodeBlockLength(const EntropyCodeBlockLength& entropy,
-                       int length,
-                       int* storage_ix, uint8_t* storage) {
-  int len_code = BlockLengthPrefix(length);
-  int extra_bits = BlockLengthExtraBits(len_code);
-  int extra_bits_value = length - BlockLengthOffset(len_code);
-  WriteBits(entropy.depth_[len_code], entropy.bits_[len_code],
-            storage_ix, storage);
-  if (extra_bits > 0) {
-    WriteBits(extra_bits, extra_bits_value, storage_ix, storage);
-  }
-}
-
-void ComputeBlockTypeShortCodes(BlockSplit* split) {
-  if (split->num_types_ <= 1) {
-    split->num_types_ = 1;
-    return;
-  }
-  int ringbuffer[2] = { 0, 1 };
-  size_t index = 0;
-  for (int i = 0; i < split->types_.size(); ++i) {
-    int type = split->types_[i];
-    int type_code;
-    if (type == ringbuffer[index & 1]) {
-      type_code = 0;
-    } else if (type == ringbuffer[(index - 1) & 1] + 1) {
-      type_code = 1;
-    } else {
-      type_code = type + 2;
-    }
-    ringbuffer[index & 1] = type;
-    ++index;
-    split->type_codes_.push_back(type_code);
-  }
-}
-
-void BuildAndEncodeBlockSplitCode(const BlockSplit& split,
-                                  BlockSplitCode* code,
-                                  int* storage_ix, uint8_t* storage) {
-  StoreVarLenUint8(split.num_types_ - 1, storage_ix, storage);
-
-  if (split.num_types_ == 1) {
-    return;
-  }
-
-  HistogramBlockType type_histo;
-  for (int i = 1; i < split.type_codes_.size(); ++i) {
-    type_histo.Add(split.type_codes_[i]);
-  }
-  HistogramBlockLength length_histo;
-  for (int i = 0; i < split.lengths_.size(); ++i) {
-    length_histo.Add(BlockLengthPrefix(split.lengths_[i]));
-  }
-  BuildAndStoreEntropyCode(type_histo, 15, split.num_types_ + 2,
-                           &code->block_type_code,
-                           storage_ix, storage);
-  BuildAndStoreEntropyCode(length_histo, 15, kNumBlockLenPrefixes,
-                           &code->block_len_code,
-                           storage_ix, storage);
-  EncodeBlockLength(code->block_len_code, split.lengths_[0],
-                    storage_ix, storage);
-}
-
 void MoveAndEncode(const BlockSplitCode& code,
                    BlockSplitIterator* it,
                    int* storage_ix, uint8_t* storage) {
@@ -510,11 +442,7 @@ void MoveAndEncode(const BlockSplitCode& code,
     ++it->idx_;
     it->type_ = it->split_.types_[it->idx_];
     it->length_ = it->split_.lengths_[it->idx_];
-    int type_code = it->split_.type_codes_[it->idx_];
-    WriteBits(code.block_type_code.depth_[type_code],
-              code.block_type_code.bits_[type_code],
-              storage_ix, storage);
-    EncodeBlockLength(code.block_len_code, it->length_, storage_ix, storage);
+    StoreBlockSwitch(code, it->idx_, storage_ix, storage);
   }
   --it->length_;
 }
@@ -558,9 +486,6 @@ void BuildMetaBlock(const EncodingParams& params,
              &mb->literal_split,
              &mb->command_split,
              &mb->distance_split);
-  ComputeBlockTypeShortCodes(&mb->literal_split);
-  ComputeBlockTypeShortCodes(&mb->command_split);
-  ComputeBlockTypeShortCodes(&mb->distance_split);
 
   mb->literal_context_modes.resize(mb->literal_split.num_types_,
                                    mb->params.literal_context_mode);
@@ -630,12 +555,24 @@ void StoreMetaBlock(const MetaBlock& mb,
   BlockSplitCode literal_split_code;
   BlockSplitCode command_split_code;
   BlockSplitCode distance_split_code;
-  BuildAndEncodeBlockSplitCode(mb.literal_split, &literal_split_code,
-                               storage_ix, storage);
-  BuildAndEncodeBlockSplitCode(mb.command_split, &command_split_code,
-                               storage_ix, storage);
-  BuildAndEncodeBlockSplitCode(mb.distance_split, &distance_split_code,
-                               storage_ix, storage);
+  BuildAndStoreBlockSplitCode(mb.literal_split.types_,
+                              mb.literal_split.lengths_,
+                              mb.literal_split.num_types_,
+                              9,  // quality
+                              &literal_split_code,
+                              storage_ix, storage);
+  BuildAndStoreBlockSplitCode(mb.command_split.types_,
+                              mb.command_split.lengths_,
+                              mb.command_split.num_types_,
+                              9,  // quality
+                              &command_split_code,
+                              storage_ix, storage);
+  BuildAndStoreBlockSplitCode(mb.distance_split.types_,
+                              mb.distance_split.lengths_,
+                              mb.distance_split.num_types_,
+                              9,  // quality
+                              &distance_split_code,
+                              storage_ix, storage);
   WriteBits(2, mb.params.distance_postfix_bits, storage_ix, storage);
   WriteBits(4,
             mb.params.num_direct_distance_codes >>

enc/prefix.cc

@@ -21,23 +21,6 @@
 
 namespace brotli {
 
-// Represents the range of values belonging to a prefix code:
-// [offset, offset + 2^nbits)
-struct PrefixCodeRange {
-  int offset;
-  int nbits;
-};
-
-static const PrefixCodeRange kBlockLengthPrefixCode[kNumBlockLenPrefixes] = {
-  {   1,  2}, {    5,  2}, {  9,   2}, {  13,  2},
-  {  17,  3}, {   25,  3}, {  33,  3}, {  41,  3},
-  {  49,  4}, {   65,  4}, {  81,  4}, {  97,  4},
-  { 113,  5}, {  145,  5}, { 177,  5}, { 209,  5},
-  { 241,  6}, {  305,  6}, { 369,  7}, { 497,  8},
-  { 753,  9}, { 1265, 10}, {2289, 11}, {4337, 12},
-  {8433, 13}, {16625, 24}
-};
-
 static const PrefixCodeRange kInsertLengthPrefixCode[kNumInsertLenPrefixes] = {
   {   0,  0}, {   1,  0}, {  2,   0}, {    3,  0},
   {   4,  0}, {   5,  0}, {  6,   1}, {    8,  1},
@@ -145,22 +128,4 @@ void PrefixEncodeCopyDistance(int distance_code,
   *extra_bits = (distance_code - offset) >> postfix_bits;
 }
 
-int BlockLengthPrefix(int length) {
-  for (int i = 0; i < kNumBlockLenPrefixes; ++i) {
-    const PrefixCodeRange& range = kBlockLengthPrefixCode[i];
-    if (length >= range.offset && length < range.offset + (1 << range.nbits)) {
-      return i;
-    }
-  }
-  return -1;
-}
-
-int BlockLengthExtraBits(int length_code) {
-  return kBlockLengthPrefixCode[length_code].nbits;
-}
-
-int BlockLengthOffset(int length_code) {
-  return kBlockLengthPrefixCode[length_code].offset;
-}
-
 }  // namespace brotli

enc/prefix.h

@@ -29,6 +29,13 @@ static const int kNumBlockLenPrefixes = 26;
 static const int kNumDistanceShortCodes = 16;
 static const int kNumDistancePrefixes = 520;
 
+// Represents the range of values belonging to a prefix code:
+// [offset, offset + 2^nbits)
+struct PrefixCodeRange {
+  int offset;
+  int nbits;
+};
+
 int CommandPrefix(int insert_length, int copy_length);
 int InsertLengthExtraBits(int prefix);
 int InsertLengthOffset(int prefix);
@@ -42,9 +49,25 @@ void PrefixEncodeCopyDistance(int distance_code,
                               int* nbits,
                               uint32_t* extra_bits);
 
-int BlockLengthPrefix(int length);
-int BlockLengthExtraBits(int prefix);
-int BlockLengthOffset(int prefix);
+static const PrefixCodeRange kBlockLengthPrefixCode[kNumBlockLenPrefixes] = {
+  {   1,  2}, {    5,  2}, {  9,   2}, {  13,  2},
+  {  17,  3}, {   25,  3}, {  33,  3}, {  41,  3},
+  {  49,  4}, {   65,  4}, {  81,  4}, {  97,  4},
+  { 113,  5}, {  145,  5}, { 177,  5}, { 209,  5},
+  { 241,  6}, {  305,  6}, { 369,  7}, { 497,  8},
+  { 753,  9}, { 1265, 10}, {2289, 11}, {4337, 12},
+  {8433, 13}, {16625, 24}
+};
+
+inline void GetBlockLengthPrefixCode(int len,
+                                     int* code, int* n_extra, int* extra) {
+  *code = 0;
+  while (*code < 25 && len >= kBlockLengthPrefixCode[*code + 1].offset) {
+    ++(*code);
+  }
+  *n_extra = kBlockLengthPrefixCode[*code].nbits;
+  *extra = len - kBlockLengthPrefixCode[*code].offset;
+}
 
 }  // namespace brotli