Merge pull request #93 from szabadka/master

Don't do any block splitting for quality 1.

enc/backward_references.cc

@@ -37,7 +37,8 @@ void CreateBackwardReferences(size_t num_bytes,
                               int* dist_cache,
                               int* last_insert_len,
                               Command* commands,
-                              int* num_commands) {
+                              int* num_commands,
+                              int* num_literals) {
   if (num_bytes >= 3 && position >= 3) {
     // Prepare the hashes for three last bytes of the last write.
     // These could not be calculated before, since they require knowledge
@@ -204,6 +205,7 @@ void CreateBackwardReferences(size_t num_bytes,
       }
       Command cmd(insert_length, best_len, best_len_code, distance_code);
       *commands++ = cmd;
+      *num_literals += insert_length;
       insert_length = 0;
       if (kUseDictionary) {
         ++i;
@@ -301,70 +303,71 @@ void CreateBackwardReferences(size_t num_bytes,
                               int* dist_cache,
                               int* last_insert_len,
                               Command* commands,
-                              int* num_commands) {
+                              int* num_commands,
+                              int* num_literals) {
   switch (hash_type) {
     case 1:
       CreateBackwardReferences<Hashers::H1, false, false>(
           num_bytes, position, ringbuffer, ringbuffer_mask,
           literal_cost, literal_cost_mask, max_backward_limit, base_min_score,
           quality, hashers->hash_h1.get(), dist_cache, last_insert_len,
-          commands, num_commands);
+          commands, num_commands, num_literals);
       break;
     case 2:
       CreateBackwardReferences<Hashers::H2, false, false>(
           num_bytes, position, ringbuffer, ringbuffer_mask,
           literal_cost, literal_cost_mask, max_backward_limit, base_min_score,
           quality, hashers->hash_h2.get(), dist_cache, last_insert_len,
-          commands, num_commands);
+          commands, num_commands, num_literals);
       break;
     case 3:
       CreateBackwardReferences<Hashers::H3, false, false>(
           num_bytes, position, ringbuffer, ringbuffer_mask,
           literal_cost, literal_cost_mask, max_backward_limit, base_min_score,
           quality, hashers->hash_h3.get(), dist_cache, last_insert_len,
-          commands, num_commands);
+          commands, num_commands, num_literals);
       break;
     case 4:
       CreateBackwardReferences<Hashers::H4, false, false>(
           num_bytes, position, ringbuffer, ringbuffer_mask,
           literal_cost, literal_cost_mask, max_backward_limit, base_min_score,
           quality, hashers->hash_h4.get(), dist_cache, last_insert_len,
-          commands, num_commands);
+          commands, num_commands, num_literals);
       break;
     case 5:
       CreateBackwardReferences<Hashers::H5, false, false>(
           num_bytes, position, ringbuffer, ringbuffer_mask,
           literal_cost, literal_cost_mask, max_backward_limit, base_min_score,
           quality, hashers->hash_h5.get(), dist_cache, last_insert_len,
-          commands, num_commands);
+          commands, num_commands, num_literals);
       break;
     case 6:
       CreateBackwardReferences<Hashers::H6, false, false>(
           num_bytes, position, ringbuffer, ringbuffer_mask,
           literal_cost, literal_cost_mask, max_backward_limit, base_min_score,
           quality, hashers->hash_h6.get(), dist_cache, last_insert_len,
-          commands, num_commands);
+          commands, num_commands, num_literals);
       break;
     case 7:
       CreateBackwardReferences<Hashers::H7, false, false>(
           num_bytes, position, ringbuffer, ringbuffer_mask,
           literal_cost, literal_cost_mask, max_backward_limit, base_min_score,
           quality, hashers->hash_h7.get(), dist_cache, last_insert_len,
-          commands, num_commands);
+          commands, num_commands, num_literals);
       break;
     case 8:
       CreateBackwardReferences<Hashers::H8, true, true>(
           num_bytes, position, ringbuffer, ringbuffer_mask,
           literal_cost, literal_cost_mask, max_backward_limit, base_min_score,
           quality, hashers->hash_h8.get(), dist_cache, last_insert_len,
-          commands, num_commands);
+          commands, num_commands, num_literals);
       break;
     case 9:
       CreateBackwardReferences<Hashers::H9, true, false>(
           num_bytes, position, ringbuffer, ringbuffer_mask,
           literal_cost, literal_cost_mask, max_backward_limit, base_min_score,
           quality, hashers->hash_h9.get(), dist_cache, last_insert_len,
-          commands, num_commands);
+          commands, num_commands, num_literals);
       break;
     default:
       break;

enc/backward_references.h

@@ -39,7 +39,8 @@ void CreateBackwardReferences(size_t num_bytes,
                               int* dist_cache,
                               int* last_insert_len,
                               Command* commands,
-                              int* num_commands);
+                              int* num_commands,
+                              int* num_literals);
 
 }  // namespace brotli
 

enc/brotli_bit_stream.cc

@@ -778,6 +778,91 @@ bool StoreMetaBlock(const uint8_t* input,
   return true;
 }
 
+bool StoreMetaBlockTrivial(const uint8_t* input,
+                           size_t start_pos,
+                           size_t length,
+                           size_t mask,
+                           bool is_last,
+                           const brotli::Command *commands,
+                           size_t n_commands,
+                           int *storage_ix,
+                           uint8_t *storage) {
+  if (!StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage)) {
+    return false;
+  }
+
+  if (length == 0) {
+    // Only the last meta-block can be empty, so jump to next byte.
+    JumpToByteBoundary(storage_ix, storage);
+    return true;
+  }
+
+  HistogramLiteral lit_histo;
+  HistogramCommand cmd_histo;
+  HistogramDistance dist_histo;
+
+  size_t pos = start_pos;
+  for (int i = 0; i < n_commands; ++i) {
+    const Command cmd = commands[i];
+    cmd_histo.Add(cmd.cmd_prefix_);
+    for (int j = 0; j < cmd.insert_len_; ++j) {
+      lit_histo.Add(input[pos & mask]);
+      ++pos;
+    }
+    pos += cmd.copy_len_;
+    if (cmd.copy_len_ > 0 && cmd.cmd_prefix_ >= 128) {
+      dist_histo.Add(cmd.dist_prefix_);
+    }
+  }
+
+  WriteBits(13, 0, storage_ix, storage);
+
+  std::vector<uint8_t> lit_depth(256);
+  std::vector<uint16_t> lit_bits(256);
+  std::vector<uint8_t> cmd_depth(kNumCommandPrefixes);
+  std::vector<uint16_t> cmd_bits(kNumCommandPrefixes);
+  std::vector<uint8_t> dist_depth(64);
+  std::vector<uint16_t> dist_bits(64);
+
+  BuildAndStoreHuffmanTree(&lit_histo.data_[0], 256,
+                           &lit_depth[0], &lit_bits[0],
+                           storage_ix, storage);
+  BuildAndStoreHuffmanTree(&cmd_histo.data_[0], kNumCommandPrefixes,
+                           &cmd_depth[0], &cmd_bits[0],
+                           storage_ix, storage);
+  BuildAndStoreHuffmanTree(&dist_histo.data_[0], 64,
+                           &dist_depth[0], &dist_bits[0],
+                           storage_ix, storage);
+
+  pos = start_pos;
+  for (int i = 0; i < n_commands; ++i) {
+    const Command cmd = commands[i];
+    const int cmd_code = cmd.cmd_prefix_;
+    const int lennumextra = cmd.cmd_extra_ >> 48;
+    const uint64_t lenextra = cmd.cmd_extra_ & 0xffffffffffffULL;
+    WriteBits(cmd_depth[cmd_code], cmd_bits[cmd_code], storage_ix, storage);
+    WriteBits(lennumextra, lenextra, storage_ix, storage);
+    for (int j = 0; j < cmd.insert_len_; j++) {
+      const uint8_t literal = input[pos & mask];
+      WriteBits(lit_depth[literal], lit_bits[literal], storage_ix, storage);
+      ++pos;
+    }
+    pos += cmd.copy_len_;
+    if (cmd.copy_len_ > 0 && cmd.cmd_prefix_ >= 128) {
+      const int dist_code = cmd.dist_prefix_;
+      const int distnumextra = cmd.dist_extra_ >> 24;
+      const int distextra = cmd.dist_extra_ & 0xffffff;
+      WriteBits(dist_depth[dist_code], dist_bits[dist_code],
+                storage_ix, storage);
+      WriteBits(distnumextra, distextra, storage_ix, storage);
+    }
+  }
+  if (is_last) {
+    JumpToByteBoundary(storage_ix, storage);
+  }
+  return true;
+}
+
 // This is for storing uncompressed blocks (simple raw storage of
 // bytes-as-bytes).
 bool StoreUncompressedMetaBlock(bool final_block,

enc/brotli_bit_stream.h

@@ -120,6 +120,18 @@ bool StoreMetaBlock(const uint8_t* input,
                     int *storage_ix,
                     uint8_t *storage);
 
+// Stores the meta-block without doing any block splitting, just collects
+// one histogram per block category and uses that for entropy coding.
+bool StoreMetaBlockTrivial(const uint8_t* input,
+                           size_t start_pos,
+                           size_t length,
+                           size_t mask,
+                           bool is_last,
+                           const brotli::Command *commands,
+                           size_t n_commands,
+                           int *storage_ix,
+                           uint8_t *storage);
+
 // This is for storing uncompressed blocks (simple raw storage of
 // bytes-as-bytes).
 bool StoreUncompressedMetaBlock(bool final_block,

enc/encode.cc

@@ -134,6 +134,7 @@ BrotliCompressor::BrotliCompressor(BrotliParams params)
       hashers_(new Hashers()),
       input_pos_(0),
       num_commands_(0),
+      num_literals_(0),
       last_insert_len_(0),
       last_flush_pos_(0),
       last_processed_pos_(0),
@@ -148,7 +149,7 @@ BrotliCompressor::BrotliCompressor(BrotliParams params)
     params_.lgwin = kMaxWindowBits;
   }
   if (params_.lgblock == 0) {
-    params_.lgblock = 16;
+    params_.lgblock = params_.quality == 1 ? 14 : 16;
     if (params_.quality >= 9 && params_.lgwin > params_.lgblock) {
       params_.lgblock = std::min(21, params_.lgwin);
     }
@@ -322,9 +323,15 @@ bool BrotliCompressor::WriteBrotliData(const bool is_last,
                            dist_cache_,
                            &last_insert_len_,
                            &commands_[num_commands_],
-                           &num_commands_);
+                           &num_commands_,
+                           &num_literals_);
 
+  // For quality 1 there is no block splitting, so we buffer at most this much
+  // literals and commands.
+  static const int kMaxNumDelayedSymbols = 0x2fff;
   if (!is_last && !force_flush &&
+      (params_.quality > 1 ||
+       (num_literals_ + num_commands_ < kMaxNumDelayedSymbols)) &&
       num_commands_ + (input_block_size() >> 1) < cmd_buffer_size_ &&
       input_pos_ + input_block_size() <= last_flush_pos_ + mask + 1) {
     // Everything will happen later.
@@ -337,6 +344,7 @@ bool BrotliCompressor::WriteBrotliData(const bool is_last,
   if (last_insert_len_ > 0) {
     brotli::Command cmd(last_insert_len_);
     commands_[num_commands_++] = cmd;
+    num_literals_ += last_insert_len_;
     last_insert_len_ = 0;
   }
 
@@ -357,11 +365,7 @@ bool BrotliCompressor::WriteMetaBlockInternal(const bool is_last,
 
   bool uncompressed = false;
   if (num_commands_ < (bytes >> 8) + 2) {
-    int num_literals = 0;
-    for (int i = 0; i < num_commands_; ++i) {
-      num_literals += commands_[i].insert_len_;
-    }
-    if (num_literals > 0.99 * bytes) {
+    if (num_literals_ > 0.99 * bytes) {
       int literal_histo[256] = { 0 };
       static const int kSampleRate = 13;
       static const double kMinEntropy = 7.92;
@@ -404,34 +408,43 @@ bool BrotliCompressor::WriteMetaBlockInternal(const bool is_last,
     }
     int literal_context_mode = utf8_mode ? CONTEXT_UTF8 : CONTEXT_SIGNED;
     MetaBlockSplit mb;
-    if (params_.greedy_block_split) {
-      BuildMetaBlockGreedy(data, last_flush_pos_, mask,
-                           commands_.get(), num_commands_,
-                           &mb);
+    if (params_.quality == 1) {
+      if (!StoreMetaBlockTrivial(data, last_flush_pos_, bytes, mask, is_last,
+                                 commands_.get(), num_commands_,
+                                 &storage_ix,
+                                 &storage[0])) {
+        return false;
+      }
     } else {
-      BuildMetaBlock(data, last_flush_pos_, mask,
-                     prev_byte_, prev_byte2_,
-                     commands_.get(), num_commands_,
-                     literal_context_mode,
-                     params_.enable_context_modeling,
-                     &mb);
-    }
-    if (params_.quality >= 3) {
-      OptimizeHistograms(num_direct_distance_codes,
-                         distance_postfix_bits,
-                         &mb);
-    }
-    if (!StoreMetaBlock(data, last_flush_pos_, bytes, mask,
-                        prev_byte_, prev_byte2_,
-                        is_last,
-                        num_direct_distance_codes,
-                        distance_postfix_bits,
-                        literal_context_mode,
-                        commands_.get(), num_commands_,
-                        mb,
-                        &storage_ix,
-                        &storage[0])) {
-      return false;
+      if (params_.greedy_block_split) {
+        BuildMetaBlockGreedy(data, last_flush_pos_, mask,
+                             commands_.get(), num_commands_,
+                             &mb);
+      } else {
+        BuildMetaBlock(data, last_flush_pos_, mask,
+                       prev_byte_, prev_byte2_,
+                       commands_.get(), num_commands_,
+                       literal_context_mode,
+                       params_.enable_context_modeling,
+                       &mb);
+      }
+      if (params_.quality >= 3) {
+        OptimizeHistograms(num_direct_distance_codes,
+                           distance_postfix_bits,
+                           &mb);
+      }
+      if (!StoreMetaBlock(data, last_flush_pos_, bytes, mask,
+                          prev_byte_, prev_byte2_,
+                          is_last,
+                          num_direct_distance_codes,
+                          distance_postfix_bits,
+                          literal_context_mode,
+                          commands_.get(), num_commands_,
+                          mb,
+                          &storage_ix,
+                          &storage[0])) {
+        return false;
+      }
     }
     if (bytes + 4 < (storage_ix >> 3)) {
       // Restore the distance cache and last byte.
@@ -451,6 +464,7 @@ bool BrotliCompressor::WriteMetaBlockInternal(const bool is_last,
   prev_byte_ = data[(last_flush_pos_ - 1) & mask];
   prev_byte2_ = data[(last_flush_pos_ - 2) & mask];
   num_commands_ = 0;
+  num_literals_ = 0;
   *output = &storage[0];
   *out_size = storage_ix >> 3;
   return true;

enc/encode.h

@@ -148,6 +148,7 @@ class BrotliCompressor {
   size_t cmd_buffer_size_;
   std::unique_ptr<Command[]> commands_;
   int num_commands_;
+  int num_literals_;
   int last_insert_len_;
   size_t last_flush_pos_;
   size_t last_processed_pos_;

enc/encode_parallel.cc

@@ -175,6 +175,7 @@ bool WriteMetaBlockParallel(const BrotliParams& params,
   // Compute backward references.
   int last_insert_len = 0;
   int num_commands = 0;
+  int num_literals = 0;
   double base_min_score = 8.115;
   int max_backward_distance = (1 << params.lgwin) - 16;
   int dist_cache[4] = { -4, -4, -4, -4 };
@@ -191,10 +192,12 @@ bool WriteMetaBlockParallel(const BrotliParams& params,
       dist_cache,
       &last_insert_len,
       &commands[0],
-      &num_commands);
+      &num_commands,
+      &num_literals);
   commands.resize(num_commands);
   if (last_insert_len > 0) {
     commands.push_back(Command(last_insert_len));
+    num_literals += last_insert_len;
   }
 
   // Build the meta-block.