From 618287b3734816a2e86bfc2a7e3922fee944b521 Mon Sep 17 00:00:00 2001 From: Zoltan Szabadka Date: Fri, 12 Jun 2015 16:50:49 +0200 Subject: [PATCH] Deprecate greedy_block_split and enable_context_modeling brotli params. These affected only quality 11, and now it does not make sense to disable block splitting or context modeling because most of the time is spent in zopfli anyway. Now all speed vs size compromises are controlled by the quality param. --- enc/backward_references.cc | 41 +++++++++++++----------------------- enc/backward_references.h | 1 - enc/cluster.h | 16 -------------- enc/encode.cc | 11 ++-------- enc/encode.h | 3 ++- enc/encode_parallel.cc | 5 +---- enc/metablock.cc | 43 +++++++++++--------------------------- enc/metablock.h | 1 - 8 files changed, 32 insertions(+), 89 deletions(-) diff --git a/enc/backward_references.cc b/enc/backward_references.cc index 99922f0..47b42c4 100644 --- a/enc/backward_references.cc +++ b/enc/backward_references.cc @@ -305,8 +305,6 @@ void ZopfliIterate(size_t num_bytes, const uint8_t* ringbuffer, size_t ringbuffer_mask, const size_t max_backward_limit, - const double base_min_score, - const int quality, const ZopfliCostModel& model, const std::vector& num_matches, const std::vector& matches, @@ -479,7 +477,6 @@ void CreateBackwardReferences(size_t num_bytes, const uint8_t* ringbuffer, size_t ringbuffer_mask, const size_t max_backward_limit, - const double base_min_score, const int quality, Hasher* hasher, int* dist_cache, @@ -508,13 +505,16 @@ void CreateBackwardReferences(size_t num_bytes, const int random_heuristics_window_size = quality < 9 ? 64 : 512; int apply_random_heuristics = i + random_heuristics_window_size; + // Minimum score to accept a backward reference. + const int kMinScore = 4.0; + while (i + 3 < i_end) { int max_length = i_end - i; size_t max_distance = std::min(i + i_diff, max_backward_limit); int best_len = 0; int best_len_code = 0; int best_dist = 0; - double best_score = base_min_score; + double best_score = kMinScore; bool match_found = hasher->FindLongestMatch( ringbuffer, ringbuffer_mask, dist_cache, i + i_diff, max_length, max_distance, @@ -527,7 +527,7 @@ void CreateBackwardReferences(size_t num_bytes, int best_len_2 = quality < 5 ? std::min(best_len - 1, max_length) : 0; int best_len_code_2 = 0; int best_dist_2 = 0; - double best_score_2 = base_min_score; + double best_score_2 = kMinScore; max_distance = std::min(i + i_diff + 1, max_backward_limit); hasher->Store(ringbuffer + i, i + i_diff); match_found = hasher->FindLongestMatch( @@ -616,7 +616,6 @@ void CreateBackwardReferences(size_t num_bytes, const float* literal_cost, size_t literal_cost_mask, const size_t max_backward_limit, - const double base_min_score, const int quality, Hashers* hashers, int hash_type, @@ -691,8 +690,7 @@ void CreateBackwardReferences(size_t num_bytes, *last_insert_len = orig_last_insert_len; memcpy(dist_cache, orig_dist_cache, 4 * sizeof(dist_cache[0])); ZopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, - max_backward_limit, base_min_score, - quality, model, num_matches, matches, dist_cache, + max_backward_limit, model, num_matches, matches, dist_cache, last_insert_len, commands, num_commands, num_literals); } return; @@ -701,64 +699,55 @@ void CreateBackwardReferences(size_t num_bytes, switch (hash_type) { case 1: CreateBackwardReferences( - num_bytes, position, ringbuffer, ringbuffer_mask, - max_backward_limit, base_min_score, + num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit, quality, hashers->hash_h1.get(), dist_cache, last_insert_len, commands, num_commands, num_literals); break; case 2: CreateBackwardReferences( - num_bytes, position, ringbuffer, ringbuffer_mask, - max_backward_limit, base_min_score, + num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit, quality, hashers->hash_h2.get(), dist_cache, last_insert_len, commands, num_commands, num_literals); break; case 3: CreateBackwardReferences( - num_bytes, position, ringbuffer, ringbuffer_mask, - max_backward_limit, base_min_score, + num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit, quality, hashers->hash_h3.get(), dist_cache, last_insert_len, commands, num_commands, num_literals); break; case 4: CreateBackwardReferences( - num_bytes, position, ringbuffer, ringbuffer_mask, - max_backward_limit, base_min_score, + num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit, quality, hashers->hash_h4.get(), dist_cache, last_insert_len, commands, num_commands, num_literals); break; case 5: CreateBackwardReferences( - num_bytes, position, ringbuffer, ringbuffer_mask, - max_backward_limit, base_min_score, + num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit, quality, hashers->hash_h5.get(), dist_cache, last_insert_len, commands, num_commands, num_literals); break; case 6: CreateBackwardReferences( - num_bytes, position, ringbuffer, ringbuffer_mask, - max_backward_limit, base_min_score, + num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit, quality, hashers->hash_h6.get(), dist_cache, last_insert_len, commands, num_commands, num_literals); break; case 7: CreateBackwardReferences( - num_bytes, position, ringbuffer, ringbuffer_mask, - max_backward_limit, base_min_score, + num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit, quality, hashers->hash_h7.get(), dist_cache, last_insert_len, commands, num_commands, num_literals); break; case 8: CreateBackwardReferences( - num_bytes, position, ringbuffer, ringbuffer_mask, - max_backward_limit, base_min_score, + num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit, quality, hashers->hash_h8.get(), dist_cache, last_insert_len, commands, num_commands, num_literals); break; case 9: CreateBackwardReferences( - num_bytes, position, ringbuffer, ringbuffer_mask, - max_backward_limit, base_min_score, + num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit, quality, hashers->hash_h9.get(), dist_cache, last_insert_len, commands, num_commands, num_literals); break; diff --git a/enc/backward_references.h b/enc/backward_references.h index b3d7dd0..64bc7e8 100644 --- a/enc/backward_references.h +++ b/enc/backward_references.h @@ -36,7 +36,6 @@ void CreateBackwardReferences(size_t num_bytes, const float* literal_cost, size_t literal_cost_mask, const size_t max_backward_limit, - const double base_min_score, const int quality, Hashers* hashers, int hash_type, diff --git a/enc/cluster.h b/enc/cluster.h index d14a727..500debd 100644 --- a/enc/cluster.h +++ b/enc/cluster.h @@ -244,22 +244,6 @@ void HistogramReindex(std::vector* out, } } -template -void ClusterHistogramsTrivial(const std::vector& in, - int num_contexts, int num_blocks, - int max_histograms, - std::vector* out, - std::vector* histogram_symbols) { - out->resize(num_blocks); - for (int i = 0; i < num_blocks; ++i) { - (*out)[i].Clear(); - for (int j = 0; j < num_contexts; ++j) { - (*out)[i].AddHistogram(in[i * num_contexts + j]); - histogram_symbols->push_back(i); - } - } -} - // Clusters similar histograms in 'in' together, the selected histograms are // placed in 'out', and for each index in 'in', *histogram_symbols will // indicate which of the 'out' histograms is the best approximation. diff --git a/enc/encode.cc b/enc/encode.cc index a2320bb..66bed34 100644 --- a/enc/encode.cc +++ b/enc/encode.cc @@ -160,10 +160,6 @@ BrotliCompressor::BrotliCompressor(BrotliParams params) params_.lgblock = std::min(kMaxInputBlockBits, std::max(kMinInputBlockBits, params_.lgblock)); } - if (params_.quality <= 9) { - params_.greedy_block_split = true; - params_.enable_context_modeling = false; - } // Set maximum distance, see section 9.1. of the spec. max_backward_distance_ = (1 << params_.lgwin) - 16; @@ -288,7 +284,7 @@ bool BrotliCompressor::WriteBrotliData(const bool is_last, } bool utf8_mode = - params_.enable_context_modeling && + params_.quality >= 9 && IsMostlyUTF8(&data[last_processed_pos_ & mask], bytes, kMinUTF8Ratio); if (literal_cost_.get()) { @@ -302,12 +298,10 @@ bool BrotliCompressor::WriteBrotliData(const bool is_last, data, literal_cost_.get()); } } - double base_min_score = params_.enable_context_modeling ? 8.115 : 4.0; CreateBackwardReferences(bytes, last_processed_pos_, data, mask, literal_cost_.get(), literal_cost_mask_, max_backward_distance_, - base_min_score, params_.quality, hashers_.get(), hash_type_, @@ -451,7 +445,7 @@ bool BrotliCompressor::WriteMetaBlockInternal(const bool is_last, } else { MetaBlockSplit mb; int literal_context_mode = utf8_mode ? CONTEXT_UTF8 : CONTEXT_SIGNED; - if (params_.greedy_block_split) { + if (params_.quality <= 9) { int num_literal_contexts = 1; const int* literal_context_map = NULL; DecideOverLiteralContextModeling(data, last_flush_pos_, bytes, mask, @@ -477,7 +471,6 @@ bool BrotliCompressor::WriteMetaBlockInternal(const bool is_last, prev_byte_, prev_byte2_, commands_.get(), num_commands_, literal_context_mode, - params_.enable_context_modeling, &mb); } if (params_.quality >= kMinQualityForOptimizeHistograms) { diff --git a/enc/encode.h b/enc/encode.h index b894c95..7f45d5f 100644 --- a/enc/encode.h +++ b/enc/encode.h @@ -65,7 +65,8 @@ struct BrotliParams { // If set to 0, the value will be set based on the quality. int lgblock; - // These settings will be respected only if quality > 9. + // These settings are deprecated and will be ignored. + // All speed vs. size compromises are controlled by the quality param. bool enable_dictionary; bool enable_transforms; bool greedy_block_split; diff --git a/enc/encode_parallel.cc b/enc/encode_parallel.cc index c61f630..13bffc8 100644 --- a/enc/encode_parallel.cc +++ b/enc/encode_parallel.cc @@ -173,7 +173,6 @@ bool WriteMetaBlockParallel(const BrotliParams& params, int last_insert_len = 0; int num_commands = 0; int num_literals = 0; - double base_min_score = 8.115; int max_backward_distance = (1 << params.lgwin) - 16; int dist_cache[4] = { -4, -4, -4, -4 }; std::vector commands((input_size + 1) >> 1); @@ -182,7 +181,6 @@ bool WriteMetaBlockParallel(const BrotliParams& params, &input[0], mask, &literal_cost[0], mask, max_backward_distance, - base_min_score, params.quality, hashers.get(), hash_type, @@ -206,7 +204,7 @@ bool WriteMetaBlockParallel(const BrotliParams& params, RecomputeDistancePrefixes(&commands, num_direct_distance_codes, distance_postfix_bits); - if (params.greedy_block_split) { + if (params.quality <= 9) { BuildMetaBlockGreedy(&input[0], input_pos, mask, commands.data(), commands.size(), &mb); @@ -215,7 +213,6 @@ bool WriteMetaBlockParallel(const BrotliParams& params, prev_byte, prev_byte2, commands.data(), commands.size(), literal_context_mode, - true, &mb); } diff --git a/enc/metablock.cc b/enc/metablock.cc index 44f8267..f4d22da 100644 --- a/enc/metablock.cc +++ b/enc/metablock.cc @@ -32,7 +32,6 @@ void BuildMetaBlock(const uint8_t* ringbuffer, const Command* cmds, size_t num_commands, int literal_context_mode, - bool enable_context_modeling, MetaBlockSplit* mb) { SplitBlock(cmds, num_commands, &ringbuffer[pos & mask], @@ -68,38 +67,20 @@ void BuildMetaBlock(const uint8_t* ringbuffer, static const int kMaxNumberOfHistograms = 256; mb->literal_histograms = literal_histograms; - if (enable_context_modeling) { - ClusterHistograms(literal_histograms, - 1 << kLiteralContextBits, - mb->literal_split.num_types, - kMaxNumberOfHistograms, - &mb->literal_histograms, - &mb->literal_context_map); - } else { - ClusterHistogramsTrivial(literal_histograms, - 1 << kLiteralContextBits, - mb->literal_split.num_types, - kMaxNumberOfHistograms, - &mb->literal_histograms, - &mb->literal_context_map); - } + ClusterHistograms(literal_histograms, + 1 << kLiteralContextBits, + mb->literal_split.num_types, + kMaxNumberOfHistograms, + &mb->literal_histograms, + &mb->literal_context_map); mb->distance_histograms = distance_histograms; - if (enable_context_modeling) { - ClusterHistograms(distance_histograms, - 1 << kDistanceContextBits, - mb->distance_split.num_types, - kMaxNumberOfHistograms, - &mb->distance_histograms, - &mb->distance_context_map); - } else { - ClusterHistogramsTrivial(distance_histograms, - 1 << kDistanceContextBits, - mb->distance_split.num_types, - kMaxNumberOfHistograms, - &mb->distance_histograms, - &mb->distance_context_map); - } + ClusterHistograms(distance_histograms, + 1 << kDistanceContextBits, + mb->distance_split.num_types, + kMaxNumberOfHistograms, + &mb->distance_histograms, + &mb->distance_context_map); } // Greedy block splitter for one block category (literal, command or distance). diff --git a/enc/metablock.h b/enc/metablock.h index 1946436..f98a2e3 100644 --- a/enc/metablock.h +++ b/enc/metablock.h @@ -53,7 +53,6 @@ void BuildMetaBlock(const uint8_t* ringbuffer, const Command* cmds, size_t num_commands, int literal_context_mode, - bool enable_context_modleing, MetaBlockSplit* mb); // Uses a fast greedy block splitter that tries to merge current block with the