AuroraMiddleware/brotli
1
0
Fork 0
mirror of https://github.com/google/brotli.git synced 2024-11-26 13:20:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #119 from szabadka/master

Browse Source 
Deprecate greedy_block_split and enable_context_modeling brotli params.
This commit is contained in:
szabadka 2015-06-12 16:55:09 +02:00
commit 67d26e0d46
8 changed files with 32 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
enc/backward_references.cc
View File

@ -305,8 +305,6 @@ void ZopfliIterate(size_t num_bytes,
const uint8_t* ringbuffer, const uint8_t* ringbuffer,
size_t ringbuffer_mask, size_t ringbuffer_mask,
const size_t max_backward_limit, const size_t max_backward_limit,
const double base_min_score,
const int quality,
const ZopfliCostModel& model, const ZopfliCostModel& model,
const std::vector<int>& num_matches, const std::vector<int>& num_matches,
const std::vector<BackwardMatch>& matches, const std::vector<BackwardMatch>& matches,
@ -479,7 +477,6 @@ void CreateBackwardReferences(size_t num_bytes,
const uint8_t* ringbuffer, const uint8_t* ringbuffer,
size_t ringbuffer_mask, size_t ringbuffer_mask,
const size_t max_backward_limit, const size_t max_backward_limit,
const double base_min_score,
const int quality, const int quality,
Hasher* hasher, Hasher* hasher,
int* dist_cache, int* dist_cache,
@ -508,13 +505,16 @@ void CreateBackwardReferences(size_t num_bytes,
const int random_heuristics_window_size = quality < 9 ? 64 : 512; const int random_heuristics_window_size = quality < 9 ? 64 : 512;
int apply_random_heuristics = i + random_heuristics_window_size; int apply_random_heuristics = i + random_heuristics_window_size;
// Minimum score to accept a backward reference.
const int kMinScore = 4.0;
while (i + 3 < i_end) { while (i + 3 < i_end) {
int max_length = i_end - i; int max_length = i_end - i;
size_t max_distance = std::min(i + i_diff, max_backward_limit); size_t max_distance = std::min(i + i_diff, max_backward_limit);
int best_len = 0; int best_len = 0;
int best_len_code = 0; int best_len_code = 0;
int best_dist = 0; int best_dist = 0;
double best_score = base_min_score; double best_score = kMinScore;
bool match_found = hasher->FindLongestMatch( bool match_found = hasher->FindLongestMatch(
ringbuffer, ringbuffer_mask, ringbuffer, ringbuffer_mask,
dist_cache, i + i_diff, max_length, max_distance, dist_cache, i + i_diff, max_length, max_distance,
@ -527,7 +527,7 @@ void CreateBackwardReferences(size_t num_bytes,
int best_len_2 = quality < 5 ? std::min(best_len - 1, max_length) : 0; int best_len_2 = quality < 5 ? std::min(best_len - 1, max_length) : 0;
int best_len_code_2 = 0; int best_len_code_2 = 0;
int best_dist_2 = 0; int best_dist_2 = 0;
double best_score_2 = base_min_score; double best_score_2 = kMinScore;
max_distance = std::min(i + i_diff + 1, max_backward_limit); max_distance = std::min(i + i_diff + 1, max_backward_limit);
hasher->Store(ringbuffer + i, i + i_diff); hasher->Store(ringbuffer + i, i + i_diff);
match_found = hasher->FindLongestMatch( match_found = hasher->FindLongestMatch(
@ -616,7 +616,6 @@ void CreateBackwardReferences(size_t num_bytes,
const float* literal_cost, const float* literal_cost,
size_t literal_cost_mask, size_t literal_cost_mask,
const size_t max_backward_limit, const size_t max_backward_limit,
const double base_min_score,
const int quality, const int quality,
Hashers* hashers, Hashers* hashers,
int hash_type, int hash_type,
@ -691,8 +690,7 @@ void CreateBackwardReferences(size_t num_bytes,
*last_insert_len = orig_last_insert_len; *last_insert_len = orig_last_insert_len;
memcpy(dist_cache, orig_dist_cache, 4 * sizeof(dist_cache[0])); memcpy(dist_cache, orig_dist_cache, 4 * sizeof(dist_cache[0]));
ZopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, ZopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask,
max_backward_limit, base_min_score, max_backward_limit, model, num_matches, matches, dist_cache,
quality, model, num_matches, matches, dist_cache,
last_insert_len, commands, num_commands, num_literals); last_insert_len, commands, num_commands, num_literals);
} }
return; return;
@ -701,64 +699,55 @@ void CreateBackwardReferences(size_t num_bytes,
switch (hash_type) { switch (hash_type) {
case 1: case 1:
CreateBackwardReferences<Hashers::H1>( CreateBackwardReferences<Hashers::H1>(
num_bytes, position, ringbuffer, ringbuffer_mask, num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
max_backward_limit, base_min_score,
quality, hashers->hash_h1.get(), dist_cache, last_insert_len, quality, hashers->hash_h1.get(), dist_cache, last_insert_len,
commands, num_commands, num_literals); commands, num_commands, num_literals);
break; break;
case 2: case 2:
CreateBackwardReferences<Hashers::H2>( CreateBackwardReferences<Hashers::H2>(
num_bytes, position, ringbuffer, ringbuffer_mask, num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
max_backward_limit, base_min_score,
quality, hashers->hash_h2.get(), dist_cache, last_insert_len, quality, hashers->hash_h2.get(), dist_cache, last_insert_len,
commands, num_commands, num_literals); commands, num_commands, num_literals);
break; break;
case 3: case 3:
CreateBackwardReferences<Hashers::H3>( CreateBackwardReferences<Hashers::H3>(
num_bytes, position, ringbuffer, ringbuffer_mask, num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
max_backward_limit, base_min_score,
quality, hashers->hash_h3.get(), dist_cache, last_insert_len, quality, hashers->hash_h3.get(), dist_cache, last_insert_len,
commands, num_commands, num_literals); commands, num_commands, num_literals);
break; break;
case 4: case 4:
CreateBackwardReferences<Hashers::H4>( CreateBackwardReferences<Hashers::H4>(
num_bytes, position, ringbuffer, ringbuffer_mask, num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
max_backward_limit, base_min_score,
quality, hashers->hash_h4.get(), dist_cache, last_insert_len, quality, hashers->hash_h4.get(), dist_cache, last_insert_len,
commands, num_commands, num_literals); commands, num_commands, num_literals);
break; break;
case 5: case 5:
CreateBackwardReferences<Hashers::H5>( CreateBackwardReferences<Hashers::H5>(
num_bytes, position, ringbuffer, ringbuffer_mask, num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
max_backward_limit, base_min_score,
quality, hashers->hash_h5.get(), dist_cache, last_insert_len, quality, hashers->hash_h5.get(), dist_cache, last_insert_len,
commands, num_commands, num_literals); commands, num_commands, num_literals);
break; break;
case 6: case 6:
CreateBackwardReferences<Hashers::H6>( CreateBackwardReferences<Hashers::H6>(
num_bytes, position, ringbuffer, ringbuffer_mask, num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
max_backward_limit, base_min_score,
quality, hashers->hash_h6.get(), dist_cache, last_insert_len, quality, hashers->hash_h6.get(), dist_cache, last_insert_len,
commands, num_commands, num_literals); commands, num_commands, num_literals);
break; break;
case 7: case 7:
CreateBackwardReferences<Hashers::H7>( CreateBackwardReferences<Hashers::H7>(
num_bytes, position, ringbuffer, ringbuffer_mask, num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
max_backward_limit, base_min_score,
quality, hashers->hash_h7.get(), dist_cache, last_insert_len, quality, hashers->hash_h7.get(), dist_cache, last_insert_len,
commands, num_commands, num_literals); commands, num_commands, num_literals);
break; break;
case 8: case 8:
CreateBackwardReferences<Hashers::H8>( CreateBackwardReferences<Hashers::H8>(
num_bytes, position, ringbuffer, ringbuffer_mask, num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
max_backward_limit, base_min_score,
quality, hashers->hash_h8.get(), dist_cache, last_insert_len, quality, hashers->hash_h8.get(), dist_cache, last_insert_len,
commands, num_commands, num_literals); commands, num_commands, num_literals);
break; break;
case 9: case 9:
CreateBackwardReferences<Hashers::H9>( CreateBackwardReferences<Hashers::H9>(
num_bytes, position, ringbuffer, ringbuffer_mask, num_bytes, position, ringbuffer, ringbuffer_mask, max_backward_limit,
max_backward_limit, base_min_score,
quality, hashers->hash_h9.get(), dist_cache, last_insert_len, quality, hashers->hash_h9.get(), dist_cache, last_insert_len,
commands, num_commands, num_literals); commands, num_commands, num_literals);
break; break;

1
enc/backward_references.h
View File

@ -36,7 +36,6 @@ void CreateBackwardReferences(size_t num_bytes,
const float* literal_cost, const float* literal_cost,
size_t literal_cost_mask, size_t literal_cost_mask,
const size_t max_backward_limit, const size_t max_backward_limit,
const double base_min_score,
const int quality, const int quality,
Hashers* hashers, Hashers* hashers,
int hash_type, int hash_type,

16
enc/cluster.h
View File

@ -244,22 +244,6 @@ void HistogramReindex(std::vector<HistogramType>* out,
} }
} }
template<typename HistogramType>
void ClusterHistogramsTrivial(const std::vector<HistogramType>& in,
int num_contexts, int num_blocks,
int max_histograms,
std::vector<HistogramType>* out,
std::vector<int>* histogram_symbols) {
out->resize(num_blocks);
for (int i = 0; i < num_blocks; ++i) {
(*out)[i].Clear();
for (int j = 0; j < num_contexts; ++j) {
(*out)[i].AddHistogram(in[i * num_contexts + j]);
histogram_symbols->push_back(i);
}
}
}
// Clusters similar histograms in 'in' together, the selected histograms are // Clusters similar histograms in 'in' together, the selected histograms are
// placed in 'out', and for each index in 'in', *histogram_symbols will // placed in 'out', and for each index in 'in', *histogram_symbols will
// indicate which of the 'out' histograms is the best approximation. // indicate which of the 'out' histograms is the best approximation.

11
enc/encode.cc
View File

@ -160,10 +160,6 @@ BrotliCompressor::BrotliCompressor(BrotliParams params)
params_.lgblock = std::min(kMaxInputBlockBits, params_.lgblock = std::min(kMaxInputBlockBits,
std::max(kMinInputBlockBits, params_.lgblock)); std::max(kMinInputBlockBits, params_.lgblock));
} }
if (params_.quality <= 9) {
params_.greedy_block_split = true;
params_.enable_context_modeling = false;
}
// Set maximum distance, see section 9.1. of the spec. // Set maximum distance, see section 9.1. of the spec.
max_backward_distance_ = (1 << params_.lgwin) - 16; max_backward_distance_ = (1 << params_.lgwin) - 16;
@ -288,7 +284,7 @@ bool BrotliCompressor::WriteBrotliData(const bool is_last,
} }
bool utf8_mode = bool utf8_mode =
params_.enable_context_modeling && params_.quality >= 9 &&
IsMostlyUTF8(&data[last_processed_pos_ & mask], bytes, kMinUTF8Ratio); IsMostlyUTF8(&data[last_processed_pos_ & mask], bytes, kMinUTF8Ratio);
if (literal_cost_.get()) { if (literal_cost_.get()) {
@ -302,12 +298,10 @@ bool BrotliCompressor::WriteBrotliData(const bool is_last,
data, literal_cost_.get()); data, literal_cost_.get());
} }
} }
double base_min_score = params_.enable_context_modeling ? 8.115 : 4.0;
CreateBackwardReferences(bytes, last_processed_pos_, data, mask, CreateBackwardReferences(bytes, last_processed_pos_, data, mask,
literal_cost_.get(), literal_cost_.get(),
literal_cost_mask_, literal_cost_mask_,
max_backward_distance_, max_backward_distance_,
base_min_score,
params_.quality, params_.quality,
hashers_.get(), hashers_.get(),
hash_type_, hash_type_,
@ -451,7 +445,7 @@ bool BrotliCompressor::WriteMetaBlockInternal(const bool is_last,
} else { } else {
MetaBlockSplit mb; MetaBlockSplit mb;
int literal_context_mode = utf8_mode ? CONTEXT_UTF8 : CONTEXT_SIGNED; int literal_context_mode = utf8_mode ? CONTEXT_UTF8 : CONTEXT_SIGNED;
if (params_.greedy_block_split) { if (params_.quality <= 9) {
int num_literal_contexts = 1; int num_literal_contexts = 1;
const int* literal_context_map = NULL; const int* literal_context_map = NULL;
DecideOverLiteralContextModeling(data, last_flush_pos_, bytes, mask, DecideOverLiteralContextModeling(data, last_flush_pos_, bytes, mask,
@ -477,7 +471,6 @@ bool BrotliCompressor::WriteMetaBlockInternal(const bool is_last,
prev_byte_, prev_byte2_, prev_byte_, prev_byte2_,
commands_.get(), num_commands_, commands_.get(), num_commands_,
literal_context_mode, literal_context_mode,
params_.enable_context_modeling,
&mb); &mb);
} }
if (params_.quality >= kMinQualityForOptimizeHistograms) { if (params_.quality >= kMinQualityForOptimizeHistograms) {

3
enc/encode.h
View File

@ -65,7 +65,8 @@ struct BrotliParams {
// If set to 0, the value will be set based on the quality. // If set to 0, the value will be set based on the quality.
int lgblock; int lgblock;
// These settings will be respected only if quality > 9. // These settings are deprecated and will be ignored.
// All speed vs. size compromises are controlled by the quality param.
bool enable_dictionary; bool enable_dictionary;
bool enable_transforms; bool enable_transforms;
bool greedy_block_split; bool greedy_block_split;

5
enc/encode_parallel.cc
View File

@ -173,7 +173,6 @@ bool WriteMetaBlockParallel(const BrotliParams& params,
int last_insert_len = 0; int last_insert_len = 0;
int num_commands = 0; int num_commands = 0;
int num_literals = 0; int num_literals = 0;
double base_min_score = 8.115;
int max_backward_distance = (1 << params.lgwin) - 16; int max_backward_distance = (1 << params.lgwin) - 16;
int dist_cache[4] = { -4, -4, -4, -4 }; int dist_cache[4] = { -4, -4, -4, -4 };
std::vector<Command> commands((input_size + 1) >> 1); std::vector<Command> commands((input_size + 1) >> 1);
@ -182,7 +181,6 @@ bool WriteMetaBlockParallel(const BrotliParams& params,
&input[0], mask, &input[0], mask,
&literal_cost[0], mask, &literal_cost[0], mask,
max_backward_distance, max_backward_distance,
base_min_score,
params.quality, params.quality,
hashers.get(), hashers.get(),
hash_type, hash_type,
@ -206,7 +204,7 @@ bool WriteMetaBlockParallel(const BrotliParams& params,
RecomputeDistancePrefixes(&commands, RecomputeDistancePrefixes(&commands,
num_direct_distance_codes, num_direct_distance_codes,
distance_postfix_bits); distance_postfix_bits);
if (params.greedy_block_split) { if (params.quality <= 9) {
BuildMetaBlockGreedy(&input[0], input_pos, mask, BuildMetaBlockGreedy(&input[0], input_pos, mask,
commands.data(), commands.size(), commands.data(), commands.size(),
&mb); &mb);
@ -215,7 +213,6 @@ bool WriteMetaBlockParallel(const BrotliParams& params,
prev_byte, prev_byte2, prev_byte, prev_byte2,
commands.data(), commands.size(), commands.data(), commands.size(),
literal_context_mode, literal_context_mode,
true,
&mb); &mb);
} }

43
enc/metablock.cc
View File

@ -32,7 +32,6 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
const Command* cmds, const Command* cmds,
size_t num_commands, size_t num_commands,
int literal_context_mode, int literal_context_mode,
bool enable_context_modeling,
MetaBlockSplit* mb) { MetaBlockSplit* mb) {
SplitBlock(cmds, num_commands, SplitBlock(cmds, num_commands,
&ringbuffer[pos & mask], &ringbuffer[pos & mask],
@ -68,38 +67,20 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
static const int kMaxNumberOfHistograms = 256; static const int kMaxNumberOfHistograms = 256;
mb->literal_histograms = literal_histograms; mb->literal_histograms = literal_histograms;
if (enable_context_modeling) { ClusterHistograms(literal_histograms,
ClusterHistograms(literal_histograms, 1 << kLiteralContextBits,
1 << kLiteralContextBits, mb->literal_split.num_types,
mb->literal_split.num_types, kMaxNumberOfHistograms,
kMaxNumberOfHistograms, &mb->literal_histograms,
&mb->literal_histograms, &mb->literal_context_map);
&mb->literal_context_map);
} else {
ClusterHistogramsTrivial(literal_histograms,
1 << kLiteralContextBits,
mb->literal_split.num_types,
kMaxNumberOfHistograms,
&mb->literal_histograms,
&mb->literal_context_map);
}
mb->distance_histograms = distance_histograms; mb->distance_histograms = distance_histograms;
if (enable_context_modeling) { ClusterHistograms(distance_histograms,
ClusterHistograms(distance_histograms, 1 << kDistanceContextBits,
1 << kDistanceContextBits, mb->distance_split.num_types,
mb->distance_split.num_types, kMaxNumberOfHistograms,
kMaxNumberOfHistograms, &mb->distance_histograms,
&mb->distance_histograms, &mb->distance_context_map);
&mb->distance_context_map);
} else {
ClusterHistogramsTrivial(distance_histograms,
1 << kDistanceContextBits,
mb->distance_split.num_types,
kMaxNumberOfHistograms,
&mb->distance_histograms,
&mb->distance_context_map);
}
} }
// Greedy block splitter for one block category (literal, command or distance). // Greedy block splitter for one block category (literal, command or distance).

1
enc/metablock.h
View File

@ -53,7 +53,6 @@ void BuildMetaBlock(const uint8_t* ringbuffer,
const Command* cmds, const Command* cmds,
size_t num_commands, size_t num_commands,
int literal_context_mode, int literal_context_mode,
bool enable_context_modleing,
MetaBlockSplit* mb); MetaBlockSplit* mb);
// Uses a fast greedy block splitter that tries to merge current block with the // Uses a fast greedy block splitter that tries to merge current block with the