mirror of
https://github.com/google/brotli.git
synced 2024-11-21 19:20:09 +00:00
Merge pull request #3 from szabadka/encoder
Factor out serialization functions into their own file.
This commit is contained in:
szabadka
commit
fd64d1f35a
@ -91,7 +91,7 @@ static inline int HuffmanBitCost(const uint8_t* depth, int length) {
|
||||
|
||||
// create huffman tree of huffman tree
|
||||
uint8_t cost[kCodeLengthCodes] = { 0 };
|
||||
CreateHuffmanTree(histogram, kCodeLengthCodes, 7, cost);
|
||||
CreateHuffmanTree(histogram, kCodeLengthCodes, 7, 9, cost);
|
||||
// account for rle extra bits
|
||||
cost[16] += 2;
|
||||
cost[17] += 3;
|
||||
@ -123,7 +123,7 @@ double PopulationCost(const Histogram<kSize>& histogram) {
|
||||
return 20 + histogram.total_count_;
|
||||
}
|
||||
uint8_t depth[kSize] = { 0 };
|
||||
CreateHuffmanTree(&histogram.data_[0], kSize, 15, depth);
|
||||
CreateHuffmanTree(&histogram.data_[0], kSize, 15, 9, depth);
|
||||
int bits = 0;
|
||||
for (int i = 0; i < kSize; ++i) {
|
||||
bits += histogram.data_[i] * depth[i];
|
||||
|
||||
342
enc/brotli_bit_stream.cc
Normal file
342
enc/brotli_bit_stream.cc
Normal file
@ -0,0 +1,342 @@
|
||||
// Copyright 2014 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Brotli bit stream functions to support the low level format. There are no
|
||||
// compression algorithms here, just the right ordering of bits to match the
|
||||
// specs.
|
||||
|
||||
#include "./brotli_bit_stream.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "./entropy_encode.h"
|
||||
#include "./fast_log.h"
|
||||
#include "./write_bits.h"
|
||||
|
||||
namespace brotli {
|
||||
|
||||
// returns false if fail
|
||||
// nibblesbits represents the 2 bits to encode MNIBBLES (0-3)
|
||||
bool EncodeMlen(size_t length, int* bits, int* numbits, int* nibblesbits) {
|
||||
length--; // MLEN - 1 is encoded
|
||||
int lg = length == 0 ? 1 : Log2Floor(length) + 1;
|
||||
if (lg > 28) return false;
|
||||
int mnibbles = (lg < 16 ? 16 : (lg + 3)) / 4;
|
||||
*nibblesbits = mnibbles - 4;
|
||||
*numbits = mnibbles * 4;
|
||||
*bits = length;
|
||||
return true;
|
||||
}
|
||||
|
||||
void StoreVarLenUint8(int n, int* storage_ix, uint8_t* storage) {
|
||||
if (n == 0) {
|
||||
WriteBits(1, 0, storage_ix, storage);
|
||||
} else {
|
||||
WriteBits(1, 1, storage_ix, storage);
|
||||
int nbits = Log2Floor(n);
|
||||
WriteBits(3, nbits, storage_ix, storage);
|
||||
WriteBits(nbits, n - (1 << nbits), storage_ix, storage);
|
||||
}
|
||||
}
|
||||
|
||||
void StoreCompressedMetaBlockHeader(bool final_block,
|
||||
int length,
|
||||
int* storage_ix,
|
||||
uint8_t* storage) {
|
||||
// Write ISLAST bit.
|
||||
WriteBits(1, final_block, storage_ix, storage);
|
||||
// Write ISEMPTY bit.
|
||||
if (final_block) WriteBits(1, length == 0, storage_ix, storage);
|
||||
|
||||
int lenbits;
|
||||
int nlenbits;
|
||||
int nibblesbits;
|
||||
EncodeMlen(length, &lenbits, &nlenbits, &nibblesbits);
|
||||
|
||||
WriteBits(2, nibblesbits, storage_ix, storage);
|
||||
WriteBits(nlenbits, lenbits, storage_ix, storage);
|
||||
|
||||
if (!final_block) {
|
||||
// Write ISUNCOMPRESSED bit.
|
||||
WriteBits(1, 0, storage_ix, storage);
|
||||
}
|
||||
}
|
||||
|
||||
void StoreUncompressedMetaBlockHeader(int length,
|
||||
int* storage_ix,
|
||||
uint8_t* storage) {
|
||||
// Write ISLAST bit. Uncompressed block cannot be the last one, so set to 0.
|
||||
WriteBits(1, 0, storage_ix, storage);
|
||||
int lenbits;
|
||||
int nlenbits;
|
||||
int nibblesbits;
|
||||
EncodeMlen(length, &lenbits, &nlenbits, &nibblesbits);
|
||||
WriteBits(2, nibblesbits, storage_ix, storage);
|
||||
WriteBits(nlenbits, lenbits, storage_ix, storage);
|
||||
// Write ISUNCOMPRESSED bit.
|
||||
WriteBits(1, 1, storage_ix, storage);
|
||||
}
|
||||
|
||||
void StoreHuffmanTreeOfHuffmanTreeToBitMask(
|
||||
const int num_codes,
|
||||
const uint8_t *code_length_bitdepth,
|
||||
int *storage_ix,
|
||||
uint8_t *storage) {
|
||||
static const uint8_t kStorageOrder[kCodeLengthCodes] = {
|
||||
1, 2, 3, 4, 0, 5, 17, 6, 16, 7, 8, 9, 10, 11, 12, 13, 14, 15
|
||||
};
|
||||
// The bit lengths of the Huffman code over the code length alphabet
|
||||
// are compressed with the following static Huffman code:
|
||||
// Symbol Code
|
||||
// ------ ----
|
||||
// 0 00
|
||||
// 1 1110
|
||||
// 2 110
|
||||
// 3 01
|
||||
// 4 10
|
||||
// 5 1111
|
||||
static const uint8_t kHuffmanBitLengthHuffmanCodeSymbols[6] = {
|
||||
0, 7, 3, 2, 1, 15
|
||||
};
|
||||
static const uint8_t kHuffmanBitLengthHuffmanCodeBitLengths[6] = {
|
||||
2, 4, 3, 2, 2, 4
|
||||
};
|
||||
|
||||
// Throw away trailing zeros:
|
||||
int codes_to_store = kCodeLengthCodes;
|
||||
if (num_codes > 1) {
|
||||
for (; codes_to_store > 0; --codes_to_store) {
|
||||
if (code_length_bitdepth[kStorageOrder[codes_to_store - 1]] != 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
int skip_some = 0; // skips none.
|
||||
if (code_length_bitdepth[kStorageOrder[0]] == 0 &&
|
||||
code_length_bitdepth[kStorageOrder[1]] == 0) {
|
||||
skip_some = 2; // skips two.
|
||||
if (code_length_bitdepth[kStorageOrder[2]] == 0) {
|
||||
skip_some = 3; // skips three.
|
||||
}
|
||||
}
|
||||
WriteBits(2, skip_some, storage_ix, storage);
|
||||
for (int i = skip_some; i < codes_to_store; ++i) {
|
||||
uint8_t l = code_length_bitdepth[kStorageOrder[i]];
|
||||
WriteBits(kHuffmanBitLengthHuffmanCodeBitLengths[l],
|
||||
kHuffmanBitLengthHuffmanCodeSymbols[l], storage_ix, storage);
|
||||
}
|
||||
}
|
||||
|
||||
void StoreHuffmanTreeToBitMask(
|
||||
const std::vector<uint8_t> &huffman_tree,
|
||||
const std::vector<uint8_t> &huffman_tree_extra_bits,
|
||||
const uint8_t *code_length_bitdepth,
|
||||
const std::vector<uint16_t> &code_length_bitdepth_symbols,
|
||||
int * __restrict storage_ix,
|
||||
uint8_t * __restrict storage) {
|
||||
for (int i = 0; i < huffman_tree.size(); ++i) {
|
||||
int ix = huffman_tree[i];
|
||||
WriteBits(code_length_bitdepth[ix], code_length_bitdepth_symbols[ix],
|
||||
storage_ix, storage);
|
||||
// Extra bits
|
||||
switch (ix) {
|
||||
case 16:
|
||||
WriteBits(2, huffman_tree_extra_bits[i], storage_ix, storage);
|
||||
break;
|
||||
case 17:
|
||||
WriteBits(3, huffman_tree_extra_bits[i], storage_ix, storage);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void StoreSimpleHuffmanTree(const uint8_t* depths,
|
||||
int symbols[4],
|
||||
int num_symbols,
|
||||
int max_bits,
|
||||
int *storage_ix, uint8_t *storage) {
|
||||
// value of 1 indicates a simple Huffman code
|
||||
WriteBits(2, 1, storage_ix, storage);
|
||||
WriteBits(2, num_symbols - 1, storage_ix, storage); // NSYM - 1
|
||||
|
||||
// Sort
|
||||
for (int i = 0; i < num_symbols; i++) {
|
||||
for (int j = i + 1; j < num_symbols; j++) {
|
||||
if (depths[symbols[j]] < depths[symbols[i]]) {
|
||||
std::swap(symbols[j], symbols[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (num_symbols == 2) {
|
||||
WriteBits(max_bits, symbols[0], storage_ix, storage);
|
||||
WriteBits(max_bits, symbols[1], storage_ix, storage);
|
||||
} else if (num_symbols == 3) {
|
||||
WriteBits(max_bits, symbols[0], storage_ix, storage);
|
||||
WriteBits(max_bits, symbols[1], storage_ix, storage);
|
||||
WriteBits(max_bits, symbols[2], storage_ix, storage);
|
||||
} else {
|
||||
WriteBits(max_bits, symbols[0], storage_ix, storage);
|
||||
WriteBits(max_bits, symbols[1], storage_ix, storage);
|
||||
WriteBits(max_bits, symbols[2], storage_ix, storage);
|
||||
WriteBits(max_bits, symbols[3], storage_ix, storage);
|
||||
// tree-select
|
||||
WriteBits(1, depths[symbols[0]] == 1 ? 1 : 0, storage_ix, storage);
|
||||
}
|
||||
}
|
||||
|
||||
// num = alphabet size
|
||||
// depths = symbol depths
|
||||
void StoreHuffmanTree(const uint8_t* depths, size_t num,
|
||||
int quality,
|
||||
int *storage_ix, uint8_t *storage) {
|
||||
// Write the Huffman tree into the brotli-representation.
|
||||
std::vector<uint8_t> huffman_tree;
|
||||
std::vector<uint8_t> huffman_tree_extra_bits;
|
||||
// TODO: Consider allocating these from stack.
|
||||
huffman_tree.reserve(256);
|
||||
huffman_tree_extra_bits.reserve(256);
|
||||
WriteHuffmanTree(depths, num, &huffman_tree, &huffman_tree_extra_bits);
|
||||
|
||||
// Calculate the statistics of the Huffman tree in brotli-representation.
|
||||
int huffman_tree_histogram[kCodeLengthCodes] = { 0 };
|
||||
for (int i = 0; i < huffman_tree.size(); ++i) {
|
||||
++huffman_tree_histogram[huffman_tree[i]];
|
||||
}
|
||||
|
||||
int num_codes = 0;
|
||||
int code = 0;
|
||||
for (int i = 0; i < kCodeLengthCodes; ++i) {
|
||||
if (huffman_tree_histogram[i]) {
|
||||
if (num_codes == 0) {
|
||||
code = i;
|
||||
num_codes = 1;
|
||||
} else if (num_codes == 1) {
|
||||
num_codes = 2;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate another Huffman tree to use for compressing both the
|
||||
// earlier Huffman tree with.
|
||||
// TODO: Consider allocating these from stack.
|
||||
uint8_t code_length_bitdepth[kCodeLengthCodes] = { 0 };
|
||||
std::vector<uint16_t> code_length_bitdepth_symbols(kCodeLengthCodes);
|
||||
CreateHuffmanTree(&huffman_tree_histogram[0], kCodeLengthCodes,
|
||||
5, quality, &code_length_bitdepth[0]);
|
||||
ConvertBitDepthsToSymbols(code_length_bitdepth, kCodeLengthCodes,
|
||||
code_length_bitdepth_symbols.data());
|
||||
|
||||
// Now, we have all the data, let's start storing it
|
||||
StoreHuffmanTreeOfHuffmanTreeToBitMask(num_codes, code_length_bitdepth,
|
||||
storage_ix, storage);
|
||||
|
||||
if (num_codes == 1) {
|
||||
code_length_bitdepth[code] = 0;
|
||||
}
|
||||
|
||||
// Store the real huffman tree now.
|
||||
StoreHuffmanTreeToBitMask(huffman_tree,
|
||||
huffman_tree_extra_bits,
|
||||
&code_length_bitdepth[0],
|
||||
code_length_bitdepth_symbols,
|
||||
storage_ix, storage);
|
||||
}
|
||||
|
||||
void BuildAndStoreHuffmanTree(const int *histogram,
|
||||
const int length,
|
||||
const int quality,
|
||||
uint8_t* depth,
|
||||
uint16_t* bits,
|
||||
int* storage_ix,
|
||||
uint8_t* storage) {
|
||||
int count = 0;
|
||||
int s4[4] = { 0 };
|
||||
for (size_t i = 0; i < length; i++) {
|
||||
if (histogram[i]) {
|
||||
if (count < 4) {
|
||||
s4[count] = i;
|
||||
} else if (quality < 3 && count > 4) {
|
||||
break;
|
||||
}
|
||||
count++;
|
||||
}
|
||||
}
|
||||
|
||||
int max_bits_counter = length - 1;
|
||||
int max_bits = 0;
|
||||
while (max_bits_counter) {
|
||||
max_bits_counter >>= 1;
|
||||
++max_bits;
|
||||
}
|
||||
|
||||
if (count <= 1) {
|
||||
WriteBits(4, 1, storage_ix, storage);
|
||||
WriteBits(max_bits, s4[0], storage_ix, storage);
|
||||
return;
|
||||
}
|
||||
|
||||
if (length >= 50 && count >= 16 && quality >= 3) {
|
||||
std::vector<int> counts(length);
|
||||
memcpy(&counts[0], histogram, sizeof(counts[0]) * length);
|
||||
OptimizeHuffmanCountsForRle(length, &counts[0]);
|
||||
CreateHuffmanTree(&counts[0], length, 15, quality, depth);
|
||||
} else {
|
||||
CreateHuffmanTree(histogram, length, 15, quality, depth);
|
||||
}
|
||||
ConvertBitDepthsToSymbols(depth, length, bits);
|
||||
|
||||
if (count <= 4) {
|
||||
StoreSimpleHuffmanTree(depth, s4, count, max_bits, storage_ix, storage);
|
||||
} else {
|
||||
StoreHuffmanTree(depth, length, quality, storage_ix, storage);
|
||||
}
|
||||
}
|
||||
|
||||
void StoreTrivialContextMap(int num_types,
|
||||
int context_bits,
|
||||
int* storage_ix,
|
||||
uint8_t* storage) {
|
||||
StoreVarLenUint8(num_types - 1, storage_ix, storage);
|
||||
if (num_types > 1) {
|
||||
int repeat_code = context_bits - 1;
|
||||
int repeat_bits = (1 << repeat_code) - 1;
|
||||
int alphabet_size = num_types + repeat_code;
|
||||
std::vector<int> histogram(alphabet_size);
|
||||
std::vector<uint8_t> depths(alphabet_size);
|
||||
std::vector<uint16_t> bits(alphabet_size);
|
||||
// Write RLEMAX.
|
||||
WriteBits(1, 1, storage_ix, storage);
|
||||
WriteBits(4, repeat_code - 1, storage_ix, storage);
|
||||
histogram[repeat_code] = num_types;
|
||||
histogram[0] = 1;
|
||||
for (int i = context_bits; i < alphabet_size; ++i) {
|
||||
histogram[i] = 1;
|
||||
}
|
||||
BuildAndStoreHuffmanTree(&histogram[0], alphabet_size, 1,
|
||||
&depths[0], &bits[0],
|
||||
storage_ix, storage);
|
||||
for (int i = 0; i < num_types; ++i) {
|
||||
int code = (i == 0 ? 0 : i + context_bits - 1);
|
||||
WriteBits(depths[code], bits[code], storage_ix, storage);
|
||||
WriteBits(depths[repeat_code], bits[repeat_code], storage_ix, storage);
|
||||
WriteBits(repeat_code, repeat_bits, storage_ix, storage);
|
||||
}
|
||||
// Write IMTF (inverse-move-to-front) bit.
|
||||
WriteBits(1, 1, storage_ix, storage);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace brotli
|
||||
67
enc/brotli_bit_stream.h
Normal file
67
enc/brotli_bit_stream.h
Normal file
@ -0,0 +1,67 @@
|
||||
// Copyright 2014 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Functions to convert brotli-related data structures into the
|
||||
// brotli bit stream. The functions here operate under
|
||||
// assumption that there is enough space in the storage, i.e., there are
|
||||
// no out-of-range checks anywhere.
|
||||
//
|
||||
// These functions do bit addressing into a byte array. The byte array
|
||||
// is called "storage" and the index to the bit is called storage_ix
|
||||
// in function arguments.
|
||||
|
||||
#ifndef BROTLI_ENC_BROTLI_BIT_STREAM_H_
|
||||
#define BROTLI_ENC_BROTLI_BIT_STREAM_H_
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
namespace brotli {
|
||||
|
||||
// All Store functions here will use a storage_ix, which is always the bit
|
||||
// position for the current storage.
|
||||
|
||||
// Stores a number between 0 and 255.
|
||||
void StoreVarLenUint8(int n, int* storage_ix, uint8_t* storage);
|
||||
|
||||
// Stores the compressed meta-block header.
|
||||
void StoreCompressedMetaBlockHeader(bool final_block,
|
||||
int length,
|
||||
int* storage_ix,
|
||||
uint8_t* storage);
|
||||
|
||||
// Stores the uncompressed meta-block header.
|
||||
void StoreUncompressedMetaBlockHeader(int length,
|
||||
int* storage_ix,
|
||||
uint8_t* storage);
|
||||
|
||||
// Stores a context map where the histogram type is always the block type.
|
||||
void StoreTrivialContextMap(int num_types,
|
||||
int context_bits,
|
||||
int* storage_ix,
|
||||
uint8_t* storage);
|
||||
|
||||
// Builds a Huffman tree from histogram[0:length] into depth[0:length] and
|
||||
// bits[0:length] and stores the encoded tree to the bit stream.
|
||||
void BuildAndStoreHuffmanTree(const int *histogram,
|
||||
const int length,
|
||||
const int quality,
|
||||
uint8_t* depth,
|
||||
uint16_t* bits,
|
||||
int* storage_ix,
|
||||
uint8_t* storage);
|
||||
|
||||
} // namespace brotli
|
||||
|
||||
#endif // BROTLI_ENC_BROTLI_BIT_STREAM_H_
|
||||
189
enc/encode.cc
189
enc/encode.cc
@ -22,6 +22,7 @@
|
||||
#include "./backward_references.h"
|
||||
#include "./bit_cost.h"
|
||||
#include "./block_splitter.h"
|
||||
#include "./brotli_bit_stream.h"
|
||||
#include "./cluster.h"
|
||||
#include "./context.h"
|
||||
#include "./transform.h"
|
||||
@ -65,19 +66,6 @@ double TotalBitCost(const std::vector<Histogram<kSize> >& histograms) {
|
||||
return retval;
|
||||
}
|
||||
|
||||
void EncodeVarLenUint8(int n, int* storage_ix, uint8_t* storage) {
|
||||
if (n == 0) {
|
||||
WriteBits(1, 0, storage_ix, storage);
|
||||
} else {
|
||||
WriteBits(1, 1, storage_ix, storage);
|
||||
int nbits = Log2Floor(n);
|
||||
WriteBits(3, nbits, storage_ix, storage);
|
||||
if (nbits > 0) {
|
||||
WriteBits(nbits, n - (1 << nbits), storage_ix, storage);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int ParseAsUTF8(int* symbol, const uint8_t* input, int size) {
|
||||
// ASCII
|
||||
if ((input[0] & 0x80) == 0) {
|
||||
@ -168,134 +156,6 @@ void EncodeMetaBlockLength(size_t meta_block_size,
|
||||
}
|
||||
}
|
||||
|
||||
void StoreHuffmanTreeOfHuffmanTreeToBitMask(
|
||||
const uint8_t* code_length_bitdepth,
|
||||
int* storage_ix, uint8_t* storage) {
|
||||
static const uint8_t kStorageOrder[kCodeLengthCodes] = {
|
||||
1, 2, 3, 4, 0, 5, 17, 6, 16, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
};
|
||||
// Throw away trailing zeros:
|
||||
int codes_to_store = kCodeLengthCodes;
|
||||
for (; codes_to_store > 0; --codes_to_store) {
|
||||
if (code_length_bitdepth[kStorageOrder[codes_to_store - 1]] != 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
int num_codes = 0;
|
||||
for (int i = 0; i < codes_to_store; ++i) {
|
||||
if (code_length_bitdepth[kStorageOrder[i]] != 0) {
|
||||
++num_codes;
|
||||
}
|
||||
}
|
||||
if (num_codes == 1) {
|
||||
codes_to_store = kCodeLengthCodes;
|
||||
}
|
||||
int skip_some = 0; // skips none.
|
||||
if (code_length_bitdepth[kStorageOrder[0]] == 0 &&
|
||||
code_length_bitdepth[kStorageOrder[1]] == 0) {
|
||||
skip_some = 2; // skips two.
|
||||
if (code_length_bitdepth[kStorageOrder[2]] == 0) {
|
||||
skip_some = 3; // skips three.
|
||||
}
|
||||
}
|
||||
WriteBits(2, skip_some, storage_ix, storage);
|
||||
for (int i = skip_some; i < codes_to_store; ++i) {
|
||||
uint8_t len[] = { 2, 4, 3, 2, 2, 4 };
|
||||
uint8_t bits[] = { 0, 7, 3, 2, 1, 15 };
|
||||
int v = code_length_bitdepth[kStorageOrder[i]];
|
||||
WriteBits(len[v], bits[v], storage_ix, storage);
|
||||
}
|
||||
}
|
||||
|
||||
void StoreHuffmanTreeToBitMask(
|
||||
const uint8_t* huffman_tree,
|
||||
const uint8_t* huffman_tree_extra_bits,
|
||||
const int huffman_tree_size,
|
||||
const EntropyCode<kCodeLengthCodes>& entropy,
|
||||
int* storage_ix, uint8_t* storage) {
|
||||
for (int i = 0; i < huffman_tree_size; ++i) {
|
||||
const int ix = huffman_tree[i];
|
||||
const int extra_bits = huffman_tree_extra_bits[i];
|
||||
if (entropy.count_ > 1) {
|
||||
WriteBits(entropy.depth_[ix], entropy.bits_[ix], storage_ix, storage);
|
||||
}
|
||||
switch (ix) {
|
||||
case 16:
|
||||
WriteBits(2, extra_bits, storage_ix, storage);
|
||||
break;
|
||||
case 17:
|
||||
WriteBits(3, extra_bits, storage_ix, storage);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<int kSize>
|
||||
void StoreHuffmanCodeSimple(
|
||||
const EntropyCode<kSize>& code, int alphabet_size,
|
||||
int max_bits, int* storage_ix, uint8_t* storage) {
|
||||
const uint8_t *depth = &code.depth_[0];
|
||||
int symbols[4];
|
||||
// Quadratic sort.
|
||||
int k, j;
|
||||
for (k = 0; k < code.count_; ++k) {
|
||||
symbols[k] = code.symbols_[k];
|
||||
}
|
||||
for (k = 0; k < code.count_; ++k) {
|
||||
for (j = k + 1; j < code.count_; ++j) {
|
||||
if (depth[symbols[j]] < depth[symbols[k]]) {
|
||||
int t = symbols[k];
|
||||
symbols[k] = symbols[j];
|
||||
symbols[j] = t;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Small tree marker to encode 1-4 symbols.
|
||||
WriteBits(2, 1, storage_ix, storage);
|
||||
WriteBits(2, code.count_ - 1, storage_ix, storage);
|
||||
for (int i = 0; i < code.count_; ++i) {
|
||||
WriteBits(max_bits, symbols[i], storage_ix, storage);
|
||||
}
|
||||
if (code.count_ == 4) {
|
||||
if (depth[symbols[0]] == 2 &&
|
||||
depth[symbols[1]] == 2 &&
|
||||
depth[symbols[2]] == 2 &&
|
||||
depth[symbols[3]] == 2) {
|
||||
WriteBits(1, 0, storage_ix, storage);
|
||||
} else {
|
||||
WriteBits(1, 1, storage_ix, storage);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<int kSize>
|
||||
void StoreHuffmanCodeComplex(
|
||||
const EntropyCode<kSize>& code, int alphabet_size,
|
||||
int* storage_ix, uint8_t* storage) {
|
||||
const uint8_t *depth = &code.depth_[0];
|
||||
uint8_t huffman_tree[kSize];
|
||||
uint8_t huffman_tree_extra_bits[kSize];
|
||||
int huffman_tree_size = 0;
|
||||
WriteHuffmanTree(depth,
|
||||
alphabet_size,
|
||||
&huffman_tree[0],
|
||||
&huffman_tree_extra_bits[0],
|
||||
&huffman_tree_size);
|
||||
Histogram<kCodeLengthCodes> huffman_tree_histogram;
|
||||
memset(huffman_tree_histogram.data_, 0, sizeof(huffman_tree_histogram.data_));
|
||||
for (int i = 0; i < huffman_tree_size; ++i) {
|
||||
huffman_tree_histogram.Add(huffman_tree[i]);
|
||||
}
|
||||
EntropyCode<kCodeLengthCodes> huffman_tree_entropy;
|
||||
BuildEntropyCode(huffman_tree_histogram, 5, kCodeLengthCodes,
|
||||
&huffman_tree_entropy);
|
||||
StoreHuffmanTreeOfHuffmanTreeToBitMask(
|
||||
&huffman_tree_entropy.depth_[0], storage_ix, storage);
|
||||
StoreHuffmanTreeToBitMask(&huffman_tree[0], &huffman_tree_extra_bits[0],
|
||||
huffman_tree_size, huffman_tree_entropy,
|
||||
storage_ix, storage);
|
||||
}
|
||||
|
||||
template<int kSize>
|
||||
void BuildAndStoreEntropyCode(const Histogram<kSize>& histogram,
|
||||
const int tree_limit,
|
||||
@ -304,45 +164,8 @@ void BuildAndStoreEntropyCode(const Histogram<kSize>& histogram,
|
||||
int* storage_ix, uint8_t* storage) {
|
||||
memset(code->depth_, 0, sizeof(code->depth_));
|
||||
memset(code->bits_, 0, sizeof(code->bits_));
|
||||
memset(code->symbols_, 0, sizeof(code->symbols_));
|
||||
code->count_ = 0;
|
||||
|
||||
int max_bits_counter = alphabet_size - 1;
|
||||
int max_bits = 0;
|
||||
while (max_bits_counter) {
|
||||
max_bits_counter >>= 1;
|
||||
++max_bits;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < alphabet_size; i++) {
|
||||
if (histogram.data_[i] > 0) {
|
||||
if (code->count_ < 4) code->symbols_[code->count_] = i;
|
||||
++code->count_;
|
||||
}
|
||||
}
|
||||
|
||||
if (code->count_ <= 1) {
|
||||
WriteBits(2, 1, storage_ix, storage);
|
||||
WriteBits(2, 0, storage_ix, storage);
|
||||
WriteBits(max_bits, code->symbols_[0], storage_ix, storage);
|
||||
return;
|
||||
}
|
||||
|
||||
if (alphabet_size >= 50 && code->count_ >= 16) {
|
||||
std::vector<int> counts(alphabet_size);
|
||||
memcpy(&counts[0], histogram.data_, sizeof(counts[0]) * alphabet_size);
|
||||
OptimizeHuffmanCountsForRle(alphabet_size, &counts[0]);
|
||||
CreateHuffmanTree(&counts[0], alphabet_size, tree_limit, code->depth_);
|
||||
} else {
|
||||
CreateHuffmanTree(histogram.data_, alphabet_size, tree_limit, code->depth_);
|
||||
}
|
||||
ConvertBitDepthsToSymbols(code->depth_, alphabet_size, code->bits_);
|
||||
|
||||
if (code->count_ <= 4) {
|
||||
StoreHuffmanCodeSimple(*code, alphabet_size, max_bits, storage_ix, storage);
|
||||
} else {
|
||||
StoreHuffmanCodeComplex(*code, alphabet_size, storage_ix, storage);
|
||||
}
|
||||
BuildAndStoreHuffmanTree(histogram.data_, alphabet_size, 9,
|
||||
code->depth_, code->bits_, storage_ix, storage);
|
||||
}
|
||||
|
||||
template<int kSize>
|
||||
@ -575,7 +398,7 @@ int BestMaxZeroRunLengthPrefix(const std::vector<int>& v) {
|
||||
void EncodeContextMap(const std::vector<int>& context_map,
|
||||
int num_clusters,
|
||||
int* storage_ix, uint8_t* storage) {
|
||||
EncodeVarLenUint8(num_clusters - 1, storage_ix, storage);
|
||||
StoreVarLenUint8(num_clusters - 1, storage_ix, storage);
|
||||
|
||||
if (num_clusters == 1) {
|
||||
return;
|
||||
@ -656,7 +479,7 @@ void ComputeBlockTypeShortCodes(BlockSplit* split) {
|
||||
void BuildAndEncodeBlockSplitCode(const BlockSplit& split,
|
||||
BlockSplitCode* code,
|
||||
int* storage_ix, uint8_t* storage) {
|
||||
EncodeVarLenUint8(split.num_types_ - 1, storage_ix, storage);
|
||||
StoreVarLenUint8(split.num_types_ - 1, storage_ix, storage);
|
||||
|
||||
if (split.num_types_ == 1) {
|
||||
return;
|
||||
@ -864,7 +687,6 @@ void StoreMetaBlock(const MetaBlock& mb,
|
||||
int context = (distance_it.type_ << 2) +
|
||||
((cmd.copy_length_code_ > 4) ? 3 : cmd.copy_length_code_ - 2);
|
||||
int histogram_index = mb.distance_context_map[context];
|
||||
size_t max_distance = std::min(*pos, (size_t)kMaxBackwardDistance);
|
||||
EncodeCopyDistance(cmd, distance_codes[histogram_index],
|
||||
storage_ix, storage);
|
||||
}
|
||||
@ -1015,7 +837,6 @@ void BrotliCompressor::FinishStream(
|
||||
WriteMetaBlock(0, NULL, true, encoded_size, encoded_buffer);
|
||||
}
|
||||
|
||||
|
||||
int BrotliCompressBuffer(BrotliParams params,
|
||||
size_t input_size,
|
||||
const uint8_t* input_buffer,
|
||||
|
||||
@ -59,7 +59,6 @@ class BrotliCompressor {
|
||||
// sets *encoded_size to the number of bytes written.
|
||||
void FinishStream(size_t* encoded_size, uint8_t* encoded_buffer);
|
||||
|
||||
|
||||
private:
|
||||
// Initializes the hasher with the hashes of dictionary words.
|
||||
void StoreDictionaryWordHashes();
|
||||
@ -87,7 +86,6 @@ int BrotliCompressBuffer(BrotliParams params,
|
||||
size_t* encoded_size,
|
||||
uint8_t* encoded_buffer);
|
||||
|
||||
|
||||
} // namespace brotli
|
||||
|
||||
#endif // BROTLI_ENC_ENCODE_H_
|
||||
|
||||
@ -42,7 +42,7 @@ struct HuffmanTree {
|
||||
|
||||
HuffmanTree::HuffmanTree() {}
|
||||
|
||||
// Sort the root nodes, least popular first.
|
||||
// Sort the root nodes, least popular first, break ties by value.
|
||||
bool SortHuffmanTree(const HuffmanTree &v0, const HuffmanTree &v1) {
|
||||
if (v0.total_count_ == v1.total_count_) {
|
||||
return v0.index_right_or_value_ > v1.index_right_or_value_;
|
||||
@ -50,6 +50,11 @@ bool SortHuffmanTree(const HuffmanTree &v0, const HuffmanTree &v1) {
|
||||
return v0.total_count_ < v1.total_count_;
|
||||
}
|
||||
|
||||
// Sort the root nodes, least popular first.
|
||||
bool SortHuffmanTreeFast(const HuffmanTree &v0, const HuffmanTree &v1) {
|
||||
return v0.total_count_ < v1.total_count_;
|
||||
}
|
||||
|
||||
void SetDepth(const HuffmanTree &p,
|
||||
HuffmanTree *pool,
|
||||
uint8_t *depth,
|
||||
@ -83,6 +88,7 @@ void SetDepth(const HuffmanTree &p,
|
||||
void CreateHuffmanTree(const int *data,
|
||||
const int length,
|
||||
const int tree_limit,
|
||||
const int quality,
|
||||
uint8_t *depth) {
|
||||
// For block sizes below 64 kB, we never need to do a second iteration
|
||||
// of this loop. Probably all of our block sizes will be smaller than
|
||||
@ -105,8 +111,11 @@ void CreateHuffmanTree(const int *data,
|
||||
break;
|
||||
}
|
||||
|
||||
std::sort(tree.begin(), tree.end(), SortHuffmanTree);
|
||||
|
||||
if (quality > 1) {
|
||||
std::sort(tree.begin(), tree.end(), SortHuffmanTree);
|
||||
} else {
|
||||
std::sort(tree.begin(), tree.end(), SortHuffmanTreeFast);
|
||||
}
|
||||
// The nodes are:
|
||||
// [0, n): the sorted leaf nodes that we start with.
|
||||
// [n]: we add a sentinel here.
|
||||
@ -158,12 +167,12 @@ void CreateHuffmanTree(const int *data,
|
||||
}
|
||||
}
|
||||
|
||||
void Reverse(uint8_t* v, int start, int end) {
|
||||
void Reverse(std::vector<uint8_t>* v, int start, int end) {
|
||||
--end;
|
||||
while (start < end) {
|
||||
int tmp = v[start];
|
||||
v[start] = v[end];
|
||||
v[end] = tmp;
|
||||
int tmp = (*v)[start];
|
||||
(*v)[start] = (*v)[end];
|
||||
(*v)[end] = tmp;
|
||||
++start;
|
||||
--end;
|
||||
}
|
||||
@ -173,75 +182,65 @@ void WriteHuffmanTreeRepetitions(
|
||||
const int previous_value,
|
||||
const int value,
|
||||
int repetitions,
|
||||
uint8_t* tree,
|
||||
uint8_t* extra_bits,
|
||||
int* tree_size) {
|
||||
std::vector<uint8_t> *tree,
|
||||
std::vector<uint8_t> *extra_bits_data) {
|
||||
if (previous_value != value) {
|
||||
tree[*tree_size] = value;
|
||||
extra_bits[*tree_size] = 0;
|
||||
++(*tree_size);
|
||||
tree->push_back(value);
|
||||
extra_bits_data->push_back(0);
|
||||
--repetitions;
|
||||
}
|
||||
if (repetitions == 7) {
|
||||
tree[*tree_size] = value;
|
||||
extra_bits[*tree_size] = 0;
|
||||
++(*tree_size);
|
||||
tree->push_back(value);
|
||||
extra_bits_data->push_back(0);
|
||||
--repetitions;
|
||||
}
|
||||
if (repetitions < 3) {
|
||||
for (int i = 0; i < repetitions; ++i) {
|
||||
tree[*tree_size] = value;
|
||||
extra_bits[*tree_size] = 0;
|
||||
++(*tree_size);
|
||||
tree->push_back(value);
|
||||
extra_bits_data->push_back(0);
|
||||
}
|
||||
} else {
|
||||
repetitions -= 3;
|
||||
int start = *tree_size;
|
||||
int start = tree->size();
|
||||
while (repetitions >= 0) {
|
||||
tree[*tree_size] = 16;
|
||||
extra_bits[*tree_size] = repetitions & 0x3;
|
||||
++(*tree_size);
|
||||
tree->push_back(16);
|
||||
extra_bits_data->push_back(repetitions & 0x3);
|
||||
repetitions >>= 2;
|
||||
--repetitions;
|
||||
}
|
||||
Reverse(tree, start, *tree_size);
|
||||
Reverse(extra_bits, start, *tree_size);
|
||||
Reverse(tree, start, tree->size());
|
||||
Reverse(extra_bits_data, start, tree->size());
|
||||
}
|
||||
}
|
||||
|
||||
void WriteHuffmanTreeRepetitionsZeros(
|
||||
int repetitions,
|
||||
uint8_t* tree,
|
||||
uint8_t* extra_bits,
|
||||
int* tree_size) {
|
||||
std::vector<uint8_t> *tree,
|
||||
std::vector<uint8_t> *extra_bits_data) {
|
||||
if (repetitions == 11) {
|
||||
tree[*tree_size] = 0;
|
||||
extra_bits[*tree_size] = 0;
|
||||
++(*tree_size);
|
||||
tree->push_back(0);
|
||||
extra_bits_data->push_back(0);
|
||||
--repetitions;
|
||||
}
|
||||
if (repetitions < 3) {
|
||||
for (int i = 0; i < repetitions; ++i) {
|
||||
tree[*tree_size] = 0;
|
||||
extra_bits[*tree_size] = 0;
|
||||
++(*tree_size);
|
||||
tree->push_back(0);
|
||||
extra_bits_data->push_back(0);
|
||||
}
|
||||
} else {
|
||||
repetitions -= 3;
|
||||
int start = *tree_size;
|
||||
int start = tree->size();
|
||||
while (repetitions >= 0) {
|
||||
tree[*tree_size] = 17;
|
||||
extra_bits[*tree_size] = repetitions & 0x7;
|
||||
++(*tree_size);
|
||||
tree->push_back(17);
|
||||
extra_bits_data->push_back(repetitions & 0x7);
|
||||
repetitions >>= 3;
|
||||
--repetitions;
|
||||
}
|
||||
Reverse(tree, start, *tree_size);
|
||||
Reverse(extra_bits, start, *tree_size);
|
||||
Reverse(tree, start, tree->size());
|
||||
Reverse(extra_bits_data, start, tree->size());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
int OptimizeHuffmanCountsForRle(int length, int* counts) {
|
||||
int stride;
|
||||
int limit;
|
||||
@ -371,7 +370,6 @@ int OptimizeHuffmanCountsForRle(int length, int* counts) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
static void DecideOverRleUse(const uint8_t* depth, const int length,
|
||||
bool *use_rle_for_non_zero,
|
||||
bool *use_rle_for_zero) {
|
||||
@ -379,20 +377,10 @@ static void DecideOverRleUse(const uint8_t* depth, const int length,
|
||||
int total_reps_non_zero = 0;
|
||||
int count_reps_zero = 0;
|
||||
int count_reps_non_zero = 0;
|
||||
int new_length = length;
|
||||
for (int i = 0; i < length; ++i) {
|
||||
if (depth[length - i - 1] == 0) {
|
||||
--new_length;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
for (uint32_t i = 0; i < new_length;) {
|
||||
for (uint32_t i = 0; i < length;) {
|
||||
const int value = depth[i];
|
||||
int reps = 1;
|
||||
// Find rle coding for longer codes.
|
||||
// Shorter codes seem not to benefit from rle.
|
||||
for (uint32_t k = i + 1; k < new_length && depth[k] == value; ++k) {
|
||||
for (uint32_t k = i + 1; k < length && depth[k] == value; ++k) {
|
||||
++reps;
|
||||
}
|
||||
if (reps >= 3 && value == 0) {
|
||||
@ -411,48 +399,51 @@ static void DecideOverRleUse(const uint8_t* depth, const int length,
|
||||
*use_rle_for_zero = total_reps_zero > 2;
|
||||
}
|
||||
|
||||
|
||||
void WriteHuffmanTree(const uint8_t* depth, const int length,
|
||||
uint8_t* tree,
|
||||
uint8_t* extra_bits_data,
|
||||
int* huffman_tree_size) {
|
||||
void WriteHuffmanTree(const uint8_t* depth,
|
||||
uint32_t length,
|
||||
std::vector<uint8_t> *tree,
|
||||
std::vector<uint8_t> *extra_bits_data) {
|
||||
int previous_value = 8;
|
||||
|
||||
// Throw away trailing zeros.
|
||||
int new_length = length;
|
||||
for (int i = 0; i < length; ++i) {
|
||||
if (depth[length - i - 1] == 0) {
|
||||
--new_length;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// First gather statistics on if it is a good idea to do rle.
|
||||
bool use_rle_for_non_zero;
|
||||
bool use_rle_for_zero;
|
||||
DecideOverRleUse(depth, length, &use_rle_for_non_zero, &use_rle_for_zero);
|
||||
bool use_rle_for_non_zero = false;
|
||||
bool use_rle_for_zero = false;
|
||||
if (length > 50) {
|
||||
// Find rle coding for longer codes.
|
||||
// Shorter codes seem not to benefit from rle.
|
||||
DecideOverRleUse(depth, new_length,
|
||||
&use_rle_for_non_zero, &use_rle_for_zero);
|
||||
}
|
||||
|
||||
// Actual rle coding.
|
||||
for (uint32_t i = 0; i < length;) {
|
||||
for (uint32_t i = 0; i < new_length;) {
|
||||
const int value = depth[i];
|
||||
int reps = 1;
|
||||
if (length > 50) {
|
||||
// Find rle coding for longer codes.
|
||||
// Shorter codes seem not to benefit from rle.
|
||||
if ((value != 0 && use_rle_for_non_zero) ||
|
||||
(value == 0 && use_rle_for_zero)) {
|
||||
for (uint32_t k = i + 1; k < length && depth[k] == value; ++k) {
|
||||
++reps;
|
||||
}
|
||||
if ((value != 0 && use_rle_for_non_zero) ||
|
||||
(value == 0 && use_rle_for_zero)) {
|
||||
for (uint32_t k = i + 1; k < new_length && depth[k] == value; ++k) {
|
||||
++reps;
|
||||
}
|
||||
}
|
||||
if (value == 0) {
|
||||
WriteHuffmanTreeRepetitionsZeros(reps, tree, extra_bits_data,
|
||||
huffman_tree_size);
|
||||
WriteHuffmanTreeRepetitionsZeros(reps, tree, extra_bits_data);
|
||||
} else {
|
||||
WriteHuffmanTreeRepetitions(previous_value, value, reps, tree,
|
||||
extra_bits_data, huffman_tree_size);
|
||||
WriteHuffmanTreeRepetitions(previous_value,
|
||||
value, reps, tree, extra_bits_data);
|
||||
previous_value = value;
|
||||
}
|
||||
i += reps;
|
||||
}
|
||||
// Throw away trailing zeros.
|
||||
for (; *huffman_tree_size > 0; --(*huffman_tree_size)) {
|
||||
if (tree[*huffman_tree_size - 1] > 0 && tree[*huffman_tree_size - 1] < 17) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
@ -19,6 +19,7 @@
|
||||
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
#include <vector>
|
||||
#include "./histogram.h"
|
||||
#include "./prefix.h"
|
||||
|
||||
@ -36,6 +37,7 @@ namespace brotli {
|
||||
void CreateHuffmanTree(const int *data,
|
||||
const int length,
|
||||
const int tree_limit,
|
||||
const int quality,
|
||||
uint8_t *depth);
|
||||
|
||||
// Change the population counts in a way that the consequent
|
||||
@ -46,14 +48,13 @@ void CreateHuffmanTree(const int *data,
|
||||
// counts contains the population counts.
|
||||
int OptimizeHuffmanCountsForRle(int length, int* counts);
|
||||
|
||||
|
||||
// Write a huffman tree from bit depths into the bitstream representation
|
||||
// of a Huffman tree. The generated Huffman tree is to be compressed once
|
||||
// more using a Huffman tree
|
||||
void WriteHuffmanTree(const uint8_t* depth, const int length,
|
||||
uint8_t* tree,
|
||||
uint8_t* extra_bits_data,
|
||||
int* huffman_tree_size);
|
||||
void WriteHuffmanTree(const uint8_t* depth,
|
||||
uint32_t num,
|
||||
std::vector<uint8_t> *tree,
|
||||
std::vector<uint8_t> *extra_bits_data);
|
||||
|
||||
// Get the actual bit values for a tree of bit depths.
|
||||
void ConvertBitDepthsToSymbols(const uint8_t *depth, int len, uint16_t *bits);
|
||||
@ -70,34 +71,6 @@ struct EntropyCode {
|
||||
int symbols_[4];
|
||||
};
|
||||
|
||||
template<int kSize>
|
||||
void BuildEntropyCode(const Histogram<kSize>& histogram,
|
||||
const int tree_limit,
|
||||
const int alphabet_size,
|
||||
EntropyCode<kSize>* code) {
|
||||
memset(code->depth_, 0, sizeof(code->depth_));
|
||||
memset(code->bits_, 0, sizeof(code->bits_));
|
||||
memset(code->symbols_, 0, sizeof(code->symbols_));
|
||||
code->count_ = 0;
|
||||
if (histogram.total_count_ == 0) return;
|
||||
for (int i = 0; i < kSize; ++i) {
|
||||
if (histogram.data_[i] > 0) {
|
||||
if (code->count_ < 4) code->symbols_[code->count_] = i;
|
||||
++code->count_;
|
||||
}
|
||||
}
|
||||
if (alphabet_size >= 50 && code->count_ >= 16) {
|
||||
int counts[kSize];
|
||||
memcpy(counts, &histogram.data_[0], sizeof(counts[0]) * kSize);
|
||||
OptimizeHuffmanCountsForRle(alphabet_size, counts);
|
||||
CreateHuffmanTree(counts, alphabet_size, tree_limit, &code->depth_[0]);
|
||||
} else {
|
||||
CreateHuffmanTree(&histogram.data_[0], alphabet_size, tree_limit,
|
||||
&code->depth_[0]);
|
||||
}
|
||||
ConvertBitDepthsToSymbols(&code->depth_[0], alphabet_size, &code->bits_[0]);
|
||||
}
|
||||
|
||||
static const int kCodeLengthCodes = 18;
|
||||
|
||||
// Literal entropy code.
|
||||
|
||||
Loading…
Reference in New Issue
Block a user