diff --git a/enc/block_splitter.cc b/enc/block_splitter.cc index a24d0fb..831c3e2 100644 --- a/enc/block_splitter.cc +++ b/enc/block_splitter.cc @@ -301,7 +301,7 @@ void SplitByteVector(const std::vector& data, const double block_switch_cost, BlockSplit* split) { if (data.empty()) { - split->num_types_ = 0; + split->num_types_ = 1; return; } else if (data.size() < kMinLengthForBlockSplitting) { split->num_types_ = 1; diff --git a/enc/block_splitter.h b/enc/block_splitter.h index 2a491e3..6e4682e 100644 --- a/enc/block_splitter.h +++ b/enc/block_splitter.h @@ -29,8 +29,7 @@ namespace brotli { struct BlockSplit { int num_types_; - std::vector types_; - std::vector type_codes_; + std::vector types_; std::vector lengths_; }; diff --git a/enc/brotli_bit_stream.cc b/enc/brotli_bit_stream.cc index 0024986..d47e02f 100644 --- a/enc/brotli_bit_stream.cc +++ b/enc/brotli_bit_stream.cc @@ -22,6 +22,7 @@ #include "./entropy_encode.h" #include "./fast_log.h" +#include "./prefix.h" #include "./write_bits.h" namespace brotli { @@ -305,6 +306,69 @@ void BuildAndStoreHuffmanTree(const int *histogram, } } +void StoreBlockSwitch(const BlockSplitCode& code, + const int block_ix, + int* storage_ix, + uint8_t* storage) { + if (block_ix > 0) { + int typecode = code.type_code[block_ix]; + WriteBits(code.type_depths[typecode], code.type_bits[typecode], + storage_ix, storage); + } + int lencode = code.length_prefix[block_ix]; + WriteBits(code.length_depths[lencode], code.length_bits[lencode], + storage_ix, storage); + WriteBits(code.length_nextra[block_ix], code.length_extra[block_ix], + storage_ix, storage); +} + +void BuildAndStoreBlockSplitCode(const std::vector& types, + const std::vector& lengths, + const int num_types, + const int quality, + BlockSplitCode* code, + int* storage_ix, + uint8_t* storage) { + const int num_blocks = types.size(); + std::vector type_histo(num_types + 2); + std::vector length_histo(26); + int last_type = 1; + int second_last_type = 0; + code->type_code.resize(num_blocks); + code->length_prefix.resize(num_blocks); + code->length_nextra.resize(num_blocks); + code->length_extra.resize(num_blocks); + code->type_depths.resize(num_types + 2); + code->type_bits.resize(num_types + 2); + code->length_depths.resize(26); + code->length_bits.resize(26); + for (int i = 0; i < num_blocks; ++i) { + int type = types[i]; + int type_code = (type == last_type + 1 ? 1 : + type == second_last_type ? 0 : + type + 2); + second_last_type = last_type; + last_type = type; + code->type_code[i] = type_code; + if (i > 0) ++type_histo[type_code]; + GetBlockLengthPrefixCode(lengths[i], + &code->length_prefix[i], + &code->length_nextra[i], + &code->length_extra[i]); + ++length_histo[code->length_prefix[i]]; + } + StoreVarLenUint8(num_types - 1, storage_ix, storage); + if (num_types > 1) { + BuildAndStoreHuffmanTree(&type_histo[0], num_types + 2, quality, + &code->type_depths[0], &code->type_bits[0], + storage_ix, storage); + BuildAndStoreHuffmanTree(&length_histo[0], 26, quality, + &code->length_depths[0], &code->length_bits[0], + storage_ix, storage); + StoreBlockSwitch(*code, 0, storage_ix, storage); + } +} + void StoreTrivialContextMap(int num_types, int context_bits, int* storage_ix, diff --git a/enc/brotli_bit_stream.h b/enc/brotli_bit_stream.h index 5e5ed4d..085f100 100644 --- a/enc/brotli_bit_stream.h +++ b/enc/brotli_bit_stream.h @@ -26,6 +26,7 @@ #include #include +#include namespace brotli { @@ -62,6 +63,35 @@ void BuildAndStoreHuffmanTree(const int *histogram, int* storage_ix, uint8_t* storage); +// Data structure that stores everything that is needed to encode each block +// switch command. +struct BlockSplitCode { + std::vector type_code; + std::vector length_prefix; + std::vector length_nextra; + std::vector length_extra; + std::vector type_depths; + std::vector type_bits; + std::vector length_depths; + std::vector length_bits; +}; + +// Builds a BlockSplitCode data structure from the block split given by the +// vector of block types and block lengths and stores it to the bit stream. +void BuildAndStoreBlockSplitCode(const std::vector& types, + const std::vector& lengths, + const int num_types, + const int quality, + BlockSplitCode* code, + int* storage_ix, + uint8_t* storage); + +// Stores the block switch command with index block_ix to the bit stream. +void StoreBlockSwitch(const BlockSplitCode& code, + const int block_ix, + int* storage_ix, + uint8_t* storage); + } // namespace brotli #endif // BROTLI_ENC_BROTLI_BIT_STREAM_H_ diff --git a/enc/encode.cc b/enc/encode.cc index 8870e3e..24e280e 100644 --- a/enc/encode.cc +++ b/enc/encode.cc @@ -435,74 +435,6 @@ void EncodeContextMap(const std::vector& context_map, WriteBits(1, 1, storage_ix, storage); // use move-to-front } -struct BlockSplitCode { - EntropyCodeBlockType block_type_code; - EntropyCodeBlockLength block_len_code; -}; - -void EncodeBlockLength(const EntropyCodeBlockLength& entropy, - int length, - int* storage_ix, uint8_t* storage) { - int len_code = BlockLengthPrefix(length); - int extra_bits = BlockLengthExtraBits(len_code); - int extra_bits_value = length - BlockLengthOffset(len_code); - WriteBits(entropy.depth_[len_code], entropy.bits_[len_code], - storage_ix, storage); - if (extra_bits > 0) { - WriteBits(extra_bits, extra_bits_value, storage_ix, storage); - } -} - -void ComputeBlockTypeShortCodes(BlockSplit* split) { - if (split->num_types_ <= 1) { - split->num_types_ = 1; - return; - } - int ringbuffer[2] = { 0, 1 }; - size_t index = 0; - for (int i = 0; i < split->types_.size(); ++i) { - int type = split->types_[i]; - int type_code; - if (type == ringbuffer[index & 1]) { - type_code = 0; - } else if (type == ringbuffer[(index - 1) & 1] + 1) { - type_code = 1; - } else { - type_code = type + 2; - } - ringbuffer[index & 1] = type; - ++index; - split->type_codes_.push_back(type_code); - } -} - -void BuildAndEncodeBlockSplitCode(const BlockSplit& split, - BlockSplitCode* code, - int* storage_ix, uint8_t* storage) { - StoreVarLenUint8(split.num_types_ - 1, storage_ix, storage); - - if (split.num_types_ == 1) { - return; - } - - HistogramBlockType type_histo; - for (int i = 1; i < split.type_codes_.size(); ++i) { - type_histo.Add(split.type_codes_[i]); - } - HistogramBlockLength length_histo; - for (int i = 0; i < split.lengths_.size(); ++i) { - length_histo.Add(BlockLengthPrefix(split.lengths_[i])); - } - BuildAndStoreEntropyCode(type_histo, 15, split.num_types_ + 2, - &code->block_type_code, - storage_ix, storage); - BuildAndStoreEntropyCode(length_histo, 15, kNumBlockLenPrefixes, - &code->block_len_code, - storage_ix, storage); - EncodeBlockLength(code->block_len_code, split.lengths_[0], - storage_ix, storage); -} - void MoveAndEncode(const BlockSplitCode& code, BlockSplitIterator* it, int* storage_ix, uint8_t* storage) { @@ -510,11 +442,7 @@ void MoveAndEncode(const BlockSplitCode& code, ++it->idx_; it->type_ = it->split_.types_[it->idx_]; it->length_ = it->split_.lengths_[it->idx_]; - int type_code = it->split_.type_codes_[it->idx_]; - WriteBits(code.block_type_code.depth_[type_code], - code.block_type_code.bits_[type_code], - storage_ix, storage); - EncodeBlockLength(code.block_len_code, it->length_, storage_ix, storage); + StoreBlockSwitch(code, it->idx_, storage_ix, storage); } --it->length_; } @@ -558,9 +486,6 @@ void BuildMetaBlock(const EncodingParams& params, &mb->literal_split, &mb->command_split, &mb->distance_split); - ComputeBlockTypeShortCodes(&mb->literal_split); - ComputeBlockTypeShortCodes(&mb->command_split); - ComputeBlockTypeShortCodes(&mb->distance_split); mb->literal_context_modes.resize(mb->literal_split.num_types_, mb->params.literal_context_mode); @@ -630,12 +555,24 @@ void StoreMetaBlock(const MetaBlock& mb, BlockSplitCode literal_split_code; BlockSplitCode command_split_code; BlockSplitCode distance_split_code; - BuildAndEncodeBlockSplitCode(mb.literal_split, &literal_split_code, - storage_ix, storage); - BuildAndEncodeBlockSplitCode(mb.command_split, &command_split_code, - storage_ix, storage); - BuildAndEncodeBlockSplitCode(mb.distance_split, &distance_split_code, - storage_ix, storage); + BuildAndStoreBlockSplitCode(mb.literal_split.types_, + mb.literal_split.lengths_, + mb.literal_split.num_types_, + 9, // quality + &literal_split_code, + storage_ix, storage); + BuildAndStoreBlockSplitCode(mb.command_split.types_, + mb.command_split.lengths_, + mb.command_split.num_types_, + 9, // quality + &command_split_code, + storage_ix, storage); + BuildAndStoreBlockSplitCode(mb.distance_split.types_, + mb.distance_split.lengths_, + mb.distance_split.num_types_, + 9, // quality + &distance_split_code, + storage_ix, storage); WriteBits(2, mb.params.distance_postfix_bits, storage_ix, storage); WriteBits(4, mb.params.num_direct_distance_codes >> diff --git a/enc/prefix.cc b/enc/prefix.cc index 50b671e..d158c37 100644 --- a/enc/prefix.cc +++ b/enc/prefix.cc @@ -21,23 +21,6 @@ namespace brotli { -// Represents the range of values belonging to a prefix code: -// [offset, offset + 2^nbits) -struct PrefixCodeRange { - int offset; - int nbits; -}; - -static const PrefixCodeRange kBlockLengthPrefixCode[kNumBlockLenPrefixes] = { - { 1, 2}, { 5, 2}, { 9, 2}, { 13, 2}, - { 17, 3}, { 25, 3}, { 33, 3}, { 41, 3}, - { 49, 4}, { 65, 4}, { 81, 4}, { 97, 4}, - { 113, 5}, { 145, 5}, { 177, 5}, { 209, 5}, - { 241, 6}, { 305, 6}, { 369, 7}, { 497, 8}, - { 753, 9}, { 1265, 10}, {2289, 11}, {4337, 12}, - {8433, 13}, {16625, 24} -}; - static const PrefixCodeRange kInsertLengthPrefixCode[kNumInsertLenPrefixes] = { { 0, 0}, { 1, 0}, { 2, 0}, { 3, 0}, { 4, 0}, { 5, 0}, { 6, 1}, { 8, 1}, @@ -145,22 +128,4 @@ void PrefixEncodeCopyDistance(int distance_code, *extra_bits = (distance_code - offset) >> postfix_bits; } -int BlockLengthPrefix(int length) { - for (int i = 0; i < kNumBlockLenPrefixes; ++i) { - const PrefixCodeRange& range = kBlockLengthPrefixCode[i]; - if (length >= range.offset && length < range.offset + (1 << range.nbits)) { - return i; - } - } - return -1; -} - -int BlockLengthExtraBits(int length_code) { - return kBlockLengthPrefixCode[length_code].nbits; -} - -int BlockLengthOffset(int length_code) { - return kBlockLengthPrefixCode[length_code].offset; -} - } // namespace brotli diff --git a/enc/prefix.h b/enc/prefix.h index 47974f8..cedf494 100644 --- a/enc/prefix.h +++ b/enc/prefix.h @@ -29,6 +29,13 @@ static const int kNumBlockLenPrefixes = 26; static const int kNumDistanceShortCodes = 16; static const int kNumDistancePrefixes = 520; +// Represents the range of values belonging to a prefix code: +// [offset, offset + 2^nbits) +struct PrefixCodeRange { + int offset; + int nbits; +}; + int CommandPrefix(int insert_length, int copy_length); int InsertLengthExtraBits(int prefix); int InsertLengthOffset(int prefix); @@ -42,9 +49,25 @@ void PrefixEncodeCopyDistance(int distance_code, int* nbits, uint32_t* extra_bits); -int BlockLengthPrefix(int length); -int BlockLengthExtraBits(int prefix); -int BlockLengthOffset(int prefix); +static const PrefixCodeRange kBlockLengthPrefixCode[kNumBlockLenPrefixes] = { + { 1, 2}, { 5, 2}, { 9, 2}, { 13, 2}, + { 17, 3}, { 25, 3}, { 33, 3}, { 41, 3}, + { 49, 4}, { 65, 4}, { 81, 4}, { 97, 4}, + { 113, 5}, { 145, 5}, { 177, 5}, { 209, 5}, + { 241, 6}, { 305, 6}, { 369, 7}, { 497, 8}, + { 753, 9}, { 1265, 10}, {2289, 11}, {4337, 12}, + {8433, 13}, {16625, 24} +}; + +inline void GetBlockLengthPrefixCode(int len, + int* code, int* n_extra, int* extra) { + *code = 0; + while (*code < 25 && len >= kBlockLengthPrefixCode[*code + 1].offset) { + ++(*code); + } + *n_extra = kBlockLengthPrefixCode[*code].nbits; + *extra = len - kBlockLengthPrefixCode[*code].offset; +} } // namespace brotli