diff --git a/dec/port.h b/dec/port.h index 3ea5a5d..b23757e 100644 --- a/dec/port.h +++ b/dec/port.h @@ -30,7 +30,7 @@ #include #endif -#include "../common/port.h" +#include #if defined(__arm__) || defined(__thumb__) || \ defined(_M_ARM) || defined(_M_ARMT) diff --git a/enc/brotli_bit_stream.c b/enc/brotli_bit_stream.c index 72f3163..a8eaf48 100644 --- a/enc/brotli_bit_stream.c +++ b/enc/brotli_bit_stream.c @@ -27,6 +27,12 @@ extern "C" { #endif #define MAX_HUFFMAN_TREE_SIZE (2 * BROTLI_NUM_COMMAND_SYMBOLS + 1) +/* The size of Huffman dictionary for distances assuming that NPOSTFIX = 0 and + NDIRECT = 0. */ +#define SIMPLE_DISTANCE_ALPHABET_SIZE (BROTLI_NUM_DISTANCE_SHORT_CODES + \ + (2 * BROTLI_MAX_DISTANCE_BITS)) +/* SIMPLE_DISTANCE_ALPHABET_SIZE == 64 */ +#define SIMPLE_DISTANCE_ALPHABET_BITS 6 /* Represents the range of values belonging to a prefix code: [offset, offset + 2^nbits) */ @@ -1151,12 +1157,12 @@ void BrotliStoreMetaBlockTrivial(MemoryManager* m, HistogramLiteral lit_histo; HistogramCommand cmd_histo; HistogramDistance dist_histo; - uint8_t lit_depth[256]; - uint16_t lit_bits[256]; + uint8_t lit_depth[BROTLI_NUM_LITERAL_SYMBOLS]; + uint16_t lit_bits[BROTLI_NUM_LITERAL_SYMBOLS]; uint8_t cmd_depth[BROTLI_NUM_COMMAND_SYMBOLS]; uint16_t cmd_bits[BROTLI_NUM_COMMAND_SYMBOLS]; - uint8_t dist_depth[64]; - uint16_t dist_bits[64]; + uint8_t dist_depth[SIMPLE_DISTANCE_ALPHABET_SIZE]; + uint16_t dist_bits[SIMPLE_DISTANCE_ALPHABET_SIZE]; HuffmanTree* tree; StoreCompressedMetaBlockHeader(is_last, length, storage_ix, storage); @@ -1172,13 +1178,14 @@ void BrotliStoreMetaBlockTrivial(MemoryManager* m, tree = BROTLI_ALLOC(m, HuffmanTree, MAX_HUFFMAN_TREE_SIZE); if (BROTLI_IS_OOM(m)) return; - BuildAndStoreHuffmanTree(lit_histo.data_, 256, tree, + BuildAndStoreHuffmanTree(lit_histo.data_, BROTLI_NUM_LITERAL_SYMBOLS, tree, lit_depth, lit_bits, storage_ix, storage); BuildAndStoreHuffmanTree(cmd_histo.data_, BROTLI_NUM_COMMAND_SYMBOLS, tree, cmd_depth, cmd_bits, storage_ix, storage); - BuildAndStoreHuffmanTree(dist_histo.data_, 64, tree, + BuildAndStoreHuffmanTree(dist_histo.data_, SIMPLE_DISTANCE_ALPHABET_SIZE, + tree, dist_depth, dist_bits, storage_ix, storage); BROTLI_FREE(m, tree); @@ -1245,8 +1252,8 @@ void BrotliStoreMetaBlockFast(MemoryManager* m, uint16_t lit_bits[BROTLI_NUM_LITERAL_SYMBOLS]; uint8_t cmd_depth[BROTLI_NUM_COMMAND_SYMBOLS]; uint16_t cmd_bits[BROTLI_NUM_COMMAND_SYMBOLS]; - uint8_t dist_depth[64]; - uint16_t dist_bits[64]; + uint8_t dist_depth[SIMPLE_DISTANCE_ALPHABET_SIZE]; + uint16_t dist_bits[SIMPLE_DISTANCE_ALPHABET_SIZE]; HistogramClearLiteral(&lit_histo); HistogramClearCommand(&cmd_histo); HistogramClearDistance(&dist_histo); @@ -1266,7 +1273,8 @@ void BrotliStoreMetaBlockFast(MemoryManager* m, if (BROTLI_IS_OOM(m)) return; BrotliBuildAndStoreHuffmanTreeFast(m, dist_histo.data_, dist_histo.total_count_, - /* max_bits = */ 6, + /* max_bits = */ + SIMPLE_DISTANCE_ALPHABET_BITS, dist_depth, dist_bits, storage_ix, storage); if (BROTLI_IS_OOM(m)) return; diff --git a/enc/command.h b/enc/command.h index 8037ccf..d4a3019 100644 --- a/enc/command.h +++ b/enc/command.h @@ -9,8 +9,9 @@ #ifndef BROTLI_ENC_COMMAND_H_ #define BROTLI_ENC_COMMAND_H_ +#include "../common/constants.h" +#include #include -#include "../common/port.h" #include "./fast_log.h" #include "./prefix.h" @@ -124,18 +125,25 @@ static BROTLI_INLINE void InitInsertCommand(Command* self, size_t insertlen) { self->insert_len_ = (uint32_t)insertlen; self->copy_len_ = 4 << 24; self->dist_extra_ = 0; - self->dist_prefix_ = 16; + self->dist_prefix_ = BROTLI_NUM_DISTANCE_SHORT_CODES; GetLengthCode(insertlen, 4, BROTLI_FALSE, &self->cmd_prefix_); } -static BROTLI_INLINE uint32_t CommandDistanceCode(const Command* self) { - if (self->dist_prefix_ < 16) { +static BROTLI_INLINE uint32_t CommandRestoreDistanceCode(const Command* self) { + if (self->dist_prefix_ < BROTLI_NUM_DISTANCE_SHORT_CODES) { return self->dist_prefix_; } else { uint32_t nbits = self->dist_extra_ >> 24; uint32_t extra = self->dist_extra_ & 0xffffff; - uint32_t prefix = self->dist_prefix_ - 12u - 2u * nbits; - return (prefix << nbits) + extra + 12; + /* It is assumed that the distance was first encoded with NPOSTFIX = 0 and + NDIRECT = 0, so the code itself is of this form: + BROTLI_NUM_DISTANCE_SHORT_CODES + 2 * (nbits - 1) + prefix_bit + Therefore, the following expression results in (2 + prefix_bit). */ + uint32_t prefix = + self->dist_prefix_ + 4u - BROTLI_NUM_DISTANCE_SHORT_CODES - 2u * nbits; + /* Subtract 4 for offset (Chapter 4.) and + increase by BROTLI_NUM_DISTANCE_SHORT_CODES - 1 */ + return (prefix << nbits) + extra + BROTLI_NUM_DISTANCE_SHORT_CODES - 4u; } } diff --git a/enc/compress_fragment.c b/enc/compress_fragment.c index eb8cbe7..f37ea8a 100644 --- a/enc/compress_fragment.c +++ b/enc/compress_fragment.c @@ -30,6 +30,9 @@ extern "C" { #endif +/* Same as MaxBackwardLimit(18) */ +#define MAX_DISTANCE ((1 << 18) - 16) + /* kHashMul32 multiplier has these properties: * The multiplier must be odd. Otherwise we may lose the highest bit. * No long streaks of 1s or 0s. @@ -421,13 +424,11 @@ static uint32_t kCmdHistoSeed[128] = { 1, 1, 1, 1, 0, 0, 0, 0, }; -void BrotliCompressFragmentFast(MemoryManager* m, - const uint8_t* input, size_t input_size, - BROTLI_BOOL is_last, - int* table, size_t table_size, - uint8_t cmd_depth[128], uint16_t cmd_bits[128], - size_t* cmd_code_numbits, uint8_t* cmd_code, - size_t* storage_ix, uint8_t* storage) { +static BROTLI_INLINE void BrotliCompressFragmentFastImpl( + MemoryManager* m, const uint8_t* input, size_t input_size, + BROTLI_BOOL is_last, int* table, size_t table_bits, uint8_t cmd_depth[128], + uint16_t cmd_bits[128], size_t* cmd_code_numbits, uint8_t* cmd_code, + size_t* storage_ix, uint8_t* storage) { uint32_t cmd_histo[128]; const uint8_t* ip_end; @@ -460,13 +461,7 @@ void BrotliCompressFragmentFast(MemoryManager* m, const uint8_t* ip; int last_distance; - const size_t shift = 64u - Log2FloorNonZero(table_size); - assert(table_size); - assert(table_size <= (1u << 31)); - /* table must be power of two */ - assert((table_size & (table_size - 1)) == 0); - assert(table_size - 1 == - (size_t)(MAKE_UINT64_T(0xFFFFFFFF, 0xFFFFFF) >> shift)); + const size_t shift = 64u - table_bits; if (input_size == 0) { assert(is_last); @@ -537,7 +532,7 @@ void BrotliCompressFragmentFast(MemoryManager* m, const uint8_t* next_ip = ip; const uint8_t* candidate; assert(next_emit < ip); - +trawl: do { uint32_t hash = next_hash; uint32_t bytes_between_hash_lookups = skip++ >> 5; @@ -562,6 +557,10 @@ void BrotliCompressFragmentFast(MemoryManager* m, table[hash] = (int)(ip - base_ip); } while (PREDICT_TRUE(!IsMatch(ip, candidate))); + /* Check copy distance. If candidate is not feasible, continue search. + Checking is done outside of hot loop to reduce overhead. */ + if (ip - candidate > MAX_DISTANCE) goto trawl; + /* Step 2: Emit the found match together with the literal bytes from "next_emit" to the bit stream, and then see if we can find a next macth immediately afterwards. Repeat until we find no match for the input @@ -633,6 +632,7 @@ void BrotliCompressFragmentFast(MemoryManager* m, const uint8_t* base = ip; size_t matched = 5 + FindMatchLengthWithLimit( candidate + 5, ip + 5, (size_t)(ip_end - ip) - 5); + if (ip - candidate > MAX_DISTANCE) break; ip += matched; last_distance = (int)(base - candidate); /* > 0 */ assert(0 == memcmp(base, candidate, matched)); @@ -742,6 +742,41 @@ next_block: } } +#define FOR_TABLE_BITS_(X) X(9) X(11) X(13) X(15) + +#define BAKE_METHOD_PARAM_(B) \ +static BROTLI_NOINLINE void BrotliCompressFragmentFastImpl ## B( \ + MemoryManager* m, const uint8_t* input, size_t input_size, \ + BROTLI_BOOL is_last, int* table, uint8_t cmd_depth[128], \ + uint16_t cmd_bits[128], size_t* cmd_code_numbits, uint8_t* cmd_code, \ + size_t* storage_ix, uint8_t* storage) { \ + BrotliCompressFragmentFastImpl(m, input, input_size, is_last, table, B, \ + cmd_depth, cmd_bits, cmd_code_numbits, cmd_code, storage_ix, storage); \ +} +FOR_TABLE_BITS_(BAKE_METHOD_PARAM_) +#undef BAKE_METHOD_PARAM_ + +void BrotliCompressFragmentFast( + MemoryManager* m, const uint8_t* input, size_t input_size, + BROTLI_BOOL is_last, int* table, size_t table_size, uint8_t cmd_depth[128], + uint16_t cmd_bits[128], size_t* cmd_code_numbits, uint8_t* cmd_code, + size_t* storage_ix, uint8_t* storage) { + const size_t table_bits = Log2FloorNonZero(table_size); + switch (table_bits) { +#define CASE_(B) \ + case B: \ + BrotliCompressFragmentFastImpl ## B( \ + m, input, input_size, is_last, table, cmd_depth, cmd_bits, \ + cmd_code_numbits, cmd_code, storage_ix, storage); \ + break; + FOR_TABLE_BITS_(CASE_) +#undef CASE_ + default: assert(0); break; + } +} + +#undef FOR_TABLE_BITS_ + #if defined(__cplusplus) || defined(c_plusplus) } /* extern "C" */ #endif diff --git a/enc/compress_fragment.h b/enc/compress_fragment.h index b513b4f..64e7987 100644 --- a/enc/compress_fragment.h +++ b/enc/compress_fragment.h @@ -38,7 +38,9 @@ extern "C" { REQUIRES: "input_size" is greater than zero, or "is_last" is 1. REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero. - REQUIRES: "table_size" is a power of two */ + REQUIRES: "table_size" is an odd (9, 11, 13, 15) power of two + OUTPUT: maximal copy distance <= |input_size| + OUTPUT: maximal copy distance <= MaxBackwardLimit(18) */ BROTLI_INTERNAL void BrotliCompressFragmentFast(MemoryManager* m, const uint8_t* input, size_t input_size, diff --git a/enc/compress_fragment_two_pass.c b/enc/compress_fragment_two_pass.c index bca914c..eea0773 100644 --- a/enc/compress_fragment_two_pass.c +++ b/enc/compress_fragment_two_pass.c @@ -29,6 +29,9 @@ extern "C" { #endif +/* Same as MaxBackwardLimit(18) */ +#define MAX_DISTANCE ((1 << 18) - 16) + /* kHashMul32 multiplier has these properties: * The multiplier must be odd. Otherwise we may lose the highest bit. * No long streaks of 1s or 0s. @@ -232,12 +235,12 @@ static void BrotliStoreMetaBlockHeader( BrotliWriteBits(1, (uint64_t)is_uncompressed, storage_ix, storage); } -static void CreateCommands(const uint8_t* input, size_t block_size, - size_t input_size, const uint8_t* base_ip, int* table, size_t table_size, - uint8_t** literals, uint32_t** commands) { +static BROTLI_INLINE void CreateCommands(const uint8_t* input, + size_t block_size, size_t input_size, const uint8_t* base_ip, int* table, + size_t table_bits, uint8_t** literals, uint32_t** commands) { /* "ip" is the input pointer. */ const uint8_t* ip = input; - const size_t shift = 64u - Log2FloorNonZero(table_size); + const size_t shift = 64u - table_bits; const uint8_t* ip_end = input + block_size; /* "next_emit" is a pointer to the first byte that is not covered by a previous copy. Bytes between "next_emit" and the start of the next copy or @@ -248,13 +251,6 @@ static void CreateCommands(const uint8_t* input, size_t block_size, const size_t kInputMarginBytes = 16; const size_t kMinMatchLen = 6; - assert(table_size); - assert(table_size <= (1u << 31)); - /* table must be power of two */ - assert((table_size & (table_size - 1)) == 0); - assert(table_size - 1 == - (size_t)(MAKE_UINT64_T(0xFFFFFFFF, 0xFFFFFF) >> shift)); - if (PREDICT_TRUE(block_size >= kInputMarginBytes)) { /* For the last block, we need to keep a 16 bytes margin so that we can be sure that all distances are at most window size - 16. @@ -287,7 +283,7 @@ static void CreateCommands(const uint8_t* input, size_t block_size, const uint8_t* candidate; assert(next_emit < ip); - +trawl: do { uint32_t hash = next_hash; uint32_t bytes_between_hash_lookups = skip++ >> 5; @@ -312,6 +308,10 @@ static void CreateCommands(const uint8_t* input, size_t block_size, table[hash] = (int)(ip - base_ip); } while (PREDICT_TRUE(!IsMatch(ip, candidate))); + /* Check copy distance. If candidate is not feasible, continue search. + Checking is done outside of hot loop to reduce overhead. */ + if (ip - candidate > MAX_DISTANCE) goto trawl; + /* Step 2: Emit the found match together with the literal bytes from "next_emit", and then see if we can find a next macth immediately afterwards. Repeat until we find no match for the input @@ -367,7 +367,7 @@ static void CreateCommands(const uint8_t* input, size_t block_size, } } - while (IsMatch(ip, candidate)) { + while (ip - candidate <= MAX_DISTANCE && IsMatch(ip, candidate)) { /* We have a 6-byte match at ip, and no need to emit any literal bytes prior to ip. */ const uint8_t* base = ip; @@ -504,12 +504,10 @@ static BROTLI_BOOL ShouldCompress( } } -void BrotliCompressFragmentTwoPass(MemoryManager* m, - const uint8_t* input, size_t input_size, - BROTLI_BOOL is_last, - uint32_t* command_buf, uint8_t* literal_buf, - int* table, size_t table_size, - size_t* storage_ix, uint8_t* storage) { +static BROTLI_INLINE void BrotliCompressFragmentTwoPassImpl( + MemoryManager* m, const uint8_t* input, size_t input_size, + BROTLI_BOOL is_last, uint32_t* command_buf, uint8_t* literal_buf, + int* table, size_t table_bits, size_t* storage_ix, uint8_t* storage) { /* Save the start of the first block for position and distance computations. */ const uint8_t* base_ip = input; @@ -520,7 +518,7 @@ void BrotliCompressFragmentTwoPass(MemoryManager* m, uint32_t* commands = command_buf; uint8_t* literals = literal_buf; size_t num_literals; - CreateCommands(input, block_size, input_size, base_ip, table, table_size, + CreateCommands(input, block_size, input_size, base_ip, table, table_bits, &literals, &commands); num_literals = (size_t)(literals - literal_buf); if (ShouldCompress(input, block_size, num_literals)) { @@ -552,6 +550,40 @@ void BrotliCompressFragmentTwoPass(MemoryManager* m, } } +#define FOR_TABLE_BITS_(X) \ + X(8) X(9) X(10) X(11) X(12) X(13) X(14) X(15) X(16) X(17) + +#define BAKE_METHOD_PARAM_(B) \ +static BROTLI_NOINLINE void BrotliCompressFragmentTwoPassImpl ## B( \ + MemoryManager* m, const uint8_t* input, size_t input_size, \ + BROTLI_BOOL is_last, uint32_t* command_buf, uint8_t* literal_buf, \ + int* table, size_t* storage_ix, uint8_t* storage) { \ + BrotliCompressFragmentTwoPassImpl(m, input, input_size, is_last, command_buf,\ + literal_buf, table, B, storage_ix, storage); \ +} +FOR_TABLE_BITS_(BAKE_METHOD_PARAM_) +#undef BAKE_METHOD_PARAM_ + +void BrotliCompressFragmentTwoPass( + MemoryManager* m, const uint8_t* input, size_t input_size, + BROTLI_BOOL is_last, uint32_t* command_buf, uint8_t* literal_buf, + int* table, size_t table_size, size_t* storage_ix, uint8_t* storage) { + const size_t table_bits = Log2FloorNonZero(table_size); + switch (table_bits) { +#define CASE_(B) \ + case B: \ + BrotliCompressFragmentTwoPassImpl ## B( \ + m, input, input_size, is_last, command_buf, \ + literal_buf, table, storage_ix, storage); \ + break; + FOR_TABLE_BITS_(CASE_) +#undef CASE_ + default: assert(0); break; + } +} + +#undef FOR_TABLE_BITS_ + #if defined(__cplusplus) || defined(c_plusplus) } /* extern "C" */ #endif diff --git a/enc/compress_fragment_two_pass.h b/enc/compress_fragment_two_pass.h index 83e3d6b..b75e69d 100644 --- a/enc/compress_fragment_two_pass.h +++ b/enc/compress_fragment_two_pass.h @@ -32,7 +32,9 @@ static const size_t kCompressFragmentTwoPassBlockSize = 1 << 17; REQUIRES: "command_buf" and "literal_buf" point to at least kCompressFragmentTwoPassBlockSize long arrays. REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero. - REQUIRES: "table_size" is a power of two */ + REQUIRES: "table_size" is a power of two + OUTPUT: maximal copy distance <= |input_size| + OUTPUT: maximal copy distance <= MaxBackwardLimit(18) */ BROTLI_INTERNAL void BrotliCompressFragmentTwoPass(MemoryManager* m, const uint8_t* input, size_t input_size, diff --git a/enc/context.h b/enc/context.h index 3c2f192..9cde400 100644 --- a/enc/context.h +++ b/enc/context.h @@ -9,8 +9,8 @@ #ifndef BROTLI_ENC_CONTEXT_H_ #define BROTLI_ENC_CONTEXT_H_ +#include #include -#include "../common/port.h" #if defined(__cplusplus) || defined(c_plusplus) extern "C" { diff --git a/enc/encode.c b/enc/encode.c index 274a146..ba02330 100644 --- a/enc/encode.c +++ b/enc/encode.c @@ -44,7 +44,11 @@ typedef enum BrotliEncoderStreamState { performed before getting back to default state. */ BROTLI_STREAM_FLUSH_REQUESTED = 1, /* Last metablock was produced; no more input is acceptable. */ - BROTLI_STREAM_FINISHED = 2 + BROTLI_STREAM_FINISHED = 2, + /* Flushing compressed block and writing metada block header. */ + BROTLI_STREAM_METADATA_HEAD = 3, + /* Writing metadata block body. */ + BROTLI_STREAM_METADATA_BODY = 4 } BrotliEncoderStreamState; typedef struct BrotliEncoderStateStruct { @@ -95,7 +99,12 @@ typedef struct BrotliEncoderStateStruct { uint8_t* next_out_; size_t available_out_; size_t total_out_; - uint8_t flush_buf_[2]; + /* Temporary buffer for padding flush bits or metadata block header / body. */ + union { + uint64_t u64[2]; + uint8_t u8[16]; + } tiny_buf_; + uint32_t remaining_metadata_bytes_; BrotliEncoderStreamState stream_state_; BROTLI_BOOL is_last_block_emitted_; @@ -104,7 +113,7 @@ typedef struct BrotliEncoderStateStruct { static BROTLI_BOOL EnsureInitialized(BrotliEncoderState* s); -size_t BrotliEncoderInputBlockSize(BrotliEncoderState* s) { +static size_t InputBlockSize(BrotliEncoderState* s) { if (!EnsureInitialized(s)) return 0; return (size_t)1 << s->params.lgblock; } @@ -115,7 +124,7 @@ static uint64_t UnprocessedInputSize(BrotliEncoderState* s) { static size_t RemainingInputBlockSize(BrotliEncoderState* s) { const uint64_t delta = UnprocessedInputSize(s); - size_t block_size = BrotliEncoderInputBlockSize(s); + size_t block_size = InputBlockSize(s); if (delta >= block_size) return 0; return block_size - (size_t)delta; } @@ -157,7 +166,7 @@ static void RecomputeDistancePrefixes(Command* cmds, for (i = 0; i < num_commands; ++i) { Command* cmd = &cmds[i]; if (CommandCopyLen(cmd) && cmd->cmd_prefix_ >= 128) { - PrefixEncodeCopyDistance(CommandDistanceCode(cmd), + PrefixEncodeCopyDistance(CommandRestoreDistanceCode(cmd), num_direct_distance_codes, distance_postfix_bits, &cmd->dist_prefix_, @@ -208,6 +217,12 @@ static int* GetHashTable(BrotliEncoderState* s, int quality, size_t htsize = HashTableSize(max_table_size, input_size); int* table; assert(max_table_size >= 256); + if (quality == FAST_ONE_PASS_COMPRESSION_QUALITY) { + /* Only odd shifts are supported by fast-one-pass. */ + if ((htsize & 0xAAAAA) == 0) { + htsize <<= 1; + } + } if (htsize <= sizeof(s->small_table_) / sizeof(s->small_table_[0])) { table = s->small_table_; @@ -553,10 +568,19 @@ static BROTLI_BOOL EnsureInitialized(BrotliEncoderState* s) { SanitizeParams(&s->params); s->params.lgblock = ComputeLgBlock(&s->params); + s->remaining_metadata_bytes_ = BROTLI_UINT32_MAX; + RingBufferSetup(&s->params, &s->ringbuffer_); /* Initialize last byte with stream header. */ - EncodeWindowBits(s->params.lgwin, &s->last_byte_, &s->last_byte_bits_); + { + int lgwin = s->params.lgwin; + if (s->params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY || + s->params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY) { + lgwin = BROTLI_MAX(int, lgwin, 18); + } + EncodeWindowBits(lgwin, &s->last_byte_, &s->last_byte_bits_); + } if (s->params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY) { InitCommandPrefixCodes(s->cmd_depths_, s->cmd_bits_, @@ -663,9 +687,9 @@ void BrotliEncoderDestroyInstance(BrotliEncoderState* state) { } } -void BrotliEncoderCopyInputToRingBuffer(BrotliEncoderState* s, - const size_t input_size, - const uint8_t* input_buffer) { +static void CopyInputToRingBuffer(BrotliEncoderState* s, + const size_t input_size, + const uint8_t* input_buffer) { RingBuffer* ringbuffer_ = &s->ringbuffer_; MemoryManager* m = &s->memory_manager_; if (!EnsureInitialized(s)) return; @@ -736,7 +760,7 @@ void BrotliEncoderSetCustomDictionary(BrotliEncoderState* s, size_t size, dict += size - max_dict_size; dict_size = max_dict_size; } - BrotliEncoderCopyInputToRingBuffer(s, dict_size, dict); + CopyInputToRingBuffer(s, dict_size, dict); s->last_flush_pos_ = dict_size; s->last_processed_pos_ = dict_size; if (dict_size > 0) { @@ -758,7 +782,7 @@ static BROTLI_BOOL UpdateLastProcessedPos(BrotliEncoderState* s) { return TO_BROTLI_BOOL(wrapped_input_pos < wrapped_last_processed_pos); } -BROTLI_BOOL BrotliEncoderWriteData( +static BROTLI_BOOL EncodeData( BrotliEncoderState* s, const BROTLI_BOOL is_last, const BROTLI_BOOL force_flush, size_t* out_size, uint8_t** output) { const uint64_t delta = UnprocessedInputSize(s); @@ -777,7 +801,7 @@ BROTLI_BOOL BrotliEncoderWriteData( if (s->is_last_block_emitted_) return BROTLI_FALSE; if (is_last) s->is_last_block_emitted_ = BROTLI_TRUE; - if (delta > BrotliEncoderInputBlockSize(s)) { + if (delta > InputBlockSize(s)) { return BROTLI_FALSE; } if (s->params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY && @@ -871,7 +895,7 @@ BROTLI_BOOL BrotliEncoderWriteData( /* If maximal possible additional block doesn't fit metablock, flush now. */ /* TODO: Postpone decision until next block arrives? */ const BROTLI_BOOL next_input_fits_metablock = TO_BROTLI_BOOL( - processed_bytes + BrotliEncoderInputBlockSize(s) <= max_length); + processed_bytes + InputBlockSize(s) <= max_length); /* If block splitting is not used, then flush as soon as there is some amount of commands / literals produced. */ const BROTLI_BOOL should_flush = TO_BROTLI_BOOL( @@ -942,43 +966,31 @@ BROTLI_BOOL BrotliEncoderWriteData( } } -BROTLI_BOOL BrotliEncoderWriteMetadata( - BrotliEncoderState* s, const size_t input_size, const uint8_t* input_buffer, - const BROTLI_BOOL is_last, size_t* encoded_size, uint8_t* encoded_buffer) { - uint64_t hdr_buffer_data[2]; - uint8_t* hdr_buffer = (uint8_t*)&hdr_buffer_data[0]; +/* Dumps remaining output bits and metadata header to |header|. + Returns number of produced bytes. + REQUIRED: |header| should be 8-byte aligned and at least 16 bytes long. + REQUIRED: |block_size| <= (1 << 24). */ +static size_t WriteMetadataHeader( + BrotliEncoderState* s, const size_t block_size, uint8_t* header) { size_t storage_ix; - if (!EnsureInitialized(s)) return BROTLI_FALSE; - if (input_size > (1 << 24) || input_size + 6 > *encoded_size) { - return BROTLI_FALSE; - } storage_ix = s->last_byte_bits_; - hdr_buffer[0] = s->last_byte_; - BrotliWriteBits(1, 0, &storage_ix, hdr_buffer); - BrotliWriteBits(2, 3, &storage_ix, hdr_buffer); - BrotliWriteBits(1, 0, &storage_ix, hdr_buffer); - if (input_size == 0) { - BrotliWriteBits(2, 0, &storage_ix, hdr_buffer); - *encoded_size = (storage_ix + 7u) >> 3; - memcpy(encoded_buffer, hdr_buffer, *encoded_size); - } else { - uint32_t nbits = (input_size == 1) ? 0 : - (Log2FloorNonZero((uint32_t)input_size - 1) + 1); - uint32_t nbytes = (nbits + 7) / 8; - size_t hdr_size; - BrotliWriteBits(2, nbytes, &storage_ix, hdr_buffer); - BrotliWriteBits(8 * nbytes, input_size - 1, &storage_ix, hdr_buffer); - hdr_size = (storage_ix + 7u) >> 3; - memcpy(encoded_buffer, hdr_buffer, hdr_size); - memcpy(&encoded_buffer[hdr_size], input_buffer, input_size); - *encoded_size = hdr_size + input_size; - } - if (is_last) { - encoded_buffer[(*encoded_size)++] = 3; - } + header[0] = s->last_byte_; s->last_byte_ = 0; s->last_byte_bits_ = 0; - return BROTLI_TRUE; + + BrotliWriteBits(1, 0, &storage_ix, header); + BrotliWriteBits(2, 3, &storage_ix, header); + BrotliWriteBits(1, 0, &storage_ix, header); + if (block_size == 0) { + BrotliWriteBits(2, 0, &storage_ix, header); + } else { + uint32_t nbits = (block_size == 1) ? 0 : + (Log2FloorNonZero((uint32_t)block_size - 1) + 1); + uint32_t nbytes = (nbits + 7) / 8; + BrotliWriteBits(2, nbytes, &storage_ix, header); + BrotliWriteBits(8 * nbytes, block_size - 1, &storage_ix, header); + } + return (storage_ix + 7u) >> 3; } static BROTLI_BOOL BrotliCompressBufferQuality10( @@ -1264,7 +1276,8 @@ BROTLI_BOOL BrotliEncoderCompress( } if (quality == 10) { /* TODO: Implement this direct path for all quality levels. */ - const int lg_win = BROTLI_MIN(int, 24, BROTLI_MAX(int, 16, lgwin)); + const int lg_win = BROTLI_MIN(int, kBrotliMaxWindowBits, + BROTLI_MAX(int, 16, lgwin)); int ok = BrotliCompressBufferQuality10(lg_win, input_size, input_buffer, encoded_size, encoded_buffer); if (!ok || (max_out_size && *encoded_size > max_out_size)) { @@ -1321,7 +1334,7 @@ static void InjectBytePaddingBlock(BrotliEncoderState* s) { if (s->next_out_) { destination = s->next_out_ + s->available_out_; } else { - destination = s->flush_buf_; + destination = s->tiny_buf_.u8; s->next_out_ = destination; } destination[0] = (uint8_t)seal; @@ -1329,6 +1342,32 @@ static void InjectBytePaddingBlock(BrotliEncoderState* s) { s->available_out_ += (seal_bits + 7) >> 3; } +/* Injects padding bits or pushes compressed data to output. + Returns false if nothing is done. */ +static BROTLI_BOOL InjectFlushOrPushOutput(BrotliEncoderState* s, + size_t* available_out, uint8_t** next_out, size_t* total_out) { + if (s->stream_state_ == BROTLI_STREAM_FLUSH_REQUESTED && + s->last_byte_bits_ != 0) { + InjectBytePaddingBlock(s); + return BROTLI_TRUE; + } + + if (s->available_out_ != 0 && *available_out != 0) { + size_t copy_output_size = + BROTLI_MIN(size_t, s->available_out_, *available_out); + memcpy(*next_out, s->next_out_, copy_output_size); + *next_out += copy_output_size; + *available_out -= copy_output_size; + s->next_out_ += copy_output_size; + s->available_out_ -= copy_output_size; + s->total_out_ += copy_output_size; + if (total_out) *total_out = s->total_out_; + return BROTLI_TRUE; + } + + return BROTLI_FALSE; +} + static BROTLI_BOOL BrotliEncoderCompressStreamFast( BrotliEncoderState* s, BrotliEncoderOperation op, size_t* available_in, const uint8_t** next_in, size_t* available_out, uint8_t** next_out, @@ -1366,22 +1405,7 @@ static BROTLI_BOOL BrotliEncoderCompressStreamFast( } while (BROTLI_TRUE) { - if (s->stream_state_ == BROTLI_STREAM_FLUSH_REQUESTED && - s->last_byte_bits_ != 0) { - InjectBytePaddingBlock(s); - continue; - } - - if (s->available_out_ != 0 && *available_out != 0) { - size_t copy_output_size = - BROTLI_MIN(size_t, s->available_out_, *available_out); - memcpy(*next_out, s->next_out_, copy_output_size); - *next_out += copy_output_size; - *available_out -= copy_output_size; - s->next_out_ += copy_output_size; - s->available_out_ -= copy_output_size; - s->total_out_ += copy_output_size; - if (total_out) *total_out = s->total_out_; + if (InjectFlushOrPushOutput(s, available_out, next_out, total_out)) { continue; } @@ -1463,12 +1487,96 @@ static BROTLI_BOOL BrotliEncoderCompressStreamFast( return BROTLI_TRUE; } +static BROTLI_BOOL ProcessMetadata( + BrotliEncoderState* s, size_t* available_in, const uint8_t** next_in, + size_t* available_out, uint8_t** next_out, size_t* total_out) { + if (*available_in > (1u << 24)) return BROTLI_FALSE; + /* Switch to metadata block workflow, if required. */ + if (s->stream_state_ == BROTLI_STREAM_PROCESSING) { + s->remaining_metadata_bytes_ = (uint32_t)*available_in; + s->stream_state_ = BROTLI_STREAM_METADATA_HEAD; + } + if (s->stream_state_ != BROTLI_STREAM_METADATA_HEAD && + s->stream_state_ != BROTLI_STREAM_METADATA_BODY) { + return BROTLI_FALSE; + } + + while (BROTLI_TRUE) { + if (InjectFlushOrPushOutput(s, available_out, next_out, total_out)) { + continue; + } + if (s->available_out_ != 0) break; + + if (s->input_pos_ != s->last_flush_pos_) { + BROTLI_BOOL result = EncodeData(s, BROTLI_FALSE, BROTLI_TRUE, + &s->available_out_, &s->next_out_); + if (!result) return BROTLI_FALSE; + continue; + } + + if (s->stream_state_ == BROTLI_STREAM_METADATA_HEAD) { + s->next_out_ = s->tiny_buf_.u8; + s->available_out_ = + WriteMetadataHeader(s, s->remaining_metadata_bytes_, s->next_out_); + s->stream_state_ = BROTLI_STREAM_METADATA_BODY; + continue; + } else { + /* Exit workflow only when there is no more input and no more output. + Otherwise client may continue producing empty metadata blocks. */ + if (s->remaining_metadata_bytes_ == 0) { + s->remaining_metadata_bytes_ = BROTLI_UINT32_MAX; + s->stream_state_ = BROTLI_STREAM_PROCESSING; + break; + } + if (*available_out) { + /* Directly copy input to output. */ + uint32_t copy = (uint32_t)BROTLI_MIN( + size_t, s->remaining_metadata_bytes_, *available_out); + memcpy(*next_out, *next_in, copy); + *next_in += copy; + *available_in -= copy; + s->remaining_metadata_bytes_ -= copy; + *next_out += copy; + *available_out -= copy; + } else { + /* This guarantees progress in "TakeOutput" workflow. */ + uint32_t copy = BROTLI_MIN(uint32_t, s->remaining_metadata_bytes_, 16); + s->next_out_ = s->tiny_buf_.u8; + memcpy(s->next_out_, *next_in, copy); + *next_in += copy; + *available_in -= copy; + s->remaining_metadata_bytes_ -= copy; + s->available_out_ = copy; + } + continue; + } + } + + return BROTLI_TRUE; +} + BROTLI_BOOL BrotliEncoderCompressStream( BrotliEncoderState* s, BrotliEncoderOperation op, size_t* available_in, const uint8_t** next_in, size_t* available_out,uint8_t** next_out, size_t* total_out) { if (!EnsureInitialized(s)) return BROTLI_FALSE; + /* Unfinished metadata block; check requirements. */ + if (s->remaining_metadata_bytes_ != BROTLI_UINT32_MAX) { + if (*available_in != s->remaining_metadata_bytes_) return BROTLI_FALSE; + if (op != BROTLI_OPERATION_EMIT_METADATA) return BROTLI_FALSE; + } + + if (op == BROTLI_OPERATION_EMIT_METADATA) { + return ProcessMetadata( + s, available_in, next_in, available_out, next_out, total_out); + } + + if (s->stream_state_ == BROTLI_STREAM_METADATA_HEAD || + s->stream_state_ == BROTLI_STREAM_METADATA_BODY) { + return BROTLI_FALSE; + } + if (s->stream_state_ != BROTLI_STREAM_PROCESSING && *available_in != 0) { return BROTLI_FALSE; } @@ -1483,28 +1591,13 @@ BROTLI_BOOL BrotliEncoderCompressStream( if (remaining_block_size != 0 && *available_in != 0) { size_t copy_input_size = BROTLI_MIN(size_t, remaining_block_size, *available_in); - BrotliEncoderCopyInputToRingBuffer(s, copy_input_size, *next_in); + CopyInputToRingBuffer(s, copy_input_size, *next_in); *next_in += copy_input_size; *available_in -= copy_input_size; continue; } - if (s->stream_state_ == BROTLI_STREAM_FLUSH_REQUESTED && - s->last_byte_bits_ != 0) { - InjectBytePaddingBlock(s); - continue; - } - - if (s->available_out_ != 0 && *available_out != 0) { - size_t copy_output_size = - BROTLI_MIN(size_t, s->available_out_, *available_out); - memcpy(*next_out, s->next_out_, copy_output_size); - *next_out += copy_output_size; - *available_out -= copy_output_size; - s->next_out_ += copy_output_size; - s->available_out_ -= copy_output_size; - s->total_out_ += copy_output_size; - if (total_out) *total_out = s->total_out_; + if (InjectFlushOrPushOutput(s, available_out, next_out, total_out)) { continue; } @@ -1517,7 +1610,7 @@ BROTLI_BOOL BrotliEncoderCompressStream( (*available_in == 0) && op == BROTLI_OPERATION_FINISH); BROTLI_BOOL force_flush = TO_BROTLI_BOOL( (*available_in == 0) && op == BROTLI_OPERATION_FLUSH); - BROTLI_BOOL result = BrotliEncoderWriteData(s, is_last, force_flush, + BROTLI_BOOL result = EncodeData(s, is_last, force_flush, &s->available_out_, &s->next_out_); if (!result) return BROTLI_FALSE; if (force_flush) s->stream_state_ = BROTLI_STREAM_FLUSH_REQUESTED; @@ -1567,6 +1660,22 @@ uint32_t BrotliEncoderVersion(void) { } +/* DEPRECATED >>> */ +size_t BrotliEncoderInputBlockSize(BrotliEncoderState* s) { + return InputBlockSize(s); +} +void BrotliEncoderCopyInputToRingBuffer(BrotliEncoderState* s, + const size_t input_size, + const uint8_t* input_buffer) { + CopyInputToRingBuffer(s, input_size, input_buffer); +} +BROTLI_BOOL BrotliEncoderWriteData( + BrotliEncoderState* s, const BROTLI_BOOL is_last, + const BROTLI_BOOL force_flush, size_t* out_size, uint8_t** output) { + return EncodeData(s, is_last, force_flush, out_size, output); +} +/* <<< DEPRECATED */ + #if defined(__cplusplus) || defined(c_plusplus) } /* extern "C" */ #endif diff --git a/enc/entropy_encode_static.h b/enc/entropy_encode_static.h index 8501680..2449c30 100644 --- a/enc/entropy_encode_static.h +++ b/enc/entropy_encode_static.h @@ -10,7 +10,7 @@ #define BROTLI_ENC_ENTROPY_ENCODE_STATIC_H_ #include "../common/constants.h" -#include "../common/port.h" +#include #include #include "./write_bits.h" diff --git a/enc/fast_log.h b/enc/fast_log.h index c6bdc8c..b980175 100644 --- a/enc/fast_log.h +++ b/enc/fast_log.h @@ -12,7 +12,7 @@ #include #include -#include "../common/port.h" +#include #if defined(__cplusplus) || defined(c_plusplus) extern "C" { diff --git a/enc/hash.h b/enc/hash.h index 64aaeb6..899218d 100644 --- a/enc/hash.h +++ b/enc/hash.h @@ -178,7 +178,7 @@ static BROTLI_INLINE BROTLI_BOOL SearchInStaticDictionary( return BROTLI_FALSE; } key = Hash14(data) << 1; - for (i = 0; i < (shallow ? 1 : 2); ++i, ++key) { + for (i = 0; i < (shallow ? 1u : 2u); ++i, ++key) { size_t item = kStaticDictionaryHash[key]; self->num_lookups++; if (item != 0 && diff --git a/enc/metablock.c b/enc/metablock.c index cbf7e08..4393c21 100644 --- a/enc/metablock.c +++ b/enc/metablock.c @@ -502,7 +502,8 @@ void BrotliOptimizeHistograms(size_t num_direct_distance_codes, good_for_rle); } num_distance_codes = BROTLI_NUM_DISTANCE_SHORT_CODES + - num_direct_distance_codes + (48u << distance_postfix_bits); + num_direct_distance_codes + + ((2 * BROTLI_MAX_DISTANCE_BITS) << distance_postfix_bits); for (i = 0; i < mb->distance_histograms_size; ++i) { BrotliOptimizeHuffmanCountsForRle(num_distance_codes, mb->distance_histograms[i].data_, diff --git a/enc/port.h b/enc/port.h index 3ef7f43..0d5f24c 100644 --- a/enc/port.h +++ b/enc/port.h @@ -12,7 +12,7 @@ #include #include /* memcpy */ -#include "../common/port.h" +#include #include #if defined OS_LINUX || defined OS_CYGWIN @@ -128,15 +128,6 @@ static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64(void *p, uint64_t v) { #endif -#if !defined(__cplusplus) && !defined(c_plusplus) && \ - (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) -#define BROTLI_RESTRICT restrict -#elif BROTLI_GCC_VERSION > 295 || defined(__llvm__) -#define BROTLI_RESTRICT __restrict -#else -#define BROTLI_RESTRICT -#endif - #define TEMPLATE_(T) \ static BROTLI_INLINE T brotli_min_ ## T (T a, T b) { return a < b ? a : b; } \ static BROTLI_INLINE T brotli_max_ ## T (T a, T b) { return a > b ? a : b; } diff --git a/enc/prefix.h b/enc/prefix.h index d8af9e6..e279ffe 100644 --- a/enc/prefix.h +++ b/enc/prefix.h @@ -11,7 +11,7 @@ #define BROTLI_ENC_PREFIX_H_ #include "../common/constants.h" -#include "../common/port.h" +#include #include #include "./fast_log.h" @@ -19,6 +19,8 @@ extern "C" { #endif +/* Here distance_code is an intermediate code, i.e. one of the special codes or + the actual distance increased by BROTLI_NUM_DISTANCE_SHORT_CODES - 1. */ static BROTLI_INLINE void PrefixEncodeCopyDistance(size_t distance_code, size_t num_direct_codes, size_t postfix_bits, diff --git a/include/brotli/encode.h b/include/brotli/encode.h index cd81700..0ad5164 100755 --- a/include/brotli/encode.h +++ b/include/brotli/encode.h @@ -9,14 +9,15 @@ #ifndef BROTLI_ENC_ENCODE_H_ #define BROTLI_ENC_ENCODE_H_ +#include #include #if defined(__cplusplus) || defined(c_plusplus) extern "C" { #endif -static const int kBrotliMaxWindowBits = 24; static const int kBrotliMinWindowBits = 10; +static const int kBrotliMaxWindowBits = 24; /* == BROTLI_MAX_DISTANCE_BITS */ static const int kBrotliMinInputBlockBits = 16; static const int kBrotliMaxInputBlockBits = 24; @@ -44,7 +45,13 @@ typedef enum BrotliEncoderOperation { BROTLI_OPERATION_FLUSH = 1, /* Request output stream to finish. Performed when input stream is depleted and there is enough space in output stream. */ - BROTLI_OPERATION_FINISH = 2 + BROTLI_OPERATION_FINISH = 2, + /* Emits metadata block to stream. Stream is soft-flushed before metadata + block is emitted. CAUTION: when operation is started, length of the input + buffer is interpreted as length of a metadata block; changing operation, + expanding or truncating input before metadata block is completely emitted + will cause an error; metadata block must not be greater than 16MiB. */ + BROTLI_OPERATION_EMIT_METADATA = 3 } BrotliEncoderOperation; typedef enum BrotliEncoderParameter { @@ -77,20 +84,7 @@ BrotliEncoderState* BrotliEncoderCreateInstance(brotli_alloc_func alloc_func, void BrotliEncoderDestroyInstance(BrotliEncoderState* state); /* The maximum input size that can be processed at once. */ -size_t BrotliEncoderInputBlockSize(BrotliEncoderState* state); - -/* Writes a metadata meta-block containing the given input to encoded_buffer. - |*encoded_size| should be set to the size of the encoded_buffer. - Sets |*encoded_size| to the number of bytes that was written. - Note that the given input data will not be part of the sliding window and - thus no backward references can be made to this data from subsequent - metablocks. |input_size| must not be greater than 2^24 and provided - |*encoded_size| must not be less than |input_size| + 6. - Returns false if there was an error and true otherwise. */ -BROTLI_BOOL BrotliEncoderWriteMetadata( - BrotliEncoderState* state, const size_t input_size, - const uint8_t* input_buffer, const BROTLI_BOOL is_last, - size_t* encoded_size, uint8_t* encoded_buffer); +BROTLI_DEPRECATED size_t BrotliEncoderInputBlockSize(BrotliEncoderState* state); /* Copies the given input data to the internal ring buffer of the compressor. No processing of the data occurs at this time and this function can be @@ -98,9 +92,9 @@ BROTLI_BOOL BrotliEncoderWriteMetadata( accumulated input. At most input_block_size() bytes of input data can be copied to the ring buffer, otherwise the next WriteBrotliData() will fail. */ -void BrotliEncoderCopyInputToRingBuffer(BrotliEncoderState* state, - const size_t input_size, - const uint8_t* input_buffer); +BROTLI_DEPRECATED void BrotliEncoderCopyInputToRingBuffer( + BrotliEncoderState* state, const size_t input_size, + const uint8_t* input_buffer); /* Processes the accumulated input data and sets |*out_size| to the length of the new output meta-block, or to zero if no new output meta-block has been @@ -112,15 +106,15 @@ void BrotliEncoderCopyInputToRingBuffer(BrotliEncoderState* state, use WriteMetadata() to append an empty meta-data block. Returns false if the size of the input data is larger than input_block_size(). */ -BROTLI_BOOL BrotliEncoderWriteData( +BROTLI_DEPRECATED BROTLI_BOOL BrotliEncoderWriteData( BrotliEncoderState* state, const BROTLI_BOOL is_last, const BROTLI_BOOL force_flush, size_t* out_size, uint8_t** output); /* Fills the new state with a dictionary for LZ77, warming up the ringbuffer, e.g. for custom static dictionaries for data formats. Not to be confused with the built-in transformable dictionary of Brotli. - To decode, use BrotliSetCustomDictionary() of the decoder with the same - dictionary. */ + To decode, use BrotliDecoderSetCustomDictionary() of the decoder with the + same dictionary. */ void BrotliEncoderSetCustomDictionary( BrotliEncoderState* state, size_t size, const uint8_t dict[BROTLI_ARRAY_PARAM(size)]); diff --git a/common/port.h b/include/brotli/port.h old mode 100644 new mode 100755 similarity index 100% rename from common/port.h rename to include/brotli/port.h