AuroraMiddleware/brotli
1
0
Fork 0
mirror of https://github.com/google/brotli.git synced 2024-11-24 20:40:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update docs and add more java tests (#463)

Browse Source 
* doxygenize and update API documentation
 * fix spelling
 * add "fuzz" corpus for java decoder to improve coverage
 * use upper-case-snake names for dictionary constant definitions
 * use `LDFLAGS` in conventional `Makefile`
This commit is contained in:
Eugene Kliuchnikov 2016-10-31 14:33:59 +01:00 committed by GitHub
parent a260b6ba73
commit e9b278ac6e
34 changed files with 791 additions and 344 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -23,7 +23,7 @@ $(DIRS):
mkdir -p $@ mkdir -p $@
$(EXECUTABLE): $(OBJECTS) $(EXECUTABLE): $(OBJECTS)
$(CC) $(OBJECTS) -lm -o $(BINDIR)/$(EXECUTABLE) $(CC) $(LDFLAGS) $(OBJECTS) -lm -o $(BINDIR)/$(EXECUTABLE)
lib: $(LIBOBJECTS) lib: $(LIBOBJECTS)
rm -f $(LIB_A) rm -f $(LIB_A)

4
common/dictionary.h
View File

@ -20,8 +20,8 @@ BROTLI_COMMON_API extern const uint8_t kBrotliDictionary[122784];
BROTLI_COMMON_API extern const uint32_t kBrotliDictionaryOffsetsByLength[25]; BROTLI_COMMON_API extern const uint32_t kBrotliDictionaryOffsetsByLength[25];
BROTLI_COMMON_API extern const uint8_t kBrotliDictionarySizeBitsByLength[25]; BROTLI_COMMON_API extern const uint8_t kBrotliDictionarySizeBitsByLength[25];
#define kBrotliMinDictionaryWordLength 4 #define BROTLI_MIN_DICTIONARY_WORD_LENGTH 4
#define kBrotliMaxDictionaryWordLength 24 #define BROTLI_MAX_DICTIONARY_WORD_LENGTH 24
#if defined(__cplusplus) || defined(c_plusplus) #if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */ } /* extern "C" */

2
dec/bit_reader.c
View File

@ -24,7 +24,7 @@ BROTLI_BOOL BrotliWarmupBitReader(BrotliBitReader* const br) {
size_t aligned_read_mask = (sizeof(br->val_) >> 1) - 1; size_t aligned_read_mask = (sizeof(br->val_) >> 1) - 1;
/* Fixing alignment after unaligned BrotliFillWindow would result accumulator /* Fixing alignment after unaligned BrotliFillWindow would result accumulator
overflow. If unalignment is caused by BrotliSafeReadBits, then there is overflow. If unalignment is caused by BrotliSafeReadBits, then there is
enough space in accumulator to fix aligment. */ enough space in accumulator to fix alignment. */
if (!BROTLI_ALIGNED_READ) { if (!BROTLI_ALIGNED_READ) {
aligned_read_mask = 0; aligned_read_mask = 0;
} }

18
dec/bit_reader.h
View File

@ -55,7 +55,7 @@ typedef struct {
size_t avail_in; size_t avail_in;
} BrotliBitReaderState; } BrotliBitReaderState;
/* Initializes the bitreader fields. */ /* Initializes the BrotliBitReader fields. */
BROTLI_INTERNAL void BrotliInitBitReader(BrotliBitReader* const br); BROTLI_INTERNAL void BrotliInitBitReader(BrotliBitReader* const br);
/* Ensures that accumulator is not empty. May consume one byte of input. /* Ensures that accumulator is not empty. May consume one byte of input.
@ -91,8 +91,8 @@ static BROTLI_INLINE size_t BrotliGetRemainingBytes(BrotliBitReader* br) {
return br->avail_in + (BrotliGetAvailableBits(br) >> 3); return br->avail_in + (BrotliGetAvailableBits(br) >> 3);
} }
/* Checks if there is at least num bytes left in the input ringbuffer (excluding /* Checks if there is at least |num| bytes left in the input ring-buffer
the bits remaining in br->val_). */ (excluding the bits remaining in br->val_). */
static BROTLI_INLINE BROTLI_BOOL BrotliCheckInputAmount( static BROTLI_INLINE BROTLI_BOOL BrotliCheckInputAmount(
BrotliBitReader* const br, size_t num) { BrotliBitReader* const br, size_t num) {
return TO_BROTLI_BOOL(br->avail_in >= num); return TO_BROTLI_BOOL(br->avail_in >= num);
@ -157,7 +157,7 @@ static BROTLI_INLINE uint64_t BrotliLoad64LE(const uint8_t* in) {
/* Guarantees that there are at least n_bits + 1 bits in accumulator. /* Guarantees that there are at least n_bits + 1 bits in accumulator.
Precondition: accumulator contains at least 1 bit. Precondition: accumulator contains at least 1 bit.
n_bits should be in the range [1..24] for regular build. For portable n_bits should be in the range [1..24] for regular build. For portable
non-64-bit little endian build only 16 bits are safe to request. */ non-64-bit little-endian build only 16 bits are safe to request. */
static BROTLI_INLINE void BrotliFillBitWindow( static BROTLI_INLINE void BrotliFillBitWindow(
BrotliBitReader* const br, uint32_t n_bits) { BrotliBitReader* const br, uint32_t n_bits) {
#if (BROTLI_64_BITS) #if (BROTLI_64_BITS)
@ -207,7 +207,7 @@ static BROTLI_INLINE void BrotliFillBitWindow(
#endif #endif
} }
/* Mosltly like BrotliFillBitWindow, but guarantees only 16 bits and reads no /* Mostly like BrotliFillBitWindow, but guarantees only 16 bits and reads no
more than BROTLI_SHORT_FILL_BIT_WINDOW_READ bytes of input. */ more than BROTLI_SHORT_FILL_BIT_WINDOW_READ bytes of input. */
static BROTLI_INLINE void BrotliFillBitWindow16(BrotliBitReader* const br) { static BROTLI_INLINE void BrotliFillBitWindow16(BrotliBitReader* const br) {
BrotliFillBitWindow(br, 17); BrotliFillBitWindow(br, 17);
@ -231,7 +231,7 @@ static BROTLI_INLINE BROTLI_BOOL BrotliPullByte(BrotliBitReader* const br) {
} }
/* Returns currently available bits. /* Returns currently available bits.
The number of valid bits could be calclulated by BrotliGetAvailableBits. */ The number of valid bits could be calculated by BrotliGetAvailableBits. */
static BROTLI_INLINE reg_t BrotliGetBitsUnmasked(BrotliBitReader* const br) { static BROTLI_INLINE reg_t BrotliGetBitsUnmasked(BrotliBitReader* const br) {
return br->val_ >> br->bit_pos_; return br->val_ >> br->bit_pos_;
} }
@ -244,7 +244,7 @@ static BROTLI_INLINE uint32_t BrotliGet16BitsUnmasked(
return (uint32_t)BrotliGetBitsUnmasked(br); return (uint32_t)BrotliGetBitsUnmasked(br);
} }
/* Returns the specified number of bits from br without advancing bit pos. */ /* Returns the specified number of bits from |br| without advancing bit pos. */
static BROTLI_INLINE uint32_t BrotliGetBits( static BROTLI_INLINE uint32_t BrotliGetBits(
BrotliBitReader* const br, uint32_t n_bits) { BrotliBitReader* const br, uint32_t n_bits) {
BrotliFillBitWindow(br, n_bits); BrotliFillBitWindow(br, n_bits);
@ -283,7 +283,7 @@ static BROTLI_INLINE void BrotliBitReaderUnload(BrotliBitReader* br) {
br->bit_pos_ += unused_bits; br->bit_pos_ += unused_bits;
} }
/* Reads the specified number of bits from br and advances the bit pos. /* Reads the specified number of bits from |br| and advances the bit pos.
Precondition: accumulator MUST contain at least n_bits. */ Precondition: accumulator MUST contain at least n_bits. */
static BROTLI_INLINE void BrotliTakeBits( static BROTLI_INLINE void BrotliTakeBits(
BrotliBitReader* const br, uint32_t n_bits, uint32_t* val) { BrotliBitReader* const br, uint32_t n_bits, uint32_t* val) {
@ -293,7 +293,7 @@ static BROTLI_INLINE void BrotliTakeBits(
BrotliDropBits(br, n_bits); BrotliDropBits(br, n_bits);
} }
/* Reads the specified number of bits from br and advances the bit pos. /* Reads the specified number of bits from |br| and advances the bit pos.
Assumes that there is enough input to perform BrotliFillBitWindow. */ Assumes that there is enough input to perform BrotliFillBitWindow. */
static BROTLI_INLINE uint32_t BrotliReadBits( static BROTLI_INLINE uint32_t BrotliReadBits(
BrotliBitReader* const br, uint32_t n_bits) { BrotliBitReader* const br, uint32_t n_bits) {

42
dec/decode.c
View File

@ -454,8 +454,8 @@ static BrotliDecoderErrorCode ReadSimpleHuffmanSymbols(
/* Process single decoded symbol code length: /* Process single decoded symbol code length:
A) reset the repeat variable A) reset the repeat variable
B) remember code length (if it is not 0) B) remember code length (if it is not 0)
C) extend corredponding index-chain C) extend corresponding index-chain
D) reduce the huffman space D) reduce the Huffman space
E) update the histogram E) update the histogram
*/ */
static BROTLI_INLINE void ProcessSingleCodeLength(uint32_t code_len, static BROTLI_INLINE void ProcessSingleCodeLength(uint32_t code_len,
@ -479,7 +479,7 @@ static BROTLI_INLINE void ProcessSingleCodeLength(uint32_t code_len,
value is not BROTLI_REPEAT_PREVIOUS_CODE_LENGTH, then it is a new value is not BROTLI_REPEAT_PREVIOUS_CODE_LENGTH, then it is a new
symbol-skip symbol-skip
B) Update repeat variable B) Update repeat variable
C) Check if operation is feasible (fits alphapet) C) Check if operation is feasible (fits alphabet)
D) For each symbol do the same operations as in ProcessSingleCodeLength D) For each symbol do the same operations as in ProcessSingleCodeLength
PRECONDITION: code_len == BROTLI_REPEAT_PREVIOUS_CODE_LENGTH or PRECONDITION: code_len == BROTLI_REPEAT_PREVIOUS_CODE_LENGTH or
@ -949,7 +949,7 @@ static BrotliDecoderErrorCode DecodeContextMap(uint32_t context_map_size,
if (!BrotliSafeGetBits(br, 5, &bits)) { if (!BrotliSafeGetBits(br, 5, &bits)) {
return BROTLI_DECODER_NEEDS_MORE_INPUT; return BROTLI_DECODER_NEEDS_MORE_INPUT;
} }
if ((bits & 1) != 0) { /* Use RLE for zeroes. */ if ((bits & 1) != 0) { /* Use RLE for zeros. */
s->max_run_length_prefix = (bits >> 1) + 1; s->max_run_length_prefix = (bits >> 1) + 1;
BrotliDropBits(br, 5); BrotliDropBits(br, 5);
} else { } else {
@ -1031,7 +1031,7 @@ rleCode:
} }
} }
/* Decodes a command or literal and updates block type ringbuffer. /* Decodes a command or literal and updates block type ring-buffer.
Reads 3..54 bits. */ Reads 3..54 bits. */
static BROTLI_INLINE BROTLI_BOOL DecodeBlockTypeAndLength( static BROTLI_INLINE BROTLI_BOOL DecodeBlockTypeAndLength(
int safe, BrotliDecoderState* s, int tree_type) { int safe, BrotliDecoderState* s, int tree_type) {
@ -1176,7 +1176,7 @@ static size_t UnwrittenBytes(const BrotliDecoderState* s, BROTLI_BOOL wrap) {
/* Dumps output. /* Dumps output.
Returns BROTLI_DECODER_NEEDS_MORE_OUTPUT only if there is more output to push Returns BROTLI_DECODER_NEEDS_MORE_OUTPUT only if there is more output to push
and either ringbuffer is as big as window size, or |force| is true. and either ring-buffer is as big as window size, or |force| is true.
*/ */
static BrotliDecoderErrorCode BROTLI_NOINLINE WriteRingBuffer( static BrotliDecoderErrorCode BROTLI_NOINLINE WriteRingBuffer(
BrotliDecoderState* s, size_t* available_out, uint8_t** next_out, BrotliDecoderState* s, size_t* available_out, uint8_t** next_out,
@ -1228,15 +1228,15 @@ static void BROTLI_NOINLINE WrapRingBuffer(BrotliDecoderState* s) {
} }
} }
/* Allocates ringbuffer. /* Allocates ring-buffer.
s->ringbuffer_size MUST be updated by BrotliCalculateRingBufferSize before s->ringbuffer_size MUST be updated by BrotliCalculateRingBufferSize before
this function is called. this function is called.
Last two bytes of ringbuffer are initialized to 0, so context calculation Last two bytes of ring-buffer are initialized to 0, so context calculation
could be done uniformly for the first two and all other positions. could be done uniformly for the first two and all other positions.
Custom dictionary, if any, is copied to the end of ringbuffer. Custom dictionary, if any, is copied to the end of ring-buffer.
*/ */
static BROTLI_BOOL BROTLI_NOINLINE BrotliEnsureRingBuffer( static BROTLI_BOOL BROTLI_NOINLINE BrotliEnsureRingBuffer(
BrotliDecoderState* s) { BrotliDecoderState* s) {
@ -1296,7 +1296,7 @@ static BrotliDecoderErrorCode BROTLI_NOINLINE CopyUncompressedBlockToOutput(
if (s->pos + nbytes > s->ringbuffer_size) { if (s->pos + nbytes > s->ringbuffer_size) {
nbytes = s->ringbuffer_size - s->pos; nbytes = s->ringbuffer_size - s->pos;
} }
/* Copy remaining bytes from s->br.buf_ to ringbuffer. */ /* Copy remaining bytes from s->br.buf_ to ring-buffer. */
BrotliCopyBytes(&s->ringbuffer[s->pos], &s->br, (size_t)nbytes); BrotliCopyBytes(&s->ringbuffer[s->pos], &s->br, (size_t)nbytes);
s->pos += nbytes; s->pos += nbytes;
s->meta_block_remaining_len -= nbytes; s->meta_block_remaining_len -= nbytes;
@ -1343,7 +1343,7 @@ static void BROTLI_NOINLINE BrotliCalculateRingBufferSize(
int min_size = s->ringbuffer_size ? s->ringbuffer_size : 1024; int min_size = s->ringbuffer_size ? s->ringbuffer_size : 1024;
int output_size; int output_size;
/* If maxumum is already reached, no further extention is reuired. */ /* If maximum is already reached, no further extension is retired. */
if (s->ringbuffer_size == window_size) { if (s->ringbuffer_size == window_size) {
return; return;
} }
@ -1354,7 +1354,7 @@ static void BROTLI_NOINLINE BrotliCalculateRingBufferSize(
} }
if (!s->ringbuffer) { if (!s->ringbuffer) {
/* Custom dictionanry counts as a "virtual" output. */ /* Custom dictionary counts as a "virtual" output. */
output_size = s->custom_dict_size; output_size = s->custom_dict_size;
} else { } else {
output_size = s->pos; output_size = s->pos;
@ -1724,8 +1724,8 @@ postReadDistance:
/* Apply copy of LZ77 back-reference, or static dictionary reference if /* Apply copy of LZ77 back-reference, or static dictionary reference if
the distance is larger than the max LZ77 distance */ the distance is larger than the max LZ77 distance */
if (s->distance_code > s->max_distance) { if (s->distance_code > s->max_distance) {
if (i >= kBrotliMinDictionaryWordLength && if (i >= BROTLI_MIN_DICTIONARY_WORD_LENGTH &&
i <= kBrotliMaxDictionaryWordLength) { i <= BROTLI_MAX_DICTIONARY_WORD_LENGTH) {
int offset = (int)kBrotliDictionaryOffsetsByLength[i]; int offset = (int)kBrotliDictionaryOffsetsByLength[i];
int word_id = s->distance_code - s->max_distance - 1; int word_id = s->distance_code - s->max_distance - 1;
uint32_t shift = kBrotliDictionarySizeBitsByLength[i]; uint32_t shift = kBrotliDictionarySizeBitsByLength[i];
@ -1771,9 +1771,9 @@ postReadDistance:
s->dist_rb[s->dist_rb_idx & 3] = s->distance_code; s->dist_rb[s->dist_rb_idx & 3] = s->distance_code;
++s->dist_rb_idx; ++s->dist_rb_idx;
s->meta_block_remaining_len -= i; s->meta_block_remaining_len -= i;
/* There are 32+ bytes of slack in the ringbuffer allocation. /* There are 32+ bytes of slack in the ring-buffer allocation.
Also, we have 16 short codes, that make these 16 bytes irrelevant Also, we have 16 short codes, that make these 16 bytes irrelevant
in the ringbuffer. Let's copy over them as a first guess. in the ring-buffer. Let's copy over them as a first guess.
*/ */
memmove16(copy_dst, copy_src); memmove16(copy_dst, copy_src);
if (src_end > pos && dst_end > src_start) { if (src_end > pos && dst_end > src_start) {
@ -1866,7 +1866,7 @@ BrotliDecoderResult BrotliDecoderDecompress(
/* Invariant: input stream is never overconsumed: /* Invariant: input stream is never overconsumed:
* invalid input implies that the whole stream is invalid -> any amount of * invalid input implies that the whole stream is invalid -> any amount of
input could be read and discarded input could be read and discarded
* when result is "needs more input", then at leat one more byte is REQUIRED * when result is "needs more input", then at least one more byte is REQUIRED
to complete decoding; all input data MUST be consumed by decoder, so to complete decoding; all input data MUST be consumed by decoder, so
client could swap the input buffer client could swap the input buffer
* when result is "needs more output" decoder MUST ensure that it doesn't * when result is "needs more output" decoder MUST ensure that it doesn't
@ -1899,12 +1899,12 @@ BrotliDecoderResult BrotliDecoderDecompressStream(
for (;;) { for (;;) {
if (result != BROTLI_DECODER_SUCCESS) { /* Error, needs more input/output */ if (result != BROTLI_DECODER_SUCCESS) { /* Error, needs more input/output */
if (result == BROTLI_DECODER_NEEDS_MORE_INPUT) { if (result == BROTLI_DECODER_NEEDS_MORE_INPUT) {
if (s->ringbuffer != 0) { /* Proactively push output. */ if (s->ringbuffer != 0) { /* Pro-actively push output. */
WriteRingBuffer(s, available_out, next_out, total_out, BROTLI_TRUE); WriteRingBuffer(s, available_out, next_out, total_out, BROTLI_TRUE);
} }
if (s->buffer_length != 0) { /* Used with internal buffer. */ if (s->buffer_length != 0) { /* Used with internal buffer. */
if (br->avail_in == 0) { /* Successfully finished read transaction. */ if (br->avail_in == 0) { /* Successfully finished read transaction. */
/* Accamulator contains less than 8 bits, because internal buffer /* Accumulator contains less than 8 bits, because internal buffer
is expanded byte-by-byte until it is enough to complete read. */ is expanded byte-by-byte until it is enough to complete read. */
s->buffer_length = 0; s->buffer_length = 0;
/* Switch to input stream and restart. */ /* Switch to input stream and restart. */
@ -1949,8 +1949,8 @@ BrotliDecoderResult BrotliDecoderDecompressStream(
s->buffer_length = 0; s->buffer_length = 0;
} else { } else {
/* Using input stream in last iteration. When decoder switches to input /* Using input stream in last iteration. When decoder switches to input
stream it has less than 8 bits in accamulator, so it is safe to stream it has less than 8 bits in accumulator, so it is safe to
return unused accamulator bits there. */ return unused accumulator bits there. */
BrotliBitReaderUnload(br); BrotliBitReaderUnload(br);
*available_in = br->avail_in; *available_in = br->avail_in;
*next_in = br->next_in; *next_in = br->next_in;

2
dec/port.h
View File

@ -60,7 +60,7 @@
#define BROTLI_ALIGNED_READ (!!1) #define BROTLI_ALIGNED_READ (!!1)
#elif defined(BROTLI_TARGET_X86) || defined(BROTLI_TARGET_X64) || \ #elif defined(BROTLI_TARGET_X86) || defined(BROTLI_TARGET_X64) || \
defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8) defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8)
/* Allow unaligned read only for whitelisted CPUs. */ /* Allow unaligned read only for white-listed CPUs. */
#define BROTLI_ALIGNED_READ (!!0) #define BROTLI_ALIGNED_READ (!!0)
#else #else
#define BROTLI_ALIGNED_READ (!!1) #define BROTLI_ALIGNED_READ (!!1)

2
dec/state.h
View File

@ -160,7 +160,7 @@ struct BrotliDecoderStateStruct {
int distance_code; int distance_code;
/* For partial write operations */ /* For partial write operations */
size_t rb_roundtrips; /* How many times we went around the ringbuffer */ size_t rb_roundtrips; /* How many times we went around the ring-buffer */
size_t partial_pos_out; /* How much output to the user in total */ size_t partial_pos_out; /* How much output to the user in total */
/* For ReadHuffmanCode */ /* For ReadHuffmanCode */

2
dec/transform.h
View File

@ -247,7 +247,7 @@ static int ToUpperCase(uint8_t* p) {
} }
return 1; return 1;
} }
/* An overly simplified uppercasing model for utf-8. */ /* An overly simplified uppercasing model for UTF-8. */
if (p[0] < 0xe0) { if (p[0] < 0xe0) {
p[1] ^= 32; p[1] ^= 32;
return 2; return 2;

4
enc/backward_references.h
View File

@ -45,7 +45,7 @@ typedef struct ZopfliNode {
/* This union holds information used by dynamic-programming. During forward /* This union holds information used by dynamic-programming. During forward
pass |cost| it used to store the goal function. When node is processed its pass |cost| it used to store the goal function. When node is processed its
|cost| is invalidated in favor of |shortcut|. On path backtracing pass |cost| is invalidated in favor of |shortcut|. On path back-tracing pass
|next| is assigned the offset to next node on the path. */ |next| is assigned the offset to next node on the path. */
union { union {
/* Smallest cost to get to this byte from the beginning, as found so far. */ /* Smallest cost to get to this byte from the beginning, as found so far. */
@ -64,7 +64,7 @@ BROTLI_INTERNAL void BrotliInitZopfliNodes(ZopfliNode* array, size_t length);
position + num_bytes. position + num_bytes.
On return, path->size() is the number of commands found and path[i] is the On return, path->size() is the number of commands found and path[i] is the
length of the ith command (copy length plus insert length). length of the i-th command (copy length plus insert length).
Note that the sum of the lengths of all commands can be less than num_bytes. Note that the sum of the lengths of all commands can be less than num_bytes.
On return, the nodes[0..num_bytes] array will have the following On return, the nodes[0..num_bytes] array will have the following

2
enc/backward_references_inc.h
View File

@ -80,7 +80,7 @@ static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
position + 2 * sr.len + random_heuristics_window_size; position + 2 * sr.len + random_heuristics_window_size;
max_distance = BROTLI_MIN(size_t, position, max_backward_limit); max_distance = BROTLI_MIN(size_t, position, max_backward_limit);
{ {
/* The first 16 codes are special shortcodes, /* The first 16 codes are special short-codes,
and the minimum offset is 1. */ and the minimum offset is 1. */
size_t distance_code = size_t distance_code =
ComputeDistanceCode(sr.distance, max_distance, dist_cache); ComputeDistanceCode(sr.distance, max_distance, dist_cache);

2
enc/block_splitter_inc.h
View File

@ -61,7 +61,7 @@ static void FN(RefineEntropyCodes)(const DataType* data, size_t length,
} }
} }
/* Assigns a block id from the range [0, vec.size()) to each data element /* Assigns a block id from the range [0, num_histograms) to each data element
in data[0..length) and fills in block_id[0..length) with the assigned values. in data[0..length) and fills in block_id[0..length) with the assigned values.
Returns the number of blocks, i.e. one plus the number of block switches. */ Returns the number of blocks, i.e. one plus the number of block switches. */
static size_t FN(FindBlocks)(const DataType* data, const size_t length, static size_t FN(FindBlocks)(const DataType* data, const size_t length,

6
enc/brotli_bit_stream.c
View File

@ -82,7 +82,7 @@ static BROTLI_INLINE size_t NextBlockTypeCode(
return type_code; return type_code;
} }
/* nibblesbits represents the 2 bits to encode MNIBBLES (0-3) /* |nibblesbits| represents the 2 bits to encode MNIBBLES (0-3)
REQUIRES: length > 0 REQUIRES: length > 0
REQUIRES: length <= (1 << 24) */ REQUIRES: length <= (1 << 24) */
static void BrotliEncodeMlen(size_t length, uint64_t* bits, static void BrotliEncodeMlen(size_t length, uint64_t* bits,
@ -349,7 +349,7 @@ void BrotliStoreHuffmanTree(const uint8_t* depths, size_t num,
code_length_bitdepth[code] = 0; code_length_bitdepth[code] = 0;
} }
/* Store the real huffman tree now. */ /* Store the real Huffman tree now. */
BrotliStoreHuffmanTreeToBitMask(huffman_tree_size, BrotliStoreHuffmanTreeToBitMask(huffman_tree_size,
huffman_tree, huffman_tree,
huffman_tree_extra_bits, huffman_tree_extra_bits,
@ -554,7 +554,7 @@ void BrotliBuildAndStoreHuffmanTreeFast(MemoryManager* m,
/* Complex Huffman Tree */ /* Complex Huffman Tree */
StoreStaticCodeLengthCode(storage_ix, storage); StoreStaticCodeLengthCode(storage_ix, storage);
/* Actual rle coding. */ /* Actual RLE coding. */
for (i = 0; i < length;) { for (i = 0; i < length;) {
const uint8_t value = depth[i]; const uint8_t value = depth[i];
size_t reps = 1; size_t reps = 1;

8
enc/compress_fragment.c
View File

@ -36,7 +36,7 @@ extern "C" {
/* kHashMul32 multiplier has these properties: /* kHashMul32 multiplier has these properties:
* The multiplier must be odd. Otherwise we may lose the highest bit. * The multiplier must be odd. Otherwise we may lose the highest bit.
* No long streaks of 1s or 0s. * No long streaks of ones or zeros.
* There is no effort to ensure that it is a prime, the oddity is enough * There is no effort to ensure that it is a prime, the oddity is enough
for this use. for this use.
* The number has been tuned heuristically against compression benchmarks. */ * The number has been tuned heuristically against compression benchmarks. */
@ -136,7 +136,7 @@ static void BuildAndStoreCommandPrefixCode(const uint32_t histogram[128],
BrotliCreateHuffmanTree(histogram, 64, 15, tree, depth); BrotliCreateHuffmanTree(histogram, 64, 15, tree, depth);
BrotliCreateHuffmanTree(&histogram[64], 64, 14, tree, &depth[64]); BrotliCreateHuffmanTree(&histogram[64], 64, 14, tree, &depth[64]);
/* We have to jump through a few hoopes here in order to compute /* We have to jump through a few hoops here in order to compute
the command bits because the symbols are in a different order than in the command bits because the symbols are in a different order than in
the full alphabet. This looks complicated, but having the symbols the full alphabet. This looks complicated, but having the symbols
in this order in the command bits saves a few branches in the Emit* in this order in the command bits saves a few branches in the Emit*
@ -526,7 +526,7 @@ static BROTLI_INLINE void BrotliCompressFragmentFastImpl(
and doesn't bother looking for matches everywhere. and doesn't bother looking for matches everywhere.
The "skip" variable keeps track of how many bytes there are since the The "skip" variable keeps track of how many bytes there are since the
last match; dividing it by 32 (ie. right-shifting by five) gives the last match; dividing it by 32 (i.e. right-shifting by five) gives the
number of bytes to move ahead for each iteration. */ number of bytes to move ahead for each iteration. */
uint32_t skip = 32; uint32_t skip = 32;
@ -563,7 +563,7 @@ trawl:
if (ip - candidate > MAX_DISTANCE) goto trawl; if (ip - candidate > MAX_DISTANCE) goto trawl;
/* Step 2: Emit the found match together with the literal bytes from /* Step 2: Emit the found match together with the literal bytes from
"next_emit" to the bit stream, and then see if we can find a next macth "next_emit" to the bit stream, and then see if we can find a next match
immediately afterwards. Repeat until we find no match for the input immediately afterwards. Repeat until we find no match for the input
without emitting some literal bytes. */ without emitting some literal bytes. */

6
enc/compress_fragment_two_pass.c
View File

@ -35,7 +35,7 @@ extern "C" {
/* kHashMul32 multiplier has these properties: /* kHashMul32 multiplier has these properties:
* The multiplier must be odd. Otherwise we may lose the highest bit. * The multiplier must be odd. Otherwise we may lose the highest bit.
* No long streaks of 1s or 0s. * No long streaks of ones or zeros.
* There is no effort to ensure that it is a prime, the oddity is enough * There is no effort to ensure that it is a prime, the oddity is enough
for this use. for this use.
* The number has been tuned heuristically against compression benchmarks. */ * The number has been tuned heuristically against compression benchmarks. */
@ -75,7 +75,7 @@ static void BuildAndStoreCommandPrefixCode(
uint16_t cmd_bits[64]; uint16_t cmd_bits[64];
BrotliCreateHuffmanTree(histogram, 64, 15, tree, depth); BrotliCreateHuffmanTree(histogram, 64, 15, tree, depth);
BrotliCreateHuffmanTree(&histogram[64], 64, 14, tree, &depth[64]); BrotliCreateHuffmanTree(&histogram[64], 64, 14, tree, &depth[64]);
/* We have to jump through a few hoopes here in order to compute /* We have to jump through a few hoops here in order to compute
the command bits because the symbols are in a different order than in the command bits because the symbols are in a different order than in
the full alphabet. This looks complicated, but having the symbols the full alphabet. This looks complicated, but having the symbols
in this order in the command bits saves a few branches in the Emit* in this order in the command bits saves a few branches in the Emit*
@ -314,7 +314,7 @@ trawl:
if (ip - candidate > MAX_DISTANCE) goto trawl; if (ip - candidate > MAX_DISTANCE) goto trawl;
/* Step 2: Emit the found match together with the literal bytes from /* Step 2: Emit the found match together with the literal bytes from
"next_emit", and then see if we can find a next macth immediately "next_emit", and then see if we can find a next match immediately
afterwards. Repeat until we find no match for the input afterwards. Repeat until we find no match for the input
without emitting some literal bytes. */ without emitting some literal bytes. */

57
enc/encode.c
View File

@ -45,7 +45,7 @@ typedef enum BrotliEncoderStreamState {
BROTLI_STREAM_FLUSH_REQUESTED = 1, BROTLI_STREAM_FLUSH_REQUESTED = 1,
/* Last metablock was produced; no more input is acceptable. */ /* Last metablock was produced; no more input is acceptable. */
BROTLI_STREAM_FINISHED = 2, BROTLI_STREAM_FINISHED = 2,
/* Flushing compressed block and writing metada block header. */ /* Flushing compressed block and writing meta-data block header. */
BROTLI_STREAM_METADATA_HEAD = 3, BROTLI_STREAM_METADATA_HEAD = 3,
/* Writing metadata block body. */ /* Writing metadata block body. */
BROTLI_STREAM_METADATA_BODY = 4 BROTLI_STREAM_METADATA_BODY = 4
@ -133,7 +133,7 @@ BROTLI_BOOL BrotliEncoderSetParameter(
BrotliEncoderState* state, BrotliEncoderParameter p, uint32_t value) { BrotliEncoderState* state, BrotliEncoderParameter p, uint32_t value) {
/* Changing parameters on the fly is not implemented yet. */ /* Changing parameters on the fly is not implemented yet. */
if (state->is_initialized_) return BROTLI_FALSE; if (state->is_initialized_) return BROTLI_FALSE;
/* TODO: Validate/clamp params here. */ /* TODO: Validate/clamp parameters here. */
switch (p) { switch (p) {
case BROTLI_PARAM_MODE: case BROTLI_PARAM_MODE:
state->params.mode = (BrotliEncoderMode)value; state->params.mode = (BrotliEncoderMode)value;
@ -175,13 +175,13 @@ static void RecomputeDistancePrefixes(Command* cmds,
} }
} }
/* Wraps 64-bit input position to 32-bit ringbuffer position preserving /* Wraps 64-bit input position to 32-bit ring-buffer position preserving
"not-a-first-lap" feature. */ "not-a-first-lap" feature. */
static uint32_t WrapPosition(uint64_t position) { static uint32_t WrapPosition(uint64_t position) {
uint32_t result = (uint32_t)position; uint32_t result = (uint32_t)position;
uint64_t gb = position >> 30; uint64_t gb = position >> 30;
if (gb > 2) { if (gb > 2) {
/* Wrap every 2GiB; The first 3GB are continous. */ /* Wrap every 2GiB; The first 3GB are continuous. */
result = (result & ((1u << 30) - 1)) | ((uint32_t)((gb - 1) & 1) + 1) << 30; result = (result & ((1u << 30) - 1)) | ((uint32_t)((gb - 1) & 1) + 1) << 30;
} }
return result; return result;
@ -306,8 +306,8 @@ static void InitCommandPrefixCodes(uint8_t cmd_depths[128],
/* Decide about the context map based on the ability of the prediction /* Decide about the context map based on the ability of the prediction
ability of the previous byte UTF8-prefix on the next byte. The ability of the previous byte UTF8-prefix on the next byte. The
prediction ability is calculated as shannon entropy. Here we need prediction ability is calculated as Shannon entropy. Here we need
shannon entropy instead of 'BitsEntropy' since the prefix will be Shannon entropy instead of 'BitsEntropy' since the prefix will be
encoded with the remaining 6 bits of the following byte, and encoded with the remaining 6 bits of the following byte, and
BitsEntropy will assume that symbol to be stored alone using Huffman BitsEntropy will assume that symbol to be stored alone using Huffman
coding. */ coding. */
@ -382,7 +382,7 @@ static void DecideOverLiteralContextModeling(const uint8_t* input,
if (quality < MIN_QUALITY_FOR_CONTEXT_MODELING || length < 64) { if (quality < MIN_QUALITY_FOR_CONTEXT_MODELING || length < 64) {
return; return;
} else { } else {
/* Gather bigram data of the UTF8 byte prefixes. To make the analysis of /* Gather bi-gram data of the UTF8 byte prefixes. To make the analysis of
UTF8 data faster we only examine 64 byte long strides at every 4kB UTF8 data faster we only examine 64 byte long strides at every 4kB
intervals. */ intervals. */
const size_t end_pos = start_pos + length; const size_t end_pos = start_pos + length;
@ -687,6 +687,13 @@ void BrotliEncoderDestroyInstance(BrotliEncoderState* state) {
} }
} }
/*
Copies the given input data to the internal ring buffer of the compressor.
No processing of the data occurs at this time and this function can be
called multiple times before calling WriteBrotliData() to process the
accumulated input. At most input_block_size() bytes of input data can be
copied to the ring buffer, otherwise the next WriteBrotliData() will fail.
*/
static void CopyInputToRingBuffer(BrotliEncoderState* s, static void CopyInputToRingBuffer(BrotliEncoderState* s,
const size_t input_size, const size_t input_size,
const uint8_t* input_buffer) { const uint8_t* input_buffer) {
@ -714,8 +721,8 @@ static void CopyInputToRingBuffer(BrotliEncoderState* s,
reading new bytes from the input. However, at the last few indexes of reading new bytes from the input. However, at the last few indexes of
the ring buffer, there are not enough bytes to build full-length the ring buffer, there are not enough bytes to build full-length
substrings from. Since the hash table always contains full-length substrings from. Since the hash table always contains full-length
substrings, we erase with dummy 0s here to make sure that those substrings, we erase with dummy zeros here to make sure that those
substrings will contain 0s at the end instead of uninitialized substrings will contain zeros at the end instead of uninitialized
data. data.
Please note that erasing is not necessary (because the Please note that erasing is not necessary (because the
@ -724,21 +731,21 @@ static void CopyInputToRingBuffer(BrotliEncoderState* s,
skip erasing if we have already gone around at least once in skip erasing if we have already gone around at least once in
the ring buffer. the ring buffer.
Only clear during the first round of ringbuffer writes. On Only clear during the first round of ring-buffer writes. On
subsequent rounds data in the ringbuffer would be affected. */ subsequent rounds data in the ring-buffer would be affected. */
if (ringbuffer_->pos_ <= ringbuffer_->mask_) { if (ringbuffer_->pos_ <= ringbuffer_->mask_) {
/* This is the first time when the ring buffer is being written. /* This is the first time when the ring buffer is being written.
We clear 7 bytes just after the bytes that have been copied from We clear 7 bytes just after the bytes that have been copied from
the input buffer. the input buffer.
The ringbuffer has a "tail" that holds a copy of the beginning, The ring-buffer has a "tail" that holds a copy of the beginning,
but only once the ring buffer has been fully written once, i.e., but only once the ring buffer has been fully written once, i.e.,
pos <= mask. For the first time, we need to write values pos <= mask. For the first time, we need to write values
in this tail (where index may be larger than mask), so that in this tail (where index may be larger than mask), so that
we have exactly defined behavior and don't read un-initialized we have exactly defined behavior and don't read uninitialized
memory. Due to performance reasons, hashing reads data using a memory. Due to performance reasons, hashing reads data using a
LOAD64, which can go 7 bytes beyond the bytes written in the LOAD64, which can go 7 bytes beyond the bytes written in the
ringbuffer. */ ring-buffer. */
memset(ringbuffer_->buffer_ + ringbuffer_->pos_, 0, 7); memset(ringbuffer_->buffer_ + ringbuffer_->pos_, 0, 7);
} }
} }
@ -782,6 +789,18 @@ static BROTLI_BOOL UpdateLastProcessedPos(BrotliEncoderState* s) {
return TO_BROTLI_BOOL(wrapped_input_pos < wrapped_last_processed_pos); return TO_BROTLI_BOOL(wrapped_input_pos < wrapped_last_processed_pos);
} }
/*
Processes the accumulated input data and sets |*out_size| to the length of
the new output meta-block, or to zero if no new output meta-block has been
created (in this case the processed input data is buffered internally).
If |*out_size| is positive, |*output| points to the start of the output
data. If |is_last| or |force_flush| is BROTLI_TRUE, an output meta-block is
always created. However, until |is_last| is BROTLI_TRUE encoder may retain up
to 7 bits of the last byte of output. To force encoder to dump the remaining
bits use WriteMetadata() to append an empty meta-data block.
Returns BROTLI_FALSE if the size of the input data is larger than
input_block_size().
*/
static BROTLI_BOOL EncodeData( static BROTLI_BOOL EncodeData(
BrotliEncoderState* s, const BROTLI_BOOL is_last, BrotliEncoderState* s, const BROTLI_BOOL is_last,
const BROTLI_BOOL force_flush, size_t* out_size, uint8_t** output) { const BROTLI_BOOL force_flush, size_t* out_size, uint8_t** output) {
@ -863,7 +882,7 @@ static BROTLI_BOOL EncodeData(
if (newsize > s->cmd_alloc_size_) { if (newsize > s->cmd_alloc_size_) {
Command* new_commands; Command* new_commands;
/* Reserve a bit more memory to allow merging with a next block /* Reserve a bit more memory to allow merging with a next block
without realloc: that would impact speed. */ without reallocation: that would impact speed. */
newsize += (bytes / 4) + 16; newsize += (bytes / 4) + 16;
s->cmd_alloc_size_ = newsize; s->cmd_alloc_size_ = newsize;
new_commands = BROTLI_ALLOC(m, Command, newsize); new_commands = BROTLI_ALLOC(m, Command, newsize);
@ -1076,7 +1095,7 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
will be likely big enough for the whole metablock, so that for most will be likely big enough for the whole metablock, so that for most
inputs we will not have to reallocate in later iterations. We do the inputs we will not have to reallocate in later iterations. We do the
allocation here and not before the loop, because if the input is small, allocation here and not before the loop, because if the input is small,
this will be allocated after the zopfli cost model is freed, so this this will be allocated after the Zopfli cost model is freed, so this
will not increase peak memory usage. will not increase peak memory usage.
TODO: If the first allocation is too small, increase command TODO: If the first allocation is too small, increase command
buffer size exponentially. */ buffer size exponentially. */
@ -1276,7 +1295,7 @@ BROTLI_BOOL BrotliEncoderCompress(
} }
if (quality == 10) { if (quality == 10) {
/* TODO: Implement this direct path for all quality levels. */ /* TODO: Implement this direct path for all quality levels. */
const int lg_win = BROTLI_MIN(int, kBrotliMaxWindowBits, const int lg_win = BROTLI_MIN(int, BROTLI_MAX_WINDOW_BITS,
BROTLI_MAX(int, 16, lgwin)); BROTLI_MAX(int, 16, lgwin));
int ok = BrotliCompressBufferQuality10(lg_win, input_size, input_buffer, int ok = BrotliCompressBufferQuality10(lg_win, input_size, input_buffer,
encoded_size, encoded_buffer); encoded_size, encoded_buffer);
@ -1326,7 +1345,7 @@ static void InjectBytePaddingBlock(BrotliEncoderState* s) {
uint8_t* destination; uint8_t* destination;
s->last_byte_ = 0; s->last_byte_ = 0;
s->last_byte_bits_ = 0; s->last_byte_bits_ = 0;
/* is_last = 0, data_nibbles = 11, reseved = 0, meta_nibbles = 00 */ /* is_last = 0, data_nibbles = 11, reserved = 0, meta_nibbles = 00 */
seal |= 0x6u << seal_bits; seal |= 0x6u << seal_bits;
seal_bits += 6; seal_bits += 6;
/* If we have already created storage, then append to it. /* If we have already created storage, then append to it.
@ -1605,7 +1624,7 @@ BROTLI_BOOL BrotliEncoderCompressStream(
continue; continue;
} }
/* Compress data only when internal outpuf buffer is empty, stream is not /* Compress data only when internal output buffer is empty, stream is not
finished and there is no pending flush request. */ finished and there is no pending flush request. */
if (s->available_out_ == 0 && if (s->available_out_ == 0 &&
s->stream_state_ == BROTLI_STREAM_PROCESSING) { s->stream_state_ == BROTLI_STREAM_PROCESSING) {

16
enc/entropy_encode.c
View File

@ -246,7 +246,7 @@ void BrotliOptimizeHuffmanCountsForRle(size_t length, uint32_t* counts,
size_t limit; size_t limit;
size_t sum; size_t sum;
const size_t streak_limit = 1240; const size_t streak_limit = 1240;
/* Let's make the Huffman code more compatible with rle encoding. */ /* Let's make the Huffman code more compatible with RLE encoding. */
size_t i; size_t i;
for (i = 0; i < length; i++) { for (i = 0; i < length; i++) {
if (counts[i]) { if (counts[i]) {
@ -293,10 +293,10 @@ void BrotliOptimizeHuffmanCountsForRle(size_t length, uint32_t* counts,
} }
} }
/* 2) Let's mark all population counts that already can be encoded /* 2) Let's mark all population counts that already can be encoded
with an rle code. */ with an RLE code. */
memset(good_for_rle, 0, length); memset(good_for_rle, 0, length);
{ {
/* Let's not spoil any of the existing good rle codes. /* Let's not spoil any of the existing good RLE codes.
Mark any seq of 0's that is longer as 5 as a good_for_rle. Mark any seq of 0's that is longer as 5 as a good_for_rle.
Mark any seq of non-0's that is longer as 7 as a good_for_rle. */ Mark any seq of non-0's that is longer as 7 as a good_for_rle. */
uint32_t symbol = counts[0]; uint32_t symbol = counts[0];
@ -319,7 +319,7 @@ void BrotliOptimizeHuffmanCountsForRle(size_t length, uint32_t* counts,
} }
} }
} }
/* 3) Let's replace those population counts that lead to more rle codes. /* 3) Let's replace those population counts that lead to more RLE codes.
Math here is in 24.8 fixed point representation. */ Math here is in 24.8 fixed point representation. */
stride = 0; stride = 0;
limit = 256 * (counts[0] + counts[1] + counts[2]) / 3 + 420; limit = 256 * (counts[0] + counts[1] + counts[2]) / 3 + 420;
@ -420,15 +420,15 @@ void BrotliWriteHuffmanTree(const uint8_t* depth,
} }
} }
/* First gather statistics on if it is a good idea to do rle. */ /* First gather statistics on if it is a good idea to do RLE. */
if (length > 50) { if (length > 50) {
/* Find rle coding for longer codes. /* Find RLE coding for longer codes.
Shorter codes seem not to benefit from rle. */ Shorter codes seem not to benefit from RLE. */
DecideOverRleUse(depth, new_length, DecideOverRleUse(depth, new_length,
&use_rle_for_non_zero, &use_rle_for_zero); &use_rle_for_non_zero, &use_rle_for_zero);
} }
/* Actual rle coding. */ /* Actual RLE coding. */
for (i = 0; i < new_length;) { for (i = 0; i < new_length;) {
const uint8_t value = depth[i]; const uint8_t value = depth[i];
size_t reps = 1; size_t reps = 1;

6
enc/entropy_encode.h
View File

@ -30,7 +30,7 @@ static BROTLI_INLINE void InitHuffmanTree(HuffmanTree* self, uint32_t count,
self->index_right_or_value_ = right; self->index_right_or_value_ = right;
} }
/* Returns 1 is assignment of depths succeded, otherwise 0. */ /* Returns 1 is assignment of depths succeeded, otherwise 0. */
BROTLI_INTERNAL BROTLI_BOOL BrotliSetDepth( BROTLI_INTERNAL BROTLI_BOOL BrotliSetDepth(
int p, HuffmanTree* pool, uint8_t* depth, int max_depth); int p, HuffmanTree* pool, uint8_t* depth, int max_depth);
@ -53,7 +53,7 @@ BROTLI_INTERNAL void BrotliCreateHuffmanTree(const uint32_t *data,
uint8_t *depth); uint8_t *depth);
/* Change the population counts in a way that the consequent /* Change the population counts in a way that the consequent
Huffman tree compression, especially its rle-part will be more Huffman tree compression, especially its RLE-part will be more
likely to compress this data more efficiently. likely to compress this data more efficiently.
length contains the size of the histogram. length contains the size of the histogram.
@ -62,7 +62,7 @@ BROTLI_INTERNAL void BrotliCreateHuffmanTree(const uint32_t *data,
BROTLI_INTERNAL void BrotliOptimizeHuffmanCountsForRle( BROTLI_INTERNAL void BrotliOptimizeHuffmanCountsForRle(
size_t length, uint32_t* counts, uint8_t* good_for_rle); size_t length, uint32_t* counts, uint8_t* good_for_rle);
/* Write a Huffman tree from bit depths into the bitstream representation /* Write a Huffman tree from bit depths into the bit-stream representation
of a Huffman tree. The generated Huffman tree is to be compressed once of a Huffman tree. The generated Huffman tree is to be compressed once
more using a Huffman tree */ more using a Huffman tree */
BROTLI_INTERNAL void BrotliWriteHuffmanTree(const uint8_t* depth, BROTLI_INTERNAL void BrotliWriteHuffmanTree(const uint8_t* depth,

16
enc/hash.h
View File

@ -57,7 +57,7 @@ typedef struct DictionarySearchStatictics {
/* kHashMul32 multiplier has these properties: /* kHashMul32 multiplier has these properties:
* The multiplier must be odd. Otherwise we may lose the highest bit. * The multiplier must be odd. Otherwise we may lose the highest bit.
* No long streaks of 1s or 0s. * No long streaks of ones or zeros.
* There is no effort to ensure that it is a prime, the oddity is enough * There is no effort to ensure that it is a prime, the oddity is enough
for this use. for this use.
* The number has been tuned heuristically against compression benchmarks. */ * The number has been tuned heuristically against compression benchmarks. */
@ -342,16 +342,16 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
const uint32_t key = FN(HashBytes)(&data[cur_ix_masked]); const uint32_t key = FN(HashBytes)(&data[cur_ix_masked]);
size_t prev_ix = self->buckets_[key]; size_t prev_ix = self->buckets_[key];
/* The forest index of the rightmost node of the left subtree of the new /* The forest index of the rightmost node of the left subtree of the new
root, updated as we traverse and reroot the tree of the hash bucket. */ root, updated as we traverse and re-root the tree of the hash bucket. */
size_t node_left = FN(LeftChildIndex)(self, cur_ix); size_t node_left = FN(LeftChildIndex)(self, cur_ix);
/* The forest index of the leftmost node of the right subtree of the new /* The forest index of the leftmost node of the right subtree of the new
root, updated as we traverse and reroot the tree of the hash bucket. */ root, updated as we traverse and re-root the tree of the hash bucket. */
size_t node_right = FN(RightChildIndex)(self, cur_ix); size_t node_right = FN(RightChildIndex)(self, cur_ix);
/* The match length of the rightmost node of the left subtree of the new /* The match length of the rightmost node of the left subtree of the new
root, updated as we traverse and reroot the tree of the hash bucket. */ root, updated as we traverse and re-root the tree of the hash bucket. */
size_t best_len_left = 0; size_t best_len_left = 0;
/* The match length of the leftmost node of the right subtree of the new /* The match length of the leftmost node of the right subtree of the new
root, updated as we traverse and reroot the tree of the hash bucket. */ root, updated as we traverse and re-root the tree of the hash bucket. */
size_t best_len_right = 0; size_t best_len_right = 0;
size_t depth_remaining; size_t depth_remaining;
if (should_reroot_tree) { if (should_reroot_tree) {
@ -511,14 +511,14 @@ static BROTLI_INLINE void FN(StitchToPreviousBlock)(HashToBinaryTree* self,
/* Maximum distance is window size - 16, see section 9.1. of the spec. /* Maximum distance is window size - 16, see section 9.1. of the spec.
Furthermore, we have to make sure that we don't look further back Furthermore, we have to make sure that we don't look further back
from the start of the next block than the window size, otherwise we from the start of the next block than the window size, otherwise we
could access already overwritten areas of the ringbuffer. */ could access already overwritten areas of the ring-buffer. */
const size_t max_backward = const size_t max_backward =
self->window_mask_ - BROTLI_MAX(size_t, self->window_mask_ - BROTLI_MAX(size_t,
BROTLI_WINDOW_GAP - 1, BROTLI_WINDOW_GAP - 1,
position - i); position - i);
/* We know that i + MAX_TREE_COMP_LENGTH <= position + num_bytes, i.e. the /* We know that i + MAX_TREE_COMP_LENGTH <= position + num_bytes, i.e. the
end of the current block and that we have at least end of the current block and that we have at least
MAX_TREE_COMP_LENGTH tail in the ringbuffer. */ MAX_TREE_COMP_LENGTH tail in the ring-buffer. */
FN(StoreAndFindMatches)(self, ringbuffer, i, ringbuffer_mask, FN(StoreAndFindMatches)(self, ringbuffer, i, ringbuffer_mask,
MAX_TREE_COMP_LENGTH, max_backward, NULL, NULL); MAX_TREE_COMP_LENGTH, max_backward, NULL, NULL);
} }
@ -532,7 +532,7 @@ static BROTLI_INLINE void FN(StitchToPreviousBlock)(HashToBinaryTree* self,
/* For BUCKET_SWEEP == 1, enabling the dictionary lookup makes compression /* For BUCKET_SWEEP == 1, enabling the dictionary lookup makes compression
a little faster (0.5% - 1%) and it compresses 0.15% better on small text a little faster (0.5% - 1%) and it compresses 0.15% better on small text
and html inputs. */ and HTML inputs. */
#define HASHER() H2 #define HASHER() H2
#define BUCKET_BITS 16 #define BUCKET_BITS 16

2
enc/hash_forgetful_chain_inc.h
View File

@ -94,7 +94,7 @@ static void FN(Init)(
MemoryManager* m, HashForgetfulChain* self, const uint8_t* data, MemoryManager* m, HashForgetfulChain* self, const uint8_t* data,
const BrotliEncoderParams* params, size_t position, size_t bytes, const BrotliEncoderParams* params, size_t position, size_t bytes,
BROTLI_BOOL is_last) { BROTLI_BOOL is_last) {
/* Choose which init method is faster. /* Choose which initialization method is faster.
Init() is about 100 times faster than InitForData(). */ Init() is about 100 times faster than InitForData(). */
const size_t kMaxBytesForPartialHashInit = BUCKET_SIZE >> 6; const size_t kMaxBytesForPartialHashInit = BUCKET_SIZE >> 6;
BROTLI_UNUSED(m); BROTLI_UNUSED(m);

4
enc/hash_longest_match_inc.h
View File

@ -24,7 +24,7 @@
and the older are forgotten. */ and the older are forgotten. */
#define BLOCK_SIZE (1u << BLOCK_BITS) #define BLOCK_SIZE (1u << BLOCK_BITS)
/* Mask for accessing entries in a block (in a ringbuffer manner). */ /* Mask for accessing entries in a block (in a ring-buffer manner). */
#define BLOCK_MASK ((1 << BLOCK_BITS) - 1) #define BLOCK_MASK ((1 << BLOCK_BITS) - 1)
#define HASH_MAP_SIZE (2 << BUCKET_BITS) #define HASH_MAP_SIZE (2 << BUCKET_BITS)
@ -83,7 +83,7 @@ static void FN(Init)(
MemoryManager* m, HashLongestMatch* self, const uint8_t* data, MemoryManager* m, HashLongestMatch* self, const uint8_t* data,
const BrotliEncoderParams* params, size_t position, size_t bytes, const BrotliEncoderParams* params, size_t position, size_t bytes,
BROTLI_BOOL is_last) { BROTLI_BOOL is_last) {
/* Choose which init method is faster. /* Choose which initialization method is faster.
Init() is about 100 times faster than InitForData(). */ Init() is about 100 times faster than InitForData(). */
const size_t kMaxBytesForPartialHashInit = HASH_MAP_SIZE >> 7; const size_t kMaxBytesForPartialHashInit = HASH_MAP_SIZE >> 7;
BROTLI_UNUSED(m); BROTLI_UNUSED(m);

2
enc/hash_longest_match_quickly_inc.h
View File

@ -72,7 +72,7 @@ static void FN(Init)(
MemoryManager* m, HashLongestMatchQuickly* self, const uint8_t* data, MemoryManager* m, HashLongestMatchQuickly* self, const uint8_t* data,
const BrotliEncoderParams* params, size_t position, size_t bytes, const BrotliEncoderParams* params, size_t position, size_t bytes,
BROTLI_BOOL is_last) { BROTLI_BOOL is_last) {
/* Choose which init method is faster. /* Choose which initialization method is faster.
Init() is about 100 times faster than InitForData(). */ Init() is about 100 times faster than InitForData(). */
const size_t kMaxBytesForPartialHashInit = HASH_MAP_SIZE >> 7; const size_t kMaxBytesForPartialHashInit = HASH_MAP_SIZE >> 7;
BROTLI_UNUSED(m); BROTLI_UNUSED(m);

4
enc/literal_cost.c
View File

@ -57,8 +57,8 @@ static size_t DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask,
static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask, static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
const uint8_t *data, float *cost) { const uint8_t *data, float *cost) {
/* max_utf8 is 0 (normal ascii single byte modeling), /* max_utf8 is 0 (normal ASCII single byte modeling),
1 (for 2-byte utf-8 modeling), or 2 (for 3-byte utf-8 modeling). */ 1 (for 2-byte UTF-8 modeling), or 2 (for 3-byte UTF-8 modeling). */
const size_t max_utf8 = DecideMultiByteStatsLevel(pos, len, mask, data); const size_t max_utf8 = DecideMultiByteStatsLevel(pos, len, mask, data);
size_t histogram[3][256] = { { 0 } }; size_t histogram[3][256] = { { 0 } };
size_t window_half = 495; size_t window_half = 495;

2
enc/literal_cost.h
View File

@ -18,7 +18,7 @@ extern "C" {
#endif #endif
/* Estimates how many bits the literals in the interval [pos, pos + len) in the /* Estimates how many bits the literals in the interval [pos, pos + len) in the
ringbuffer (data, mask) will take entropy coded and writes these estimates ring-buffer (data, mask) will take entropy coded and writes these estimates
to the cost[0..len) array. */ to the cost[0..len) array. */
BROTLI_INTERNAL void BrotliEstimateBitCostsForLiterals( BROTLI_INTERNAL void BrotliEstimateBitCostsForLiterals(
size_t pos, size_t len, size_t mask, const uint8_t *data, float *cost); size_t pos, size_t len, size_t mask, const uint8_t *data, float *cost);

2
enc/metablock.c
View File

@ -257,7 +257,7 @@ static void InitContextBlockSplitter(
*histograms = BROTLI_ALLOC(m, HistogramLiteral, *histograms_size); *histograms = BROTLI_ALLOC(m, HistogramLiteral, *histograms_size);
self->histograms_ = *histograms; self->histograms_ = *histograms;
if (BROTLI_IS_OOM(m)) return; if (BROTLI_IS_OOM(m)) return;
/* Clear only current historgram. */ /* Clear only current histogram. */
ClearHistogramsLiteral(&self->histograms_[0], num_contexts); ClearHistogramsLiteral(&self->histograms_[0], num_contexts);
self->last_histogram_ix_[0] = self->last_histogram_ix_[1] = 0; self->last_histogram_ix_[0] = self->last_histogram_ix_[1] = 0;
} }

21
enc/quality.h
View File

@ -39,7 +39,7 @@ typedef struct BrotliEncoderParams {
int lgblock; int lgblock;
} BrotliEncoderParams; } BrotliEncoderParams;
/* Returns hashtable size for quality levels 0 and 1. */ /* Returns hash-table size for quality levels 0 and 1. */
static BROTLI_INLINE size_t MaxHashTableSize(int quality) { static BROTLI_INLINE size_t MaxHashTableSize(int quality) {
return quality == FAST_ONE_PASS_COMPRESSION_QUALITY ? 1 << 15 : 1 << 17; return quality == FAST_ONE_PASS_COMPRESSION_QUALITY ? 1 << 15 : 1 << 17;
} }
@ -54,7 +54,7 @@ static BROTLI_INLINE size_t MaxZopfliLen(const BrotliEncoderParams* params) {
MAX_ZOPFLI_LEN_QUALITY_11; MAX_ZOPFLI_LEN_QUALITY_11;
} }
/* Number of best candidates to evaluate to expand zopfli chain. */ /* Number of best candidates to evaluate to expand Zopfli chain. */
static BROTLI_INLINE size_t MaxZopfliCandidates( static BROTLI_INLINE size_t MaxZopfliCandidates(
const BrotliEncoderParams* params) { const BrotliEncoderParams* params) {
return params->quality <= 10 ? 1 : 5; return params->quality <= 10 ? 1 : 5;
@ -63,10 +63,10 @@ static BROTLI_INLINE size_t MaxZopfliCandidates(
static BROTLI_INLINE void SanitizeParams(BrotliEncoderParams* params) { static BROTLI_INLINE void SanitizeParams(BrotliEncoderParams* params) {
params->quality = BROTLI_MIN(int, BROTLI_MAX_QUALITY, params->quality = BROTLI_MIN(int, BROTLI_MAX_QUALITY,
BROTLI_MAX(int, BROTLI_MIN_QUALITY, params->quality)); BROTLI_MAX(int, BROTLI_MIN_QUALITY, params->quality));
if (params->lgwin < kBrotliMinWindowBits) { if (params->lgwin < BROTLI_MIN_WINDOW_BITS) {
params->lgwin = kBrotliMinWindowBits; params->lgwin = BROTLI_MIN_WINDOW_BITS;
} else if (params->lgwin > kBrotliMaxWindowBits) { } else if (params->lgwin > BROTLI_MAX_WINDOW_BITS) {
params->lgwin = kBrotliMaxWindowBits; params->lgwin = BROTLI_MAX_WINDOW_BITS;
} }
} }
@ -84,8 +84,8 @@ static BROTLI_INLINE int ComputeLgBlock(const BrotliEncoderParams* params) {
lgblock = BROTLI_MIN(int, 18, params->lgwin); lgblock = BROTLI_MIN(int, 18, params->lgwin);
} }
} else { } else {
lgblock = BROTLI_MIN(int, kBrotliMaxInputBlockBits, lgblock = BROTLI_MIN(int, BROTLI_MAX_INPUT_BLOCK_BITS,
BROTLI_MAX(int, kBrotliMinInputBlockBits, lgblock)); BROTLI_MAX(int, BROTLI_MIN_INPUT_BLOCK_BITS, lgblock));
} }
return lgblock; return lgblock;
} }
@ -94,14 +94,15 @@ static BROTLI_INLINE int ComputeLgBlock(const BrotliEncoderParams* params) {
Allocate at least lgwin + 1 bits for the ring buffer so that the newly Allocate at least lgwin + 1 bits for the ring buffer so that the newly
added block fits there completely and we still get lgwin bits and at least added block fits there completely and we still get lgwin bits and at least
read_block_size_bits + 1 bits because the copy tail length needs to be read_block_size_bits + 1 bits because the copy tail length needs to be
smaller than ringbuffer size. */ smaller than ring-buffer size. */
static BROTLI_INLINE int ComputeRbBits(const BrotliEncoderParams* params) { static BROTLI_INLINE int ComputeRbBits(const BrotliEncoderParams* params) {
return 1 + BROTLI_MAX(int, params->lgwin, params->lgblock); return 1 + BROTLI_MAX(int, params->lgwin, params->lgblock);
} }
static BROTLI_INLINE size_t MaxMetablockSize( static BROTLI_INLINE size_t MaxMetablockSize(
const BrotliEncoderParams* params) { const BrotliEncoderParams* params) {
int bits = BROTLI_MIN(int, ComputeRbBits(params), kBrotliMaxInputBlockBits); int bits =
BROTLI_MIN(int, ComputeRbBits(params), BROTLI_MAX_INPUT_BLOCK_BITS);
return (size_t)1 << bits; return (size_t)1 << bits;
} }

6
enc/ringbuffer.h
View File

@ -30,7 +30,7 @@ extern "C" {
buffer_[-1] == buffer_[(1 << window_bits) - 1] and buffer_[-1] == buffer_[(1 << window_bits) - 1] and
buffer_[-2] == buffer_[(1 << window_bits) - 2]. */ buffer_[-2] == buffer_[(1 << window_bits) - 2]. */
typedef struct RingBuffer { typedef struct RingBuffer {
/* Size of the ringbuffer is (1 << window_bits) + tail_size_. */ /* Size of the ring-buffer is (1 << window_bits) + tail_size_. */
const uint32_t size_; const uint32_t size_;
const uint32_t mask_; const uint32_t mask_;
const uint32_t tail_size_; const uint32_t tail_size_;
@ -42,7 +42,7 @@ typedef struct RingBuffer {
/* The actual ring buffer containing the copy of the last two bytes, the data, /* The actual ring buffer containing the copy of the last two bytes, the data,
and the copy of the beginning as a tail. */ and the copy of the beginning as a tail. */
uint8_t *data_; uint8_t *data_;
/* The start of the ringbuffer. */ /* The start of the ring-buffer. */
uint8_t *buffer_; uint8_t *buffer_;
} RingBuffer; } RingBuffer;
@ -106,7 +106,7 @@ static BROTLI_INLINE void RingBufferWrite(
MemoryManager* m, const uint8_t *bytes, size_t n, RingBuffer* rb) { MemoryManager* m, const uint8_t *bytes, size_t n, RingBuffer* rb) {
if (rb->pos_ == 0 && n < rb->tail_size_) { if (rb->pos_ == 0 && n < rb->tail_size_) {
/* Special case for the first write: to process the first block, we don't /* Special case for the first write: to process the first block, we don't
need to allocate the whole ringbuffer and we don't need the tail need to allocate the whole ring-buffer and we don't need the tail
either. However, we do this memory usage optimization only if the either. However, we do this memory usage optimization only if the
first write is less than the tail size, which is also the input block first write is less than the tail size, which is also the input block
size, otherwise it is likely that other blocks will follow and we size, otherwise it is likely that other blocks will follow and we

2
enc/utf8_util.h
View File

@ -19,7 +19,7 @@ extern "C" {
static const double kMinUTF8Ratio = 0.75; static const double kMinUTF8Ratio = 0.75;
/* Returns 1 if at least min_fraction of the bytes between pos and /* Returns 1 if at least min_fraction of the bytes between pos and
pos + length in the (data, mask) ringbuffer is UTF8-encoded, otherwise pos + length in the (data, mask) ring-buffer is UTF8-encoded, otherwise
returns 0. */ returns 0. */
BROTLI_INTERNAL BROTLI_BOOL BrotliIsMostlyUTF8( BROTLI_INTERNAL BROTLI_BOOL BrotliIsMostlyUTF8(
const uint8_t* data, const size_t pos, const size_t mask, const uint8_t* data, const size_t pos, const size_t mask,

292
include/brotli/decode.h
View File

@ -4,7 +4,10 @@
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
/* API for Brotli decompression */ /**
* @file
* API for Brotli decompression.
*/
#ifndef BROTLI_DEC_DECODE_H_ #ifndef BROTLI_DEC_DECODE_H_
#define BROTLI_DEC_DECODE_H_ #define BROTLI_DEC_DECODE_H_
@ -16,19 +19,47 @@
extern "C" { extern "C" {
#endif #endif
/**
* Opaque structure that holds decoder state.
*
* Allocated and initialized with ::BrotliDecoderCreateInstance.
* Cleaned up and deallocated with ::BrotliDecoderDestroyInstance.
*/
typedef struct BrotliDecoderStateStruct BrotliDecoderState; typedef struct BrotliDecoderStateStruct BrotliDecoderState;
/**
* Result type for ::BrotliDecoderDecompress and
* ::BrotliDecoderDecompressStream functions.
*/
typedef enum { typedef enum {
/* Decoding error, e.g. corrupt input or memory allocation problem */ /** Decoding error, e.g. corrupted input or memory allocation problem. */
BROTLI_DECODER_RESULT_ERROR = 0, BROTLI_DECODER_RESULT_ERROR = 0,
/* Decoding successfully completed */ /** Decoding successfully completed */
BROTLI_DECODER_RESULT_SUCCESS = 1, BROTLI_DECODER_RESULT_SUCCESS = 1,
/* Partially done; should be called again with more input */ /** Partially done; should be called again with more input */
BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT = 2, BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT = 2,
/* Partially done; should be called again with more output */ /** Partially done; should be called again with more output */
BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT = 3 BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT = 3
} BrotliDecoderResult; } BrotliDecoderResult;
/**
* Template that evaluates items of ::BrotliDecoderErrorCode.
*
* Example: @code {.cpp}
* // Log Brotli error code.
* switch (brotliDecoderErrorCode) {
* #define CASE_(PREFIX, NAME, CODE) \
* case BROTLI_DECODER ## PREFIX ## NAME: \
* LOG(INFO) << "error code:" << #NAME; \
* break;
* #define NEWLINE_
* BROTLI_DECODER_ERROR_CODES_LIST(CASE_, NEWLINE_)
* #undef CASE_
* #undef NEWLINE_
* default: LOG(FATAL) << "unknown brotli error code";
* }
* @endcode
*/
#define BROTLI_DECODER_ERROR_CODES_LIST(BROTLI_ERROR_CODE, SEPARATOR) \ #define BROTLI_DECODER_ERROR_CODES_LIST(BROTLI_ERROR_CODE, SEPARATOR) \
BROTLI_ERROR_CODE(_, NO_ERROR, 0) SEPARATOR \ BROTLI_ERROR_CODE(_, NO_ERROR, 0) SEPARATOR \
/* Same as BrotliDecoderResult values */ \ /* Same as BrotliDecoderResult values */ \
@ -65,112 +96,235 @@ typedef enum {
BROTLI_ERROR_CODE(_ERROR_ALLOC_, CONTEXT_MAP, -25) SEPARATOR \ BROTLI_ERROR_CODE(_ERROR_ALLOC_, CONTEXT_MAP, -25) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_ALLOC_, RING_BUFFER_1, -26) SEPARATOR \ BROTLI_ERROR_CODE(_ERROR_ALLOC_, RING_BUFFER_1, -26) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_ALLOC_, RING_BUFFER_2, -27) SEPARATOR \ BROTLI_ERROR_CODE(_ERROR_ALLOC_, RING_BUFFER_2, -27) SEPARATOR \
/* -28..-29 codes are reserved for dynamic ringbuffer allocation */ \ /* -28..-29 codes are reserved for dynamic ring-buffer allocation */ \
BROTLI_ERROR_CODE(_ERROR_ALLOC_, BLOCK_TYPE_TREES, -30) SEPARATOR \ BROTLI_ERROR_CODE(_ERROR_ALLOC_, BLOCK_TYPE_TREES, -30) SEPARATOR \
\ \
/* "Impossible" states */ \ /* "Impossible" states */ \
BROTLI_ERROR_CODE(_ERROR_, UNREACHABLE, -31) BROTLI_ERROR_CODE(_ERROR_, UNREACHABLE, -31)
/**
* Error code for detailed logging / production debugging.
*
* See ::BrotliDecoderGetErrorCode and ::BROTLI_LAST_ERROR_CODE.
*/
typedef enum { typedef enum {
#define BROTLI_COMMA_ , #define BROTLI_COMMA_ ,
#define BROTLI_ERROR_CODE_ENUM_ITEM_(PREFIX, NAME, CODE) \ #define BROTLI_ERROR_CODE_ENUM_ITEM_(PREFIX, NAME, CODE) \
BROTLI_DECODER ## PREFIX ## NAME = CODE BROTLI_DECODER ## PREFIX ## NAME = CODE
BROTLI_DECODER_ERROR_CODES_LIST(BROTLI_ERROR_CODE_ENUM_ITEM_, BROTLI_COMMA_) BROTLI_DECODER_ERROR_CODES_LIST(BROTLI_ERROR_CODE_ENUM_ITEM_, BROTLI_COMMA_)
} BrotliDecoderErrorCode;
#undef BROTLI_ERROR_CODE_ENUM_ITEM_ #undef BROTLI_ERROR_CODE_ENUM_ITEM_
#undef BROTLI_COMMA_ #undef BROTLI_COMMA_
} BrotliDecoderErrorCode;
/**
* The value of the last error code, negative integer.
*
* All other error code values are in the range from ::BROTLI_LAST_ERROR_CODE
* to @c -1. There are also 4 other possible non-error codes @c 0 .. @c 3 in
* ::BrotliDecoderErrorCode enumeration.
*/
#define BROTLI_LAST_ERROR_CODE BROTLI_DECODER_ERROR_UNREACHABLE #define BROTLI_LAST_ERROR_CODE BROTLI_DECODER_ERROR_UNREACHABLE
/* Creates the instance of BrotliDecoderState and initializes it. |alloc_func| /**
and |free_func| MUST be both zero or both non-zero. In the case they are both * Creates an instance of ::BrotliDecoderState and initializes it.
zero, default memory allocators are used. |opaque| is passed to |alloc_func| *
and |free_func| when they are called. */ * @p alloc_func and @p free_func @b MUST be both zero or both non-zero. In the
* case they are both zero, default memory allocators are used. @p opaque is
* passed to @p alloc_func and @p free_func when they are called.
*
* @param alloc_func custom memory allocation function
* @param free_func custom memory fee function
* @param opaque custom memory manager handle
* @returns @c 0 if instance can not be allocated or initialized
* @returns pointer to initialized ::BrotliDecoderState otherwise
*/
BROTLI_DEC_API BrotliDecoderState* BrotliDecoderCreateInstance( BROTLI_DEC_API BrotliDecoderState* BrotliDecoderCreateInstance(
brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque); brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
/* Deinitializes and frees BrotliDecoderState instance. */ /**
* Deinitializes and frees ::BrotliDecoderState instance.
*
* @param state decoder instance to be cleaned up and deallocated
*/
BROTLI_DEC_API void BrotliDecoderDestroyInstance(BrotliDecoderState* state); BROTLI_DEC_API void BrotliDecoderDestroyInstance(BrotliDecoderState* state);
/* Decompresses the data in |encoded_buffer| into |decoded_buffer|, and sets /**
|*decoded_size| to the decompressed length. */ * Performs one-shot memory-to-memory decompression.
*
* Decompresses the data in @p encoded_buffer into @p decoded_buffer, and sets
* @p *decoded_size to the decompressed length.
*
* @param encoded_size size of @p encoded_buffer
* @param encoded_buffer compressed data buffer with at least @p encoded_size
* addressable bytes
* @param[in, out] decoded_size @b in: size of @p decoded_buffer; \n
* @b out: length of decompressed data written to
* @p decoded_buffer
* @param decoded_buffer decompressed data destination buffer
* @returns ::BROTLI_DECODER_RESULT_ERROR if input is corrupted, memory
* allocation failed, or @p decoded_buffer is not large enough;
* @returns ::BROTLI_DECODER_RESULT_SUCCESS otherwise
*/
BROTLI_DEC_API BrotliDecoderResult BrotliDecoderDecompress( BROTLI_DEC_API BrotliDecoderResult BrotliDecoderDecompress(
size_t encoded_size, size_t encoded_size,
const uint8_t encoded_buffer[BROTLI_ARRAY_PARAM(encoded_size)], const uint8_t encoded_buffer[BROTLI_ARRAY_PARAM(encoded_size)],
size_t* decoded_size, size_t* decoded_size,
uint8_t decoded_buffer[BROTLI_ARRAY_PARAM(*decoded_size)]); uint8_t decoded_buffer[BROTLI_ARRAY_PARAM(*decoded_size)]);
/* Decompresses the data. Supports partial input and output. /**
* Decompresses the input stream to the output stream.
Must be called with an allocated input buffer in |*next_in| and an allocated *
output buffer in |*next_out|. The values |*available_in| and |*available_out| * The values @p *available_in and @p *available_out must specify the number of
must specify the allocated size in |*next_in| and |*next_out| respectively; * bytes addressable at @p *next_in and @p *next_out respectively.
when |*available_out| is 0, |next_out| is allowed to be NULL. * When @p *available_out is @c 0, @p next_out is allowed to be @c NULL.
*
After each call, |*available_in| will be decremented by the amount of input * After each call, @p *available_in will be decremented by the amount of input
bytes consumed, and the |*next_in| pointer will be incremented by that * bytes consumed, and the @p *next_in pointer will be incremented by that
amount. Similarly, |*available_out| will be decremented by the amount of * amount. Similarly, @p *available_out will be decremented by the amount of
output bytes written, and the |*next_out| pointer will be incremented by that * output bytes written, and the @p *next_out pointer will be incremented by
amount. |total_out|, if it is not a null-pointer, will be set to the number * that amount.
of bytes decompressed since the last state initialization. *
* @p total_out, if it is not a null-pointer, will be set to the number
Input is never overconsumed, so |next_in| and |available_in| could be passed * of bytes decompressed since the last @p state initialization.
to the next consumer after decoding is complete. */ *
* @note Input is never overconsumed, so @p next_in and @p available_in could be
* passed to the next consumer after decoding is complete.
*
* @param state decoder instance
* @param[in, out] available_in @b in: amount of available input; \n
* @b out: amount of unused input
* @param[in, out] next_in pointer to the next compressed byte
* @param[in, out] available_out @b in: length of output buffer; \n
* @b out: remaining size of output buffer
* @param[in, out] next_out output buffer cursor;
* can be @c NULL if @p available_out is @c 0
* @param[out] total_out number of bytes decompressed so far; can be @c NULL
* @returns ::BROTLI_DECODER_RESULT_ERROR if input is corrupted, memory
* allocation failed, arguments were invalid, etc.;
* use ::BrotliDecoderGetErrorCode to get detailed error code
* @returns ::BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT decoding is blocked until
* more output space is provided
* @returns ::BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT decoding is blocked until
* more input data is provided
* @returns ::BROTLI_DECODER_RESULT_SUCCESS decoding is finished, no more
* input might be consumed and no more output will be produced
*/
BROTLI_DEC_API BrotliDecoderResult BrotliDecoderDecompressStream( BROTLI_DEC_API BrotliDecoderResult BrotliDecoderDecompressStream(
BrotliDecoderState* s, size_t* available_in, const uint8_t** next_in, BrotliDecoderState* state, size_t* available_in, const uint8_t** next_in,
size_t* available_out, uint8_t** next_out, size_t* total_out); size_t* available_out, uint8_t** next_out, size_t* total_out);
/* Fills the new state with a dictionary for LZ77, warming up the ringbuffer, /**
e.g. for custom static dictionaries for data formats. * Prepends LZ77 dictionary.
Not to be confused with the built-in transformable dictionary of Brotli. *
|size| should be less or equal to 2^24 (16MiB), otherwise the dictionary will * Fills the fresh ::BrotliDecoderState with additional data corpus for LZ77
be ignored. The dictionary must exist in memory until decoding is done and * backward references.
is owned by the caller. To use: *
1) Allocate and initialize state with BrotliCreateInstance * @note Not to be confused with the static dictionary (see RFC7932 section 8).
2) Use BrotliDecoderSetCustomDictionary * @warning The dictionary must exist in memory until decoding is done and
3) Use BrotliDecoderDecompressStream * is owned by the caller.
4) Clean up and free state with BrotliDestroyState *
*/ * Workflow:
* -# Allocate and initialize state with ::BrotliDecoderCreateInstance
* -# Invoke ::BrotliDecoderSetCustomDictionary
* -# Use ::BrotliDecoderDecompressStream
* -# Clean up and free state with ::BrotliDecoderDestroyInstance
*
* @param state decoder instance
* @param size length of @p dict; should be less or equal to 2^24 (16MiB),
* otherwise the dictionary will be ignored
* @param dict "dictionary"; @b MUST be the same as used during compression
*/
BROTLI_DEC_API void BrotliDecoderSetCustomDictionary( BROTLI_DEC_API void BrotliDecoderSetCustomDictionary(
BrotliDecoderState* s, size_t size, BrotliDecoderState* state, size_t size,
const uint8_t dict[BROTLI_ARRAY_PARAM(size)]); const uint8_t dict[BROTLI_ARRAY_PARAM(size)]);
/* Returns BROTLI_TRUE, if decoder has some unconsumed output. /**
Otherwise returns BROTLI_FALSE. */ * Checks if decoder has more output.
*
* @param state decoder instance
* @returns ::BROTLI_TRUE, if decoder has some unconsumed output
* @returns ::BROTLI_FALSE otherwise
*/
BROTLI_DEC_API BROTLI_BOOL BrotliDecoderHasMoreOutput( BROTLI_DEC_API BROTLI_BOOL BrotliDecoderHasMoreOutput(
const BrotliDecoderState* s); const BrotliDecoderState* state);
/* Returns pointer to internal output buffer. /**
Set |size| to zero, to request all the continous output produced by decoder * Acquires pointer to internal output buffer.
so far. Otherwise no more than |size| bytes output will be provided. *
After return |size| contains the size of output buffer region available for * This method is used to make language bindings easier and more efficient:
reading. |size| bytes of output are considered consumed for all consecutive * -# push data to ::BrotliDecoderDecompressStream,
calls to the instance methods; returned pointer becomes invalidated as well. * until ::BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT is reported
* -# use ::BrotliDecoderTakeOutput to peek bytes and copy to language-specific
This method is created to make language bindings easier and more efficient: * entity
1. push data to DecompressStream, until "needs more output" is reported *
2. use TakeOutput to peek bytes and copy to language-specific entity * Also this could be useful if there is an output stream that is able to
Also this could be useful if there is an output stream that is able to * consume all the provided data (e.g. when data is saved to file system).
consume all the provided data (e.g. when data is saved to file system). *
* @attention After every call to ::BrotliDecoderTakeOutput @p *size bytes of
* output are considered consumed for all consecutive calls to the
* instance methods; returned pointer becomes invalidated as well.
*
* @note Decoder output is not guaranteed to be contiguous. This means that
* after the size-unrestricted call to ::BrotliDecoderTakeOutput,
* immediate next call to ::BrotliDecoderTakeOutput may return more data.
*
* @param state decoder instance
* @param[in, out] size @b in: number of bytes caller is ready to take, @c 0 if
* any amount could be handled; \n
* @b out: amount of data pointed by returned pointer and
* considered consumed; \n
* out value is never greater than in value, unless it is @c 0
* @returns pointer to output data
*/ */
BROTLI_DEC_API const uint8_t* BrotliDecoderTakeOutput( BROTLI_DEC_API const uint8_t* BrotliDecoderTakeOutput(
BrotliDecoderState* s, size_t* size); BrotliDecoderState* state, size_t* size);
/* Returns BROTLI_TRUE, if decoder has already received some input bytes. /**
Otherwise returns BROTLI_FALSE. */ * Checks if instance has already consumed input.
BROTLI_DEC_API BROTLI_BOOL BrotliDecoderIsUsed(const BrotliDecoderState* s); *
* Instance that returns ::BROTLI_FALSE is considered "fresh" and could be
* reused.
*
* @param state decoder instance
* @returns ::BROTLI_TRUE if decoder has already used some input bytes
* @returns ::BROTLI_FALSE otherwise
*/
BROTLI_DEC_API BROTLI_BOOL BrotliDecoderIsUsed(const BrotliDecoderState* state);
/* Returns BROTLI_TRUE, if decoder is in a state where we reached the end of the /**
input and produced all of the output; returns BROTLI_FALSE otherwise. */ * Checks if decoder instance reached the final state.
BROTLI_BOOL BrotliDecoderIsFinished(const BrotliDecoderState* s); *
* @param state decoder instance
* @returns ::BROTLI_TRUE if decoder is in a state where it reached the end of
* the input and produced all of the output
* @returns ::BROTLI_FALSE otherwise
*/
BROTLI_BOOL BrotliDecoderIsFinished(const BrotliDecoderState* state);
/* Returns detailed error code after BrotliDecoderDecompressStream returns /**
BROTLI_DECODER_RESULT_ERROR. */ * Acquires a detailed error code.
BrotliDecoderErrorCode BrotliDecoderGetErrorCode(const BrotliDecoderState* s); *
* Should be used only after ::BrotliDecoderDecompressStream returns
* ::BROTLI_DECODER_RESULT_ERROR.
*
* See also ::BrotliDecoderErrorString
*
* @param state decoder instance
* @returns last saved error code
*/
BrotliDecoderErrorCode BrotliDecoderGetErrorCode(
const BrotliDecoderState* state);
/**
* Converts error code to a c-string.
*/
BROTLI_DEC_API const char* BrotliDecoderErrorString(BrotliDecoderErrorCode c); BROTLI_DEC_API const char* BrotliDecoderErrorString(BrotliDecoderErrorCode c);
/* Decoder version. Look at BROTLI_VERSION for more information. */ /**
* Gets a decoder library version.
*
* Look at BROTLI_VERSION for more information.
*/
BROTLI_DEC_API uint32_t BrotliDecoderVersion(void); BROTLI_DEC_API uint32_t BrotliDecoderVersion(void);
#if defined(__cplusplus) || defined(c_plusplus) #if defined(__cplusplus) || defined(c_plusplus)

472
include/brotli/encode.h
View File

@ -4,7 +4,10 @@
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
/* API for Brotli compression. */ /**
* @file
* API for Brotli compression.
*/
#ifndef BROTLI_ENC_ENCODE_H_ #ifndef BROTLI_ENC_ENCODE_H_
#define BROTLI_ENC_ENCODE_H_ #define BROTLI_ENC_ENCODE_H_
@ -16,212 +19,405 @@
extern "C" { extern "C" {
#endif #endif
static const int kBrotliMinWindowBits = 10; /** Minimal value for ::BROTLI_PARAM_LGWIN parameter. */
static const int kBrotliMaxWindowBits = 24; /* == BROTLI_MAX_DISTANCE_BITS */ #define BROTLI_MIN_WINDOW_BITS 10
static const int kBrotliMinInputBlockBits = 16; /**
static const int kBrotliMaxInputBlockBits = 24; * Maximal value for ::BROTLI_PARAM_LGWIN parameter.
*
* @note equal to @c BROTLI_MAX_DISTANCE_BITS constant.
*/
#define BROTLI_MAX_WINDOW_BITS 24
/** Minimal value for ::BROTLI_PARAM_LGBLOCK parameter. */
#define BROTLI_MIN_INPUT_BLOCK_BITS 16
/** Maximal value for ::BROTLI_PARAM_LGBLOCK parameter. */
#define BROTLI_MAX_INPUT_BLOCK_BITS 24
/** Minimal value for ::BROTLI_PARAM_QUALITY parameter. */
#define BROTLI_MIN_QUALITY 0 #define BROTLI_MIN_QUALITY 0
/** Maximal value for ::BROTLI_PARAM_QUALITY parameter. */
#define BROTLI_MAX_QUALITY 11 #define BROTLI_MAX_QUALITY 11
BROTLI_DEPRECATED static const int kBrotliMinWindowBits =
BROTLI_MIN_WINDOW_BITS;
BROTLI_DEPRECATED static const int kBrotliMaxWindowBits =
BROTLI_MAX_WINDOW_BITS;
/** Options for ::BROTLI_PARAM_MODE parameter. */
typedef enum BrotliEncoderMode { typedef enum BrotliEncoderMode {
/* Default compression mode. The compressor does not know anything in /**
advance about the properties of the input. */ * Default compression mode.
*
* In this mode compressor does not know anything in advance about the
* properties of the input.
*/
BROTLI_MODE_GENERIC = 0, BROTLI_MODE_GENERIC = 0,
/* Compression mode for UTF-8 format text input. */ /** Compression mode for UTF-8 formated text input. */
BROTLI_MODE_TEXT = 1, BROTLI_MODE_TEXT = 1,
/* Compression mode used in WOFF 2.0. */ /** Compression mode used in WOFF 2.0. */
BROTLI_MODE_FONT = 2 BROTLI_MODE_FONT = 2
} BrotliEncoderMode; } BrotliEncoderMode;
/** Default value for ::BROTLI_PARAM_QUALITY parameter. */
#define BROTLI_DEFAULT_QUALITY 11 #define BROTLI_DEFAULT_QUALITY 11
/** Default value for ::BROTLI_PARAM_LGWIN parameter. */
#define BROTLI_DEFAULT_WINDOW 22 #define BROTLI_DEFAULT_WINDOW 22
/** Default value for ::BROTLI_PARAM_MODE parameter. */
#define BROTLI_DEFAULT_MODE BROTLI_MODE_GENERIC #define BROTLI_DEFAULT_MODE BROTLI_MODE_GENERIC
/** Operations that can be performed by streaming encoder. */
typedef enum BrotliEncoderOperation { typedef enum BrotliEncoderOperation {
/**
* Process input.
*
* Encoder may postpone producing output, until it has processed enough input.
*/
BROTLI_OPERATION_PROCESS = 0, BROTLI_OPERATION_PROCESS = 0,
/* Request output stream to flush. Performed when input stream is depleted /**
and there is enough space in output stream. */ * Produce output for all processed input.
*
* Actual flush is performed when input stream is depleted and there is enough
* space in output stream. This means that client should repeat
* ::BROTLI_OPERATION_FLUSH operation until @p available_in becomes @c 0, and
* ::BrotliEncoderHasMoreOutput returns ::BROTLI_FALSE.
*
* @warning Until flush is complete, client @b SHOULD @b NOT swap,
* reduce or extend input stream.
*
* When flush is complete, output data will be sufficient for decoder to
* reproduce all the given input.
*/
BROTLI_OPERATION_FLUSH = 1, BROTLI_OPERATION_FLUSH = 1,
/* Request output stream to finish. Performed when input stream is depleted /**
and there is enough space in output stream. */ * Finalize the stream.
*
* Actual finalization is performed when input stream is depleted and there is
* enough space in output stream. This means that client should repeat
* ::BROTLI_OPERATION_FLUSH operation until @p available_in becomes @c 0, and
* ::BrotliEncoderHasMoreOutput returns ::BROTLI_FALSE.
*
* @warning Until finalization is complete, client @b SHOULD @b NOT swap,
* reduce or extend input stream.
*
* Helper function ::BrotliEncoderIsFinished checks if stream is finalized and
* output fully dumped.
*
* Adding more input data to finalized stream is impossible.
*/
BROTLI_OPERATION_FINISH = 2, BROTLI_OPERATION_FINISH = 2,
/* Emits metadata block to stream. Stream is soft-flushed before metadata /**
block is emitted. CAUTION: when operation is started, length of the input * Emit metadata block to stream.
buffer is interpreted as length of a metadata block; changing operation, *
expanding or truncating input before metadata block is completely emitted * Metadata is opaque to Brotli: neither encoder, nor decoder processes this
will cause an error; metadata block must not be greater than 16MiB. */ * data or relies on it. It may be used to pass some extra information from
* encoder client to decoder client without interfering with main data stream.
*
* @note Encoder may emit empty metadata blocks internally, to pad encoded
* stream to byte boundary.
*
* @warning Until emitting metadata is complete client @b SHOULD @b NOT swap,
* reduce or extend input stream.
*
* @warning The whole content of input buffer is considered to be the content
* of metadata block. Do @b NOT @e append metadata to input stream,
* before it is depleted with other operations.
*
* Stream is soft-flushed before metadata block is emitted. Metadata block
* @b MUST be no longer than than 16MiB.
*/
BROTLI_OPERATION_EMIT_METADATA = 3 BROTLI_OPERATION_EMIT_METADATA = 3
} BrotliEncoderOperation; } BrotliEncoderOperation;
/** Options to be used with ::BrotliEncoderSetParameter. */
typedef enum BrotliEncoderParameter { typedef enum BrotliEncoderParameter {
/**
* Tune encoder for specific input.
*
* ::BrotliEncoderMode enumerates all available values.
*/
BROTLI_PARAM_MODE = 0, BROTLI_PARAM_MODE = 0,
/* Controls the compression-speed vs compression-density tradeoffs. The higher /**
the quality, the slower the compression. Range is 0 to 11. */ * The main compression speed-density lever.
*
* The higher the quality, the slower the compression. Range is
* from ::BROTLI_MIN_QUALITY to ::BROTLI_MAX_QUALITY.
*/
BROTLI_PARAM_QUALITY = 1, BROTLI_PARAM_QUALITY = 1,
/* Base 2 logarithm of the sliding window size. Range is 10 to 24. */ /**
* Recommended sliding LZ77 window size.
*
* Encoder may reduce this value, e.g. if input is much smaller than
* window size.
*
* Window size is `(1 << value) - 16`.
*
* Range is from ::BROTLI_MIN_WINDOW_BITS to ::BROTLI_MAX_WINDOW_BITS.
*/
BROTLI_PARAM_LGWIN = 2, BROTLI_PARAM_LGWIN = 2,
/* Base 2 logarithm of the maximum input block size. Range is 16 to 24. /**
If set to 0, the value will be set based on the quality. */ * Recommended input block size.
*
* Encoder may reduce this value, e.g. if input is much smaller than input
* block size.
*
* Range is from ::BROTLI_MIN_INPUT_BLOCK_BITS to
* ::BROTLI_MAX_INPUT_BLOCK_BITS.
*
* @note Bigger input block size allows better compression, but consumes more
* memory. \n The rough formula of memory used for temporary input
* storage is `3 << lgBlock`.
*/
BROTLI_PARAM_LGBLOCK = 3 BROTLI_PARAM_LGBLOCK = 3
} BrotliEncoderParameter; } BrotliEncoderParameter;
/* A state can not be reused for multiple brotli streams. */ /**
* Opaque structure that holds encoder state.
*
* Allocated and initialized with ::BrotliEncoderCreateInstance.
* Cleaned up and deallocated with ::BrotliEncoderDestroyInstance.
*/
typedef struct BrotliEncoderStateStruct BrotliEncoderState; typedef struct BrotliEncoderStateStruct BrotliEncoderState;
/* Sets the specified parameter to the given encoder instance. /**
Returns BROTLI_FALSE if parameter is unrecognized, or value is invalid. * Sets the specified parameter to the given encoder instance.
Returns BROTLI_FALSE if value of parameter can not be changed at current *
encoder state (e.g. when encoding is started, window size might be already * @param state encoder instance
encoded and therefore it is impossible to change it). * @param param parameter to set
Returns BROTLI_TRUE if value is accepted. CAUTION: invalid values might be * @param value new parameter value
accepted in case they would not break encoding process. */ * @returns ::BROTLI_FALSE if parameter is unrecognized, or value is invalid
* @returns ::BROTLI_FALSE if value of parameter can not be changed at current
* encoder state (e.g. when encoding is started, window size might be
* already encoded and therefore it is impossible to change it)
* @returns ::BROTLI_TRUE if value is accepted
* @warning invalid values might be accepted in case they would not break
* encoding process.
*/
BROTLI_ENC_API BROTLI_BOOL BrotliEncoderSetParameter( BROTLI_ENC_API BROTLI_BOOL BrotliEncoderSetParameter(
BrotliEncoderState* state, BrotliEncoderParameter p, uint32_t value); BrotliEncoderState* state, BrotliEncoderParameter param, uint32_t value);
/* Creates the instance of BrotliEncoderState and initializes it. /**
|alloc_func| and |free_func| MUST be both zero or both non-zero. In the case * Creates an instance of ::BrotliEncoderState and initializes it.
they are both zero, default memory allocators are used. |opaque| is passed to *
|alloc_func| and |free_func| when they are called. */ * @p alloc_func and @p free_func @b MUST be both zero or both non-zero. In the
* case they are both zero, default memory allocators are used. @p opaque is
* passed to @p alloc_func and @p free_func when they are called.
*
* @param alloc_func custom memory allocation function
* @param free_func custom memory fee function
* @param opaque custom memory manager handle
* @returns @c 0 if instance can not be allocated or initialized
* @returns pointer to initialized ::BrotliEncoderState otherwise
*/
BROTLI_ENC_API BrotliEncoderState* BrotliEncoderCreateInstance( BROTLI_ENC_API BrotliEncoderState* BrotliEncoderCreateInstance(
brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque); brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
/* Deinitializes and frees BrotliEncoderState instance. */ /**
* Deinitializes and frees ::BrotliEncoderState instance.
*
* @param state decoder instance to be cleaned up and deallocated
*/
BROTLI_ENC_API void BrotliEncoderDestroyInstance(BrotliEncoderState* state); BROTLI_ENC_API void BrotliEncoderDestroyInstance(BrotliEncoderState* state);
/* The maximum input size that can be processed at once. */ /** @deprecated Calculates maximum input size that can be processed at once. */
BROTLI_DEPRECATED BROTLI_ENC_API size_t BrotliEncoderInputBlockSize( BROTLI_DEPRECATED BROTLI_ENC_API size_t BrotliEncoderInputBlockSize(
BrotliEncoderState* state); BrotliEncoderState* state);
/* Copies the given input data to the internal ring buffer of the compressor. /** @deprecated Copies the given input data to the internal ring buffer. */
No processing of the data occurs at this time and this function can be
called multiple times before calling WriteBrotliData() to process the
accumulated input. At most input_block_size() bytes of input data can be
copied to the ring buffer, otherwise the next WriteBrotliData() will fail.
*/
BROTLI_DEPRECATED BROTLI_ENC_API void BrotliEncoderCopyInputToRingBuffer( BROTLI_DEPRECATED BROTLI_ENC_API void BrotliEncoderCopyInputToRingBuffer(
BrotliEncoderState* state, const size_t input_size, BrotliEncoderState* state, const size_t input_size,
const uint8_t* input_buffer); const uint8_t* input_buffer);
/* Processes the accumulated input data and sets |*out_size| to the length of /** @deprecated Processes the accumulated input. */
the new output meta-block, or to zero if no new output meta-block has been
created (in this case the processed input data is buffered internally).
If |*out_size| is positive, |*output| points to the start of the output
data. If |is_last| or |force_flush| is BROTLI_TRUE, an output meta-block is
always created. However, until |is_last| is BROTLI_TRUE encoder may retain up
to 7 bits of the last byte of output. To force encoder to dump the remaining
bits use WriteMetadata() to append an empty meta-data block.
Returns BROTLI_FALSE if the size of the input data is larger than
input_block_size(). */
BROTLI_DEPRECATED BROTLI_ENC_API BROTLI_BOOL BrotliEncoderWriteData( BROTLI_DEPRECATED BROTLI_ENC_API BROTLI_BOOL BrotliEncoderWriteData(
BrotliEncoderState* state, const BROTLI_BOOL is_last, BrotliEncoderState* state, const BROTLI_BOOL is_last,
const BROTLI_BOOL force_flush, size_t* out_size, uint8_t** output); const BROTLI_BOOL force_flush, size_t* out_size, uint8_t** output);
/* Fills the new state with a dictionary for LZ77, warming up the ringbuffer, /**
e.g. for custom static dictionaries for data formats. * Prepends imaginary LZ77 dictionary.
Not to be confused with the built-in transformable dictionary of Brotli. *
To decode, use BrotliDecoderSetCustomDictionary() of the decoder with the * Fills the fresh ::BrotliEncoderState with additional data corpus for LZ77
same dictionary. */ * backward references.
*
* @note Not to be confused with the static dictionary (see RFC7932 section 8).
*
* Workflow:
* -# Allocate and initialize state with ::BrotliEncoderCreateInstance
* -# Set ::BROTLI_PARAM_LGWIN parameter
* -# Invoke ::BrotliEncoderSetCustomDictionary
* -# Use ::BrotliEncoderCompressStream
* -# Clean up and free state with ::BrotliEncoderDestroyInstance
*
* @param state encoder instance
* @param size length of @p dict; at most "window size" bytes are used
* @param dict "dictionary"; @b MUST use same dictionary during decompression
*/
BROTLI_ENC_API void BrotliEncoderSetCustomDictionary( BROTLI_ENC_API void BrotliEncoderSetCustomDictionary(
BrotliEncoderState* state, size_t size, BrotliEncoderState* state, size_t size,
const uint8_t dict[BROTLI_ARRAY_PARAM(size)]); const uint8_t dict[BROTLI_ARRAY_PARAM(size)]);
/* Returns buffer size that is large enough to contain BrotliEncoderCompress /**
output for any input. * Calculates the output size bound for the given @p input_size.
CAUTION: result is not applicable to BrotliEncoderCompressStream output, *
because every "flush" adds extra overhead bytes, and some encoder settings * @warning Result is not applicable to ::BrotliEncoderCompressStream output,
(e.g. quality 0 and 1) might imply a "soft flush" after every chunk of input. * because every "flush" adds extra overhead bytes, and some encoder
Returns 0 if result does not fit size_t. */ * settings (e.g. quality @c 0 and @c 1) might imply a "soft flush"
* after every chunk of input.
*
* @param input_size size of projected input
* @returns @c 0 if result does not fit @c size_t
*/
BROTLI_ENC_API size_t BrotliEncoderMaxCompressedSize(size_t input_size); BROTLI_ENC_API size_t BrotliEncoderMaxCompressedSize(size_t input_size);
/* Compresses the data in |input_buffer| into |encoded_buffer|, and sets /**
|*encoded_size| to the compressed length. * Performs one-shot memory-to-memory compression.
BROTLI_DEFAULT_QUALITY, BROTLI_DEFAULT_WINDOW and BROTLI_DEFAULT_MODE should *
be used as |quality|, |lgwin| and |mode| if there are no specific * Compresses the data in @p input_buffer into @p encoded_buffer, and sets
requirements to encoder speed and compression ratio. * @p *encoded_size to the compressed length.
If compression fails, |*encoded_size| is set to 0. *
If BrotliEncoderMaxCompressedSize(|input_size|) is not zero, then * @note If ::BrotliEncoderMaxCompressedSize(@p input_size) returns non-zero
|*encoded_size| is never set to the bigger value. * value, then output is guaranteed to be no longer than that.
Returns BROTLI_FALSE if there was an error and BROTLI_TRUE otherwise. */ *
* @param quality quality parameter value, e.g. ::BROTLI_DEFAULT_QUALITY
* @param lgwin lgwin parameter value, e.g. ::BROTLI_DEFAULT_WINDOW
* @param mode mode parameter value, e.g. ::BROTLI_DEFAULT_MODE
* @param input_size size of @p input_buffer
* @param input_buffer input data buffer with at least @p input_size
* addressable bytes
* @param[in, out] encoded_size @b in: size of @p encoded_buffer; \n
* @b out: length of compressed data written to
* @p encoded_buffer, or @c 0 if compression fails
* @param encoded_buffer compressed data destination buffer
* @returns ::BROTLI_FALSE in case of compression error
* @returns ::BROTLI_FALSE if output buffer is too small
* @returns ::BROTLI_TRUE otherwise
*/
BROTLI_ENC_API BROTLI_BOOL BrotliEncoderCompress( BROTLI_ENC_API BROTLI_BOOL BrotliEncoderCompress(
int quality, int lgwin, BrotliEncoderMode mode, size_t input_size, int quality, int lgwin, BrotliEncoderMode mode, size_t input_size,
const uint8_t input_buffer[BROTLI_ARRAY_PARAM(input_size)], const uint8_t input_buffer[BROTLI_ARRAY_PARAM(input_size)],
size_t* encoded_size, size_t* encoded_size,
uint8_t encoded_buffer[BROTLI_ARRAY_PARAM(*encoded_size)]); uint8_t encoded_buffer[BROTLI_ARRAY_PARAM(*encoded_size)]);
/* Progressively compress input stream and push produced bytes to output stream. /**
Internally workflow consists of 3 tasks: * Compresses input stream to output stream.
* (optional) copy input data to internal buffer *
* actually compress data and (optionally) store it to internal buffer * The values @p *available_in and @p *available_out must specify the number of
* (optional) copy compressed bytes from internal buffer to output stream * bytes addressable at @p *next_in and @p *next_out respectively.
Whenever all 3 tasks can't move forward anymore, or error occurs, this * When @p *available_out is @c 0, @p next_out is allowed to be @c NULL.
method returns. *
* After each call, @p *available_in will be decremented by the amount of input
|available_in| and |next_in| represent input stream; when X bytes of input * bytes consumed, and the @p *next_in pointer will be incremented by that
are consumed, X is subtracted from |available_in| and added to |next_in|. * amount. Similarly, @p *available_out will be decremented by the amount of
|available_out| and |next_out| represent output stream; when Y bytes are * output bytes written, and the @p *next_out pointer will be incremented by
pushed to output, Y is subtracted from |available_out| and added to * that amount.
|next_out|. |total_out|, if it is not a null-pointer, is assigned to the *
total amount of bytes pushed by the instance of encoder to output. * @p total_out, if it is not a null-pointer, will be set to the number
* of bytes decompressed since the last @p state initialization.
|op| is used to perform flush or finish the stream. *
*
Flushing the stream means forcing encoding of all input passed to encoder and *
completing the current output block, so it could be fully decoded by stream * Internally workflow consists of 3 tasks:
decoder. To perform flush |op| must be set to BROTLI_OPERATION_FLUSH. Under * -# (optionally) copy input data to internal buffer
some circumstances (e.g. lack of output stream capacity) this operation would * -# actually compress data and (optionally) store it to internal buffer
require several calls to BrotliEncoderCompressStream. The method must be * -# (optionally) copy compressed bytes from internal buffer to output stream
called again until both input stream is depleted and encoder has no more * Whenever all 3 tasks can't move forward anymore, or error occurs, this
output (see BrotliEncoderHasMoreOutput) after the method is called. * method returns the control flow to caller.
*
Finishing the stream means encoding of all input passed to encoder and * @p op is used to perform flush, finish the stream, or inject metadata block.
adding specific "final" marks, so stream decoder could determine that stream * See ::BrotliEncoderOperation for more information.
is complete. To perform finish |op| must be set to BROTLI_OPERATION_FINISH. *
Under some circumstances (e.g. lack of output stream capacity) this operation * Flushing the stream means forcing encoding of all input passed to encoder and
would require several calls to BrotliEncoderCompressStream. The method must * completing the current output block, so it could be fully decoded by stream
be called again until both input stream is depleted and encoder has no more * decoder. To perform flush set @p op to ::BROTLI_OPERATION_FLUSH.
output (see BrotliEncoderHasMoreOutput) after the method is called. * Under some circumstances (e.g. lack of output stream capacity) this operation
* would require several calls to ::BrotliEncoderCompressStream. The method must
WARNING: when flushing and finishing, |op| should not change until operation * be called again until both input stream is depleted and encoder has no more
is complete; input stream should not be refilled as well. * output (see ::BrotliEncoderHasMoreOutput) after the method is called.
*
Returns BROTLI_FALSE if there was an error and BROTLI_TRUE otherwise. * Finishing the stream means encoding of all input passed to encoder and
*/ * adding specific "final" marks, so stream decoder could determine that stream
* is complete. To perform finish set @p op to ::BROTLI_OPERATION_FINISH.
* Under some circumstances (e.g. lack of output stream capacity) this operation
* would require several calls to ::BrotliEncoderCompressStream. The method must
* be called again until both input stream is depleted and encoder has no more
* output (see ::BrotliEncoderHasMoreOutput) after the method is called.
*
* @warning When flushing and finishing, @p op should not change until operation
* is complete; input stream should not be swapped, reduced or
* extended as well.
*
* @param state encoder instance
* @param op requested operation
* @param[in, out] available_in @b in: amount of available input; \n
* @b out: amount of unused input
* @param[in, out] next_in pointer to the next input byte
* @param[in, out] available_out @b in: length of output buffer; \n
* @b out: remaining size of output buffer
* @param[in, out] next_out compressed output buffer cursor;
* can be @c NULL if @p available_out is @c 0
* @param[out] total_out number of bytes produced so far; can be @c NULL
* @returns ::BROTLI_FALSE if there was an error
* @returns ::BROTLI_TRUE otherwise
*/
BROTLI_ENC_API BROTLI_BOOL BrotliEncoderCompressStream( BROTLI_ENC_API BROTLI_BOOL BrotliEncoderCompressStream(
BrotliEncoderState* s, BrotliEncoderOperation op, size_t* available_in, BrotliEncoderState* state, BrotliEncoderOperation op, size_t* available_in,
const uint8_t** next_in, size_t* available_out, uint8_t** next_out, const uint8_t** next_in, size_t* available_out, uint8_t** next_out,
size_t* total_out); size_t* total_out);
/* Check if encoder is in "finished" state, i.e. no more input is acceptable and /**
no more output will be produced. * Checks if encoder instance reached the final state.
Works only with BrotliEncoderCompressStream workflow. *
Returns BROTLI_TRUE if stream is finished and BROTLI_FALSE otherwise. */ * @param state encoder instance
BROTLI_ENC_API BROTLI_BOOL BrotliEncoderIsFinished(BrotliEncoderState* s); * @returns ::BROTLI_TRUE if encoder is in a state where it reached the end of
* the input and produced all of the output
* @returns ::BROTLI_FALSE otherwise
*/
BROTLI_ENC_API BROTLI_BOOL BrotliEncoderIsFinished(BrotliEncoderState* state);
/* Check if encoder has more output bytes in internal buffer. /**
Works only with BrotliEncoderCompressStream workflow. * Checks if encoder has more output.
Returns BROTLI_TRUE if has more output (in internal buffer) and BROTLI_FALSE *
otherwise. */ * @param state encoder instance
BROTLI_ENC_API BROTLI_BOOL BrotliEncoderHasMoreOutput(BrotliEncoderState* s); * @returns ::BROTLI_TRUE, if encoder has some unconsumed output
* @returns ::BROTLI_FALSE otherwise
*/
BROTLI_ENC_API BROTLI_BOOL BrotliEncoderHasMoreOutput(
BrotliEncoderState* state);
/* Returns pointer to internal output buffer. /**
Set |size| to zero, to request all the continous output produced by encoder * Acquires pointer to internal output buffer.
so far. Otherwise no more than |size| bytes output will be provided. *
After return |size| contains the size of output buffer region available for * This method is used to make language bindings easier and more efficient:
reading. |size| bytes of output are considered consumed for all consecutive * -# push data to ::BrotliEncoderCompressStream,
calls to the instance methods; returned pointer becomes invalidated as well. * until ::BrotliEncoderHasMoreOutput returns BROTL_TRUE
* -# use ::BrotliEncoderTakeOutput to peek bytes and copy to language-specific
This method is created to make language bindings easier and more efficient: * entity
1. push input data to CompressStream, until HasMoreOutput becomes true *
2. use TakeOutput to peek bytes and copy to language-specific entity * Also this could be useful if there is an output stream that is able to
Also this could be useful if there is an output stream that is able to * consume all the provided data (e.g. when data is saved to file system).
consume all the provided data (e.g. when data is saved to file system). *
* @attention After every call to ::BrotliEncoderTakeOutput @p *size bytes of
* output are considered consumed for all consecutive calls to the
* instance methods; returned pointer becomes invalidated as well.
*
* @note Encoder output is not guaranteed to be contiguous. This means that
* after the size-unrestricted call to ::BrotliEncoderTakeOutput,
* immediate next call to ::BrotliEncoderTakeOutput may return more data.
*
* @param state encoder instance
* @param[in, out] size @b in: number of bytes caller is ready to take, @c 0 if
* any amount could be handled; \n
* @b out: amount of data pointed by returned pointer and
* considered consumed; \n
* out value is never greater than in value, unless it is @c 0
* @returns pointer to output data
*/ */
BROTLI_ENC_API const uint8_t* BrotliEncoderTakeOutput( BROTLI_ENC_API const uint8_t* BrotliEncoderTakeOutput(
BrotliEncoderState* s, size_t* size); BrotliEncoderState* state, size_t* size);
/* Encoder version. Look at BROTLI_VERSION for more information. */ /**
* Gets an encoder library version.
*
* Look at BROTLI_VERSION for more information.
*/
BROTLI_ENC_API uint32_t BrotliEncoderVersion(void); BROTLI_ENC_API uint32_t BrotliEncoderVersion(void);
#if defined(__cplusplus) || defined(c_plusplus) #if defined(__cplusplus) || defined(c_plusplus)

57
include/brotli/types.h
View File

@ -4,7 +4,10 @@
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
/* Common types */ /**
* @file
* Common types used in decoder and encoder API.
*/
#ifndef BROTLI_COMMON_TYPES_H_ #ifndef BROTLI_COMMON_TYPES_H_
#define BROTLI_COMMON_TYPES_H_ #define BROTLI_COMMON_TYPES_H_
@ -24,15 +27,31 @@ typedef __int64 int64_t;
#include <stdint.h> #include <stdint.h>
#endif /* defined(_MSC_VER) && (_MSC_VER < 1600) */ #endif /* defined(_MSC_VER) && (_MSC_VER < 1600) */
/* BROTLI_BOOL is a portable "bool" replacement. For input parameters it is /**
preferrable either use BROTLI_TRUE and BROTLI_FALSE macros, or convert * A portable @c bool replacement.
boolean expression with TO_BROTLI_BOOL macros. *
Return values should not be tested for equality with "true", "false", * ::BROTLI_BOOL is a "documentation" type: actually it is @c int, but in API it
"BROTLI_TRUE", "BROTLI_FALSE", but rather be evaluated, for example: * denotes a type, whose only values are ::BROTLI_TRUE and ::BROTLI_FALSE.
`if (foo(enc) && !bar(dec) { bool x = !!baz(enc); }` */ *
* ::BROTLI_BOOL values passed to Brotli should either be ::BROTLI_TRUE or
* ::BROTLI_FALSE, or be a result of ::TO_BROTLI_BOOL macros.
*
* ::BROTLI_BOOL values returned by Brotli should not be tested for equality
* with @c true, @c false, ::BROTLI_TRUE, ::BROTLI_FALSE, but rather should be
* evaluated, for example: @code{.cpp}
* if (SomeBrotliFunction(encoder, BROTLI_TRUE) &&
* !OtherBrotliFunction(decoder, BROTLI_FALSE)) {
* bool x = !!YetAnotherBrotliFunction(encoder, TO_BROLTI_BOOL(2 * 2 == 4));
* DoSometing(x);
* }
* @endcode
*/
#define BROTLI_BOOL int #define BROTLI_BOOL int
/** Portable @c true replacement. */
#define BROTLI_TRUE 1 #define BROTLI_TRUE 1
/** Portable @c false replacement. */
#define BROTLI_FALSE 0 #define BROTLI_FALSE 0
/** @c bool to ::BROTLI_BOOL conversion macros. */
#define TO_BROTLI_BOOL(X) (!!(X) ? BROTLI_TRUE : BROTLI_FALSE) #define TO_BROTLI_BOOL(X) (!!(X) ? BROTLI_TRUE : BROTLI_FALSE)
#define BROTLI_MAKE_UINT64_T(high, low) ((((uint64_t)(high)) << 32) | low) #define BROTLI_MAKE_UINT64_T(high, low) ((((uint64_t)(high)) << 32) | low)
@ -40,15 +59,25 @@ typedef __int64 int64_t;
#define BROTLI_UINT32_MAX (~((uint32_t)0)) #define BROTLI_UINT32_MAX (~((uint32_t)0))
#define BROTLI_SIZE_MAX (~((size_t)0)) #define BROTLI_SIZE_MAX (~((size_t)0))
/* Allocating function pointer. Function MUST return 0 in the case of failure. /**
Otherwise it MUST return a valid pointer to a memory region of at least * Allocating function pointer type.
size length. Neither items nor size are allowed to be 0. *
opaque argument is a pointer provided by client and could be used to bind * @param opaque custom memory manager handle provided by client
function to specific object (memory pool). */ * @param size requested memory region size; can not be @c 0
* @returns @c 0 in the case of failure
* @returns a valid pointer to a memory region of at least @p size bytes
* long otherwise
*/
typedef void* (*brotli_alloc_func)(void* opaque, size_t size); typedef void* (*brotli_alloc_func)(void* opaque, size_t size);
/* Deallocating function pointer. Function SHOULD be no-op in the case the /**
address is 0. */ * Deallocating function pointer type.
*
* This function @b SHOULD do nothing if @p address is @c 0.
*
* @param opaque custom memory manager handle provided by client
* @param address memory region pointer returned by ::brotli_alloc_func, or @c 0
*/
typedef void (*brotli_free_func)(void* opaque, void* address); typedef void (*brotli_free_func)(void* opaque, void* address);
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \ #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \

14
java/integration/BUILD
View File

@ -14,10 +14,22 @@ java_binary(
) )
java_test( java_test(
name = "bundle_checker_test", name = "bundle_checker_data_test",
args = ["java/integration/test_data.zip"], args = ["java/integration/test_data.zip"],
data = ["test_data.zip"], data = ["test_data.zip"],
main_class = "org.brotli.integration.BundleChecker", main_class = "org.brotli.integration.BundleChecker",
use_testrunner = 0, use_testrunner = 0,
runtime_deps = [":bundle_checker_lib"], runtime_deps = [":bundle_checker_lib"],
) )
java_test(
name = "bundle_checker_fuzz_test",
args = [
"-s",
"java/integration/fuzz_data.zip"
],
data = ["fuzz_data.zip"],
main_class = "org.brotli.integration.BundleChecker",
use_testrunner = 0,
runtime_deps = [":bundle_checker_lib"],
)

36
java/integration/BundleChecker.java
View File

@ -18,7 +18,7 @@ import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream; import java.util.zip.ZipInputStream;
/** /**
* Decompress files and checks thier checksums. * Decompress files and (optionally) checks their checksums.
* *
* <p> File are read from ZIP archive passed as an array of bytes. Multiple checkers negotiate about * <p> File are read from ZIP archive passed as an array of bytes. Multiple checkers negotiate about
* task distribution via shared AtomicInteger counter. * task distribution via shared AtomicInteger counter.
@ -28,10 +28,15 @@ import java.util.zip.ZipInputStream;
public class BundleChecker implements Runnable { public class BundleChecker implements Runnable {
final AtomicInteger nextJob; final AtomicInteger nextJob;
final InputStream input; final InputStream input;
final boolean sanityCheck;
public BundleChecker(InputStream input, AtomicInteger nextJob) { /**
* @param sanityCheck do not calculate checksum and ignore {@link IOException}.
*/
public BundleChecker(InputStream input, AtomicInteger nextJob, boolean sanityCheck) {
this.input = input; this.input = input;
this.nextJob = nextJob; this.nextJob = nextJob;
this.sanityCheck = sanityCheck;
} }
/** ECMA CRC64 polynomial. */ /** ECMA CRC64 polynomial. */
@ -93,11 +98,18 @@ public class BundleChecker implements Runnable {
continue; continue;
} }
entryName = entry.getName(); entryName = entry.getName();
String entryCrcString = entryName.substring(0, entryName.indexOf('.')); int dotIndex = entryName.indexOf('.');
String entryCrcString = (dotIndex == -1) ? entryName : entryName.substring(0, dotIndex);
long entryCrc = new BigInteger(entryCrcString, 16).longValue(); long entryCrc = new BigInteger(entryCrcString, 16).longValue();
if (entryCrc != decompressAndCalculateCrc(zis)) { try {
if (entryCrc != decompressAndCalculateCrc(zis) && !sanityCheck) {
throw new RuntimeException("CRC mismatch"); throw new RuntimeException("CRC mismatch");
} }
} catch (IOException iox) {
if (!sanityCheck) {
throw new RuntimeException("Decompression failed", iox);
}
}
zis.closeEntry(); zis.closeEntry();
entryName = ""; entryName = "";
jobIndex = nextJob.getAndIncrement(); jobIndex = nextJob.getAndIncrement();
@ -110,11 +122,19 @@ public class BundleChecker implements Runnable {
} }
public static void main(String[] args) throws FileNotFoundException { public static void main(String[] args) throws FileNotFoundException {
if (args.length == 0) { int argsOffset = 0;
throw new RuntimeException("Usage: BundleChecker <fileX.zip> ..."); boolean sanityCheck = false;
if (args.length != 0) {
if (args[0].equals("-s")) {
sanityCheck = true;
argsOffset = 1;
} }
for (int i = 0; i < args.length; ++i) { }
new BundleChecker(new FileInputStream(args[i]), new AtomicInteger(0)).run(); if (args.length == argsOffset) {
throw new RuntimeException("Usage: BundleChecker [-s] <fileX.zip> ...");
}
for (int i = argsOffset; i < args.length; ++i) {
new BundleChecker(new FileInputStream(args[i]), new AtomicInteger(0), sanityCheck).run();
} }
} }
} }

BIN
java/integration/fuzz_data.zip Executable file
View File

Binary file not shown.

16
java/integration/pom.xml
View File

@ -39,6 +39,7 @@
<artifactId>exec-maven-plugin</artifactId> <artifactId>exec-maven-plugin</artifactId>
<executions> <executions>
<execution> <execution>
<id>data</id>
<phase>test</phase> <phase>test</phase>
<goals> <goals>
<goal>java</goal> <goal>java</goal>
@ -51,6 +52,21 @@
</arguments> </arguments>
</configuration> </configuration>
</execution> </execution>
<execution>
<id>fuzz</id>
<phase>test</phase>
<goals>
<goal>java</goal>
</goals>
<configuration>
<executable>java</executable>
<mainClass>org.brotli.integration.BundleChecker</mainClass>
<arguments>
<argument>-s</argument>
<argument>fuzz_data.zip</argument>
</arguments>
</configuration>
</execution>
</executions> </executions>
</plugin> </plugin>
</plugins> </plugins>