mirror of
https://github.com/google/brotli.git
synced 2024-11-28 14:11:07 +00:00
Update docs and add more java tests (#463)
* doxygenize and update API documentation * fix spelling * add "fuzz" corpus for java decoder to improve coverage * use upper-case-snake names for dictionary constant definitions * use `LDFLAGS` in conventional `Makefile`
This commit is contained in:
parent
a260b6ba73
commit
e9b278ac6e
2
Makefile
2
Makefile
@ -23,7 +23,7 @@ $(DIRS):
|
||||
mkdir -p $@
|
||||
|
||||
$(EXECUTABLE): $(OBJECTS)
|
||||
$(CC) $(OBJECTS) -lm -o $(BINDIR)/$(EXECUTABLE)
|
||||
$(CC) $(LDFLAGS) $(OBJECTS) -lm -o $(BINDIR)/$(EXECUTABLE)
|
||||
|
||||
lib: $(LIBOBJECTS)
|
||||
rm -f $(LIB_A)
|
||||
|
@ -20,8 +20,8 @@ BROTLI_COMMON_API extern const uint8_t kBrotliDictionary[122784];
|
||||
BROTLI_COMMON_API extern const uint32_t kBrotliDictionaryOffsetsByLength[25];
|
||||
BROTLI_COMMON_API extern const uint8_t kBrotliDictionarySizeBitsByLength[25];
|
||||
|
||||
#define kBrotliMinDictionaryWordLength 4
|
||||
#define kBrotliMaxDictionaryWordLength 24
|
||||
#define BROTLI_MIN_DICTIONARY_WORD_LENGTH 4
|
||||
#define BROTLI_MAX_DICTIONARY_WORD_LENGTH 24
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
|
@ -24,7 +24,7 @@ BROTLI_BOOL BrotliWarmupBitReader(BrotliBitReader* const br) {
|
||||
size_t aligned_read_mask = (sizeof(br->val_) >> 1) - 1;
|
||||
/* Fixing alignment after unaligned BrotliFillWindow would result accumulator
|
||||
overflow. If unalignment is caused by BrotliSafeReadBits, then there is
|
||||
enough space in accumulator to fix aligment. */
|
||||
enough space in accumulator to fix alignment. */
|
||||
if (!BROTLI_ALIGNED_READ) {
|
||||
aligned_read_mask = 0;
|
||||
}
|
||||
|
@ -55,7 +55,7 @@ typedef struct {
|
||||
size_t avail_in;
|
||||
} BrotliBitReaderState;
|
||||
|
||||
/* Initializes the bitreader fields. */
|
||||
/* Initializes the BrotliBitReader fields. */
|
||||
BROTLI_INTERNAL void BrotliInitBitReader(BrotliBitReader* const br);
|
||||
|
||||
/* Ensures that accumulator is not empty. May consume one byte of input.
|
||||
@ -91,8 +91,8 @@ static BROTLI_INLINE size_t BrotliGetRemainingBytes(BrotliBitReader* br) {
|
||||
return br->avail_in + (BrotliGetAvailableBits(br) >> 3);
|
||||
}
|
||||
|
||||
/* Checks if there is at least num bytes left in the input ringbuffer (excluding
|
||||
the bits remaining in br->val_). */
|
||||
/* Checks if there is at least |num| bytes left in the input ring-buffer
|
||||
(excluding the bits remaining in br->val_). */
|
||||
static BROTLI_INLINE BROTLI_BOOL BrotliCheckInputAmount(
|
||||
BrotliBitReader* const br, size_t num) {
|
||||
return TO_BROTLI_BOOL(br->avail_in >= num);
|
||||
@ -157,7 +157,7 @@ static BROTLI_INLINE uint64_t BrotliLoad64LE(const uint8_t* in) {
|
||||
/* Guarantees that there are at least n_bits + 1 bits in accumulator.
|
||||
Precondition: accumulator contains at least 1 bit.
|
||||
n_bits should be in the range [1..24] for regular build. For portable
|
||||
non-64-bit little endian build only 16 bits are safe to request. */
|
||||
non-64-bit little-endian build only 16 bits are safe to request. */
|
||||
static BROTLI_INLINE void BrotliFillBitWindow(
|
||||
BrotliBitReader* const br, uint32_t n_bits) {
|
||||
#if (BROTLI_64_BITS)
|
||||
@ -207,7 +207,7 @@ static BROTLI_INLINE void BrotliFillBitWindow(
|
||||
#endif
|
||||
}
|
||||
|
||||
/* Mosltly like BrotliFillBitWindow, but guarantees only 16 bits and reads no
|
||||
/* Mostly like BrotliFillBitWindow, but guarantees only 16 bits and reads no
|
||||
more than BROTLI_SHORT_FILL_BIT_WINDOW_READ bytes of input. */
|
||||
static BROTLI_INLINE void BrotliFillBitWindow16(BrotliBitReader* const br) {
|
||||
BrotliFillBitWindow(br, 17);
|
||||
@ -231,7 +231,7 @@ static BROTLI_INLINE BROTLI_BOOL BrotliPullByte(BrotliBitReader* const br) {
|
||||
}
|
||||
|
||||
/* Returns currently available bits.
|
||||
The number of valid bits could be calclulated by BrotliGetAvailableBits. */
|
||||
The number of valid bits could be calculated by BrotliGetAvailableBits. */
|
||||
static BROTLI_INLINE reg_t BrotliGetBitsUnmasked(BrotliBitReader* const br) {
|
||||
return br->val_ >> br->bit_pos_;
|
||||
}
|
||||
@ -244,7 +244,7 @@ static BROTLI_INLINE uint32_t BrotliGet16BitsUnmasked(
|
||||
return (uint32_t)BrotliGetBitsUnmasked(br);
|
||||
}
|
||||
|
||||
/* Returns the specified number of bits from br without advancing bit pos. */
|
||||
/* Returns the specified number of bits from |br| without advancing bit pos. */
|
||||
static BROTLI_INLINE uint32_t BrotliGetBits(
|
||||
BrotliBitReader* const br, uint32_t n_bits) {
|
||||
BrotliFillBitWindow(br, n_bits);
|
||||
@ -283,7 +283,7 @@ static BROTLI_INLINE void BrotliBitReaderUnload(BrotliBitReader* br) {
|
||||
br->bit_pos_ += unused_bits;
|
||||
}
|
||||
|
||||
/* Reads the specified number of bits from br and advances the bit pos.
|
||||
/* Reads the specified number of bits from |br| and advances the bit pos.
|
||||
Precondition: accumulator MUST contain at least n_bits. */
|
||||
static BROTLI_INLINE void BrotliTakeBits(
|
||||
BrotliBitReader* const br, uint32_t n_bits, uint32_t* val) {
|
||||
@ -293,7 +293,7 @@ static BROTLI_INLINE void BrotliTakeBits(
|
||||
BrotliDropBits(br, n_bits);
|
||||
}
|
||||
|
||||
/* Reads the specified number of bits from br and advances the bit pos.
|
||||
/* Reads the specified number of bits from |br| and advances the bit pos.
|
||||
Assumes that there is enough input to perform BrotliFillBitWindow. */
|
||||
static BROTLI_INLINE uint32_t BrotliReadBits(
|
||||
BrotliBitReader* const br, uint32_t n_bits) {
|
||||
|
42
dec/decode.c
42
dec/decode.c
@ -454,8 +454,8 @@ static BrotliDecoderErrorCode ReadSimpleHuffmanSymbols(
|
||||
/* Process single decoded symbol code length:
|
||||
A) reset the repeat variable
|
||||
B) remember code length (if it is not 0)
|
||||
C) extend corredponding index-chain
|
||||
D) reduce the huffman space
|
||||
C) extend corresponding index-chain
|
||||
D) reduce the Huffman space
|
||||
E) update the histogram
|
||||
*/
|
||||
static BROTLI_INLINE void ProcessSingleCodeLength(uint32_t code_len,
|
||||
@ -479,7 +479,7 @@ static BROTLI_INLINE void ProcessSingleCodeLength(uint32_t code_len,
|
||||
value is not BROTLI_REPEAT_PREVIOUS_CODE_LENGTH, then it is a new
|
||||
symbol-skip
|
||||
B) Update repeat variable
|
||||
C) Check if operation is feasible (fits alphapet)
|
||||
C) Check if operation is feasible (fits alphabet)
|
||||
D) For each symbol do the same operations as in ProcessSingleCodeLength
|
||||
|
||||
PRECONDITION: code_len == BROTLI_REPEAT_PREVIOUS_CODE_LENGTH or
|
||||
@ -949,7 +949,7 @@ static BrotliDecoderErrorCode DecodeContextMap(uint32_t context_map_size,
|
||||
if (!BrotliSafeGetBits(br, 5, &bits)) {
|
||||
return BROTLI_DECODER_NEEDS_MORE_INPUT;
|
||||
}
|
||||
if ((bits & 1) != 0) { /* Use RLE for zeroes. */
|
||||
if ((bits & 1) != 0) { /* Use RLE for zeros. */
|
||||
s->max_run_length_prefix = (bits >> 1) + 1;
|
||||
BrotliDropBits(br, 5);
|
||||
} else {
|
||||
@ -1031,7 +1031,7 @@ rleCode:
|
||||
}
|
||||
}
|
||||
|
||||
/* Decodes a command or literal and updates block type ringbuffer.
|
||||
/* Decodes a command or literal and updates block type ring-buffer.
|
||||
Reads 3..54 bits. */
|
||||
static BROTLI_INLINE BROTLI_BOOL DecodeBlockTypeAndLength(
|
||||
int safe, BrotliDecoderState* s, int tree_type) {
|
||||
@ -1176,7 +1176,7 @@ static size_t UnwrittenBytes(const BrotliDecoderState* s, BROTLI_BOOL wrap) {
|
||||
|
||||
/* Dumps output.
|
||||
Returns BROTLI_DECODER_NEEDS_MORE_OUTPUT only if there is more output to push
|
||||
and either ringbuffer is as big as window size, or |force| is true.
|
||||
and either ring-buffer is as big as window size, or |force| is true.
|
||||
*/
|
||||
static BrotliDecoderErrorCode BROTLI_NOINLINE WriteRingBuffer(
|
||||
BrotliDecoderState* s, size_t* available_out, uint8_t** next_out,
|
||||
@ -1228,15 +1228,15 @@ static void BROTLI_NOINLINE WrapRingBuffer(BrotliDecoderState* s) {
|
||||
}
|
||||
}
|
||||
|
||||
/* Allocates ringbuffer.
|
||||
/* Allocates ring-buffer.
|
||||
|
||||
s->ringbuffer_size MUST be updated by BrotliCalculateRingBufferSize before
|
||||
this function is called.
|
||||
|
||||
Last two bytes of ringbuffer are initialized to 0, so context calculation
|
||||
Last two bytes of ring-buffer are initialized to 0, so context calculation
|
||||
could be done uniformly for the first two and all other positions.
|
||||
|
||||
Custom dictionary, if any, is copied to the end of ringbuffer.
|
||||
Custom dictionary, if any, is copied to the end of ring-buffer.
|
||||
*/
|
||||
static BROTLI_BOOL BROTLI_NOINLINE BrotliEnsureRingBuffer(
|
||||
BrotliDecoderState* s) {
|
||||
@ -1296,7 +1296,7 @@ static BrotliDecoderErrorCode BROTLI_NOINLINE CopyUncompressedBlockToOutput(
|
||||
if (s->pos + nbytes > s->ringbuffer_size) {
|
||||
nbytes = s->ringbuffer_size - s->pos;
|
||||
}
|
||||
/* Copy remaining bytes from s->br.buf_ to ringbuffer. */
|
||||
/* Copy remaining bytes from s->br.buf_ to ring-buffer. */
|
||||
BrotliCopyBytes(&s->ringbuffer[s->pos], &s->br, (size_t)nbytes);
|
||||
s->pos += nbytes;
|
||||
s->meta_block_remaining_len -= nbytes;
|
||||
@ -1343,7 +1343,7 @@ static void BROTLI_NOINLINE BrotliCalculateRingBufferSize(
|
||||
int min_size = s->ringbuffer_size ? s->ringbuffer_size : 1024;
|
||||
int output_size;
|
||||
|
||||
/* If maxumum is already reached, no further extention is reuired. */
|
||||
/* If maximum is already reached, no further extension is retired. */
|
||||
if (s->ringbuffer_size == window_size) {
|
||||
return;
|
||||
}
|
||||
@ -1354,7 +1354,7 @@ static void BROTLI_NOINLINE BrotliCalculateRingBufferSize(
|
||||
}
|
||||
|
||||
if (!s->ringbuffer) {
|
||||
/* Custom dictionanry counts as a "virtual" output. */
|
||||
/* Custom dictionary counts as a "virtual" output. */
|
||||
output_size = s->custom_dict_size;
|
||||
} else {
|
||||
output_size = s->pos;
|
||||
@ -1724,8 +1724,8 @@ postReadDistance:
|
||||
/* Apply copy of LZ77 back-reference, or static dictionary reference if
|
||||
the distance is larger than the max LZ77 distance */
|
||||
if (s->distance_code > s->max_distance) {
|
||||
if (i >= kBrotliMinDictionaryWordLength &&
|
||||
i <= kBrotliMaxDictionaryWordLength) {
|
||||
if (i >= BROTLI_MIN_DICTIONARY_WORD_LENGTH &&
|
||||
i <= BROTLI_MAX_DICTIONARY_WORD_LENGTH) {
|
||||
int offset = (int)kBrotliDictionaryOffsetsByLength[i];
|
||||
int word_id = s->distance_code - s->max_distance - 1;
|
||||
uint32_t shift = kBrotliDictionarySizeBitsByLength[i];
|
||||
@ -1771,9 +1771,9 @@ postReadDistance:
|
||||
s->dist_rb[s->dist_rb_idx & 3] = s->distance_code;
|
||||
++s->dist_rb_idx;
|
||||
s->meta_block_remaining_len -= i;
|
||||
/* There are 32+ bytes of slack in the ringbuffer allocation.
|
||||
/* There are 32+ bytes of slack in the ring-buffer allocation.
|
||||
Also, we have 16 short codes, that make these 16 bytes irrelevant
|
||||
in the ringbuffer. Let's copy over them as a first guess.
|
||||
in the ring-buffer. Let's copy over them as a first guess.
|
||||
*/
|
||||
memmove16(copy_dst, copy_src);
|
||||
if (src_end > pos && dst_end > src_start) {
|
||||
@ -1866,7 +1866,7 @@ BrotliDecoderResult BrotliDecoderDecompress(
|
||||
/* Invariant: input stream is never overconsumed:
|
||||
* invalid input implies that the whole stream is invalid -> any amount of
|
||||
input could be read and discarded
|
||||
* when result is "needs more input", then at leat one more byte is REQUIRED
|
||||
* when result is "needs more input", then at least one more byte is REQUIRED
|
||||
to complete decoding; all input data MUST be consumed by decoder, so
|
||||
client could swap the input buffer
|
||||
* when result is "needs more output" decoder MUST ensure that it doesn't
|
||||
@ -1899,12 +1899,12 @@ BrotliDecoderResult BrotliDecoderDecompressStream(
|
||||
for (;;) {
|
||||
if (result != BROTLI_DECODER_SUCCESS) { /* Error, needs more input/output */
|
||||
if (result == BROTLI_DECODER_NEEDS_MORE_INPUT) {
|
||||
if (s->ringbuffer != 0) { /* Proactively push output. */
|
||||
if (s->ringbuffer != 0) { /* Pro-actively push output. */
|
||||
WriteRingBuffer(s, available_out, next_out, total_out, BROTLI_TRUE);
|
||||
}
|
||||
if (s->buffer_length != 0) { /* Used with internal buffer. */
|
||||
if (br->avail_in == 0) { /* Successfully finished read transaction. */
|
||||
/* Accamulator contains less than 8 bits, because internal buffer
|
||||
/* Accumulator contains less than 8 bits, because internal buffer
|
||||
is expanded byte-by-byte until it is enough to complete read. */
|
||||
s->buffer_length = 0;
|
||||
/* Switch to input stream and restart. */
|
||||
@ -1949,8 +1949,8 @@ BrotliDecoderResult BrotliDecoderDecompressStream(
|
||||
s->buffer_length = 0;
|
||||
} else {
|
||||
/* Using input stream in last iteration. When decoder switches to input
|
||||
stream it has less than 8 bits in accamulator, so it is safe to
|
||||
return unused accamulator bits there. */
|
||||
stream it has less than 8 bits in accumulator, so it is safe to
|
||||
return unused accumulator bits there. */
|
||||
BrotliBitReaderUnload(br);
|
||||
*available_in = br->avail_in;
|
||||
*next_in = br->next_in;
|
||||
|
@ -60,7 +60,7 @@
|
||||
#define BROTLI_ALIGNED_READ (!!1)
|
||||
#elif defined(BROTLI_TARGET_X86) || defined(BROTLI_TARGET_X64) || \
|
||||
defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8)
|
||||
/* Allow unaligned read only for whitelisted CPUs. */
|
||||
/* Allow unaligned read only for white-listed CPUs. */
|
||||
#define BROTLI_ALIGNED_READ (!!0)
|
||||
#else
|
||||
#define BROTLI_ALIGNED_READ (!!1)
|
||||
|
@ -160,7 +160,7 @@ struct BrotliDecoderStateStruct {
|
||||
int distance_code;
|
||||
|
||||
/* For partial write operations */
|
||||
size_t rb_roundtrips; /* How many times we went around the ringbuffer */
|
||||
size_t rb_roundtrips; /* How many times we went around the ring-buffer */
|
||||
size_t partial_pos_out; /* How much output to the user in total */
|
||||
|
||||
/* For ReadHuffmanCode */
|
||||
|
@ -247,7 +247,7 @@ static int ToUpperCase(uint8_t* p) {
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
/* An overly simplified uppercasing model for utf-8. */
|
||||
/* An overly simplified uppercasing model for UTF-8. */
|
||||
if (p[0] < 0xe0) {
|
||||
p[1] ^= 32;
|
||||
return 2;
|
||||
|
@ -45,7 +45,7 @@ typedef struct ZopfliNode {
|
||||
|
||||
/* This union holds information used by dynamic-programming. During forward
|
||||
pass |cost| it used to store the goal function. When node is processed its
|
||||
|cost| is invalidated in favor of |shortcut|. On path backtracing pass
|
||||
|cost| is invalidated in favor of |shortcut|. On path back-tracing pass
|
||||
|next| is assigned the offset to next node on the path. */
|
||||
union {
|
||||
/* Smallest cost to get to this byte from the beginning, as found so far. */
|
||||
@ -64,7 +64,7 @@ BROTLI_INTERNAL void BrotliInitZopfliNodes(ZopfliNode* array, size_t length);
|
||||
position + num_bytes.
|
||||
|
||||
On return, path->size() is the number of commands found and path[i] is the
|
||||
length of the ith command (copy length plus insert length).
|
||||
length of the i-th command (copy length plus insert length).
|
||||
Note that the sum of the lengths of all commands can be less than num_bytes.
|
||||
|
||||
On return, the nodes[0..num_bytes] array will have the following
|
||||
|
@ -80,7 +80,7 @@ static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
|
||||
position + 2 * sr.len + random_heuristics_window_size;
|
||||
max_distance = BROTLI_MIN(size_t, position, max_backward_limit);
|
||||
{
|
||||
/* The first 16 codes are special shortcodes,
|
||||
/* The first 16 codes are special short-codes,
|
||||
and the minimum offset is 1. */
|
||||
size_t distance_code =
|
||||
ComputeDistanceCode(sr.distance, max_distance, dist_cache);
|
||||
|
@ -61,7 +61,7 @@ static void FN(RefineEntropyCodes)(const DataType* data, size_t length,
|
||||
}
|
||||
}
|
||||
|
||||
/* Assigns a block id from the range [0, vec.size()) to each data element
|
||||
/* Assigns a block id from the range [0, num_histograms) to each data element
|
||||
in data[0..length) and fills in block_id[0..length) with the assigned values.
|
||||
Returns the number of blocks, i.e. one plus the number of block switches. */
|
||||
static size_t FN(FindBlocks)(const DataType* data, const size_t length,
|
||||
|
@ -82,7 +82,7 @@ static BROTLI_INLINE size_t NextBlockTypeCode(
|
||||
return type_code;
|
||||
}
|
||||
|
||||
/* nibblesbits represents the 2 bits to encode MNIBBLES (0-3)
|
||||
/* |nibblesbits| represents the 2 bits to encode MNIBBLES (0-3)
|
||||
REQUIRES: length > 0
|
||||
REQUIRES: length <= (1 << 24) */
|
||||
static void BrotliEncodeMlen(size_t length, uint64_t* bits,
|
||||
@ -349,7 +349,7 @@ void BrotliStoreHuffmanTree(const uint8_t* depths, size_t num,
|
||||
code_length_bitdepth[code] = 0;
|
||||
}
|
||||
|
||||
/* Store the real huffman tree now. */
|
||||
/* Store the real Huffman tree now. */
|
||||
BrotliStoreHuffmanTreeToBitMask(huffman_tree_size,
|
||||
huffman_tree,
|
||||
huffman_tree_extra_bits,
|
||||
@ -554,7 +554,7 @@ void BrotliBuildAndStoreHuffmanTreeFast(MemoryManager* m,
|
||||
/* Complex Huffman Tree */
|
||||
StoreStaticCodeLengthCode(storage_ix, storage);
|
||||
|
||||
/* Actual rle coding. */
|
||||
/* Actual RLE coding. */
|
||||
for (i = 0; i < length;) {
|
||||
const uint8_t value = depth[i];
|
||||
size_t reps = 1;
|
||||
|
@ -36,7 +36,7 @@ extern "C" {
|
||||
|
||||
/* kHashMul32 multiplier has these properties:
|
||||
* The multiplier must be odd. Otherwise we may lose the highest bit.
|
||||
* No long streaks of 1s or 0s.
|
||||
* No long streaks of ones or zeros.
|
||||
* There is no effort to ensure that it is a prime, the oddity is enough
|
||||
for this use.
|
||||
* The number has been tuned heuristically against compression benchmarks. */
|
||||
@ -136,7 +136,7 @@ static void BuildAndStoreCommandPrefixCode(const uint32_t histogram[128],
|
||||
|
||||
BrotliCreateHuffmanTree(histogram, 64, 15, tree, depth);
|
||||
BrotliCreateHuffmanTree(&histogram[64], 64, 14, tree, &depth[64]);
|
||||
/* We have to jump through a few hoopes here in order to compute
|
||||
/* We have to jump through a few hoops here in order to compute
|
||||
the command bits because the symbols are in a different order than in
|
||||
the full alphabet. This looks complicated, but having the symbols
|
||||
in this order in the command bits saves a few branches in the Emit*
|
||||
@ -526,7 +526,7 @@ static BROTLI_INLINE void BrotliCompressFragmentFastImpl(
|
||||
and doesn't bother looking for matches everywhere.
|
||||
|
||||
The "skip" variable keeps track of how many bytes there are since the
|
||||
last match; dividing it by 32 (ie. right-shifting by five) gives the
|
||||
last match; dividing it by 32 (i.e. right-shifting by five) gives the
|
||||
number of bytes to move ahead for each iteration. */
|
||||
uint32_t skip = 32;
|
||||
|
||||
@ -563,7 +563,7 @@ trawl:
|
||||
if (ip - candidate > MAX_DISTANCE) goto trawl;
|
||||
|
||||
/* Step 2: Emit the found match together with the literal bytes from
|
||||
"next_emit" to the bit stream, and then see if we can find a next macth
|
||||
"next_emit" to the bit stream, and then see if we can find a next match
|
||||
immediately afterwards. Repeat until we find no match for the input
|
||||
without emitting some literal bytes. */
|
||||
|
||||
|
@ -35,7 +35,7 @@ extern "C" {
|
||||
|
||||
/* kHashMul32 multiplier has these properties:
|
||||
* The multiplier must be odd. Otherwise we may lose the highest bit.
|
||||
* No long streaks of 1s or 0s.
|
||||
* No long streaks of ones or zeros.
|
||||
* There is no effort to ensure that it is a prime, the oddity is enough
|
||||
for this use.
|
||||
* The number has been tuned heuristically against compression benchmarks. */
|
||||
@ -75,7 +75,7 @@ static void BuildAndStoreCommandPrefixCode(
|
||||
uint16_t cmd_bits[64];
|
||||
BrotliCreateHuffmanTree(histogram, 64, 15, tree, depth);
|
||||
BrotliCreateHuffmanTree(&histogram[64], 64, 14, tree, &depth[64]);
|
||||
/* We have to jump through a few hoopes here in order to compute
|
||||
/* We have to jump through a few hoops here in order to compute
|
||||
the command bits because the symbols are in a different order than in
|
||||
the full alphabet. This looks complicated, but having the symbols
|
||||
in this order in the command bits saves a few branches in the Emit*
|
||||
@ -314,7 +314,7 @@ trawl:
|
||||
if (ip - candidate > MAX_DISTANCE) goto trawl;
|
||||
|
||||
/* Step 2: Emit the found match together with the literal bytes from
|
||||
"next_emit", and then see if we can find a next macth immediately
|
||||
"next_emit", and then see if we can find a next match immediately
|
||||
afterwards. Repeat until we find no match for the input
|
||||
without emitting some literal bytes. */
|
||||
|
||||
|
57
enc/encode.c
57
enc/encode.c
@ -45,7 +45,7 @@ typedef enum BrotliEncoderStreamState {
|
||||
BROTLI_STREAM_FLUSH_REQUESTED = 1,
|
||||
/* Last metablock was produced; no more input is acceptable. */
|
||||
BROTLI_STREAM_FINISHED = 2,
|
||||
/* Flushing compressed block and writing metada block header. */
|
||||
/* Flushing compressed block and writing meta-data block header. */
|
||||
BROTLI_STREAM_METADATA_HEAD = 3,
|
||||
/* Writing metadata block body. */
|
||||
BROTLI_STREAM_METADATA_BODY = 4
|
||||
@ -133,7 +133,7 @@ BROTLI_BOOL BrotliEncoderSetParameter(
|
||||
BrotliEncoderState* state, BrotliEncoderParameter p, uint32_t value) {
|
||||
/* Changing parameters on the fly is not implemented yet. */
|
||||
if (state->is_initialized_) return BROTLI_FALSE;
|
||||
/* TODO: Validate/clamp params here. */
|
||||
/* TODO: Validate/clamp parameters here. */
|
||||
switch (p) {
|
||||
case BROTLI_PARAM_MODE:
|
||||
state->params.mode = (BrotliEncoderMode)value;
|
||||
@ -175,13 +175,13 @@ static void RecomputeDistancePrefixes(Command* cmds,
|
||||
}
|
||||
}
|
||||
|
||||
/* Wraps 64-bit input position to 32-bit ringbuffer position preserving
|
||||
/* Wraps 64-bit input position to 32-bit ring-buffer position preserving
|
||||
"not-a-first-lap" feature. */
|
||||
static uint32_t WrapPosition(uint64_t position) {
|
||||
uint32_t result = (uint32_t)position;
|
||||
uint64_t gb = position >> 30;
|
||||
if (gb > 2) {
|
||||
/* Wrap every 2GiB; The first 3GB are continous. */
|
||||
/* Wrap every 2GiB; The first 3GB are continuous. */
|
||||
result = (result & ((1u << 30) - 1)) | ((uint32_t)((gb - 1) & 1) + 1) << 30;
|
||||
}
|
||||
return result;
|
||||
@ -306,8 +306,8 @@ static void InitCommandPrefixCodes(uint8_t cmd_depths[128],
|
||||
|
||||
/* Decide about the context map based on the ability of the prediction
|
||||
ability of the previous byte UTF8-prefix on the next byte. The
|
||||
prediction ability is calculated as shannon entropy. Here we need
|
||||
shannon entropy instead of 'BitsEntropy' since the prefix will be
|
||||
prediction ability is calculated as Shannon entropy. Here we need
|
||||
Shannon entropy instead of 'BitsEntropy' since the prefix will be
|
||||
encoded with the remaining 6 bits of the following byte, and
|
||||
BitsEntropy will assume that symbol to be stored alone using Huffman
|
||||
coding. */
|
||||
@ -382,7 +382,7 @@ static void DecideOverLiteralContextModeling(const uint8_t* input,
|
||||
if (quality < MIN_QUALITY_FOR_CONTEXT_MODELING || length < 64) {
|
||||
return;
|
||||
} else {
|
||||
/* Gather bigram data of the UTF8 byte prefixes. To make the analysis of
|
||||
/* Gather bi-gram data of the UTF8 byte prefixes. To make the analysis of
|
||||
UTF8 data faster we only examine 64 byte long strides at every 4kB
|
||||
intervals. */
|
||||
const size_t end_pos = start_pos + length;
|
||||
@ -687,6 +687,13 @@ void BrotliEncoderDestroyInstance(BrotliEncoderState* state) {
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
Copies the given input data to the internal ring buffer of the compressor.
|
||||
No processing of the data occurs at this time and this function can be
|
||||
called multiple times before calling WriteBrotliData() to process the
|
||||
accumulated input. At most input_block_size() bytes of input data can be
|
||||
copied to the ring buffer, otherwise the next WriteBrotliData() will fail.
|
||||
*/
|
||||
static void CopyInputToRingBuffer(BrotliEncoderState* s,
|
||||
const size_t input_size,
|
||||
const uint8_t* input_buffer) {
|
||||
@ -714,8 +721,8 @@ static void CopyInputToRingBuffer(BrotliEncoderState* s,
|
||||
reading new bytes from the input. However, at the last few indexes of
|
||||
the ring buffer, there are not enough bytes to build full-length
|
||||
substrings from. Since the hash table always contains full-length
|
||||
substrings, we erase with dummy 0s here to make sure that those
|
||||
substrings will contain 0s at the end instead of uninitialized
|
||||
substrings, we erase with dummy zeros here to make sure that those
|
||||
substrings will contain zeros at the end instead of uninitialized
|
||||
data.
|
||||
|
||||
Please note that erasing is not necessary (because the
|
||||
@ -724,21 +731,21 @@ static void CopyInputToRingBuffer(BrotliEncoderState* s,
|
||||
skip erasing if we have already gone around at least once in
|
||||
the ring buffer.
|
||||
|
||||
Only clear during the first round of ringbuffer writes. On
|
||||
subsequent rounds data in the ringbuffer would be affected. */
|
||||
Only clear during the first round of ring-buffer writes. On
|
||||
subsequent rounds data in the ring-buffer would be affected. */
|
||||
if (ringbuffer_->pos_ <= ringbuffer_->mask_) {
|
||||
/* This is the first time when the ring buffer is being written.
|
||||
We clear 7 bytes just after the bytes that have been copied from
|
||||
the input buffer.
|
||||
|
||||
The ringbuffer has a "tail" that holds a copy of the beginning,
|
||||
The ring-buffer has a "tail" that holds a copy of the beginning,
|
||||
but only once the ring buffer has been fully written once, i.e.,
|
||||
pos <= mask. For the first time, we need to write values
|
||||
in this tail (where index may be larger than mask), so that
|
||||
we have exactly defined behavior and don't read un-initialized
|
||||
we have exactly defined behavior and don't read uninitialized
|
||||
memory. Due to performance reasons, hashing reads data using a
|
||||
LOAD64, which can go 7 bytes beyond the bytes written in the
|
||||
ringbuffer. */
|
||||
ring-buffer. */
|
||||
memset(ringbuffer_->buffer_ + ringbuffer_->pos_, 0, 7);
|
||||
}
|
||||
}
|
||||
@ -782,6 +789,18 @@ static BROTLI_BOOL UpdateLastProcessedPos(BrotliEncoderState* s) {
|
||||
return TO_BROTLI_BOOL(wrapped_input_pos < wrapped_last_processed_pos);
|
||||
}
|
||||
|
||||
/*
|
||||
Processes the accumulated input data and sets |*out_size| to the length of
|
||||
the new output meta-block, or to zero if no new output meta-block has been
|
||||
created (in this case the processed input data is buffered internally).
|
||||
If |*out_size| is positive, |*output| points to the start of the output
|
||||
data. If |is_last| or |force_flush| is BROTLI_TRUE, an output meta-block is
|
||||
always created. However, until |is_last| is BROTLI_TRUE encoder may retain up
|
||||
to 7 bits of the last byte of output. To force encoder to dump the remaining
|
||||
bits use WriteMetadata() to append an empty meta-data block.
|
||||
Returns BROTLI_FALSE if the size of the input data is larger than
|
||||
input_block_size().
|
||||
*/
|
||||
static BROTLI_BOOL EncodeData(
|
||||
BrotliEncoderState* s, const BROTLI_BOOL is_last,
|
||||
const BROTLI_BOOL force_flush, size_t* out_size, uint8_t** output) {
|
||||
@ -863,7 +882,7 @@ static BROTLI_BOOL EncodeData(
|
||||
if (newsize > s->cmd_alloc_size_) {
|
||||
Command* new_commands;
|
||||
/* Reserve a bit more memory to allow merging with a next block
|
||||
without realloc: that would impact speed. */
|
||||
without reallocation: that would impact speed. */
|
||||
newsize += (bytes / 4) + 16;
|
||||
s->cmd_alloc_size_ = newsize;
|
||||
new_commands = BROTLI_ALLOC(m, Command, newsize);
|
||||
@ -1076,7 +1095,7 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
|
||||
will be likely big enough for the whole metablock, so that for most
|
||||
inputs we will not have to reallocate in later iterations. We do the
|
||||
allocation here and not before the loop, because if the input is small,
|
||||
this will be allocated after the zopfli cost model is freed, so this
|
||||
this will be allocated after the Zopfli cost model is freed, so this
|
||||
will not increase peak memory usage.
|
||||
TODO: If the first allocation is too small, increase command
|
||||
buffer size exponentially. */
|
||||
@ -1276,7 +1295,7 @@ BROTLI_BOOL BrotliEncoderCompress(
|
||||
}
|
||||
if (quality == 10) {
|
||||
/* TODO: Implement this direct path for all quality levels. */
|
||||
const int lg_win = BROTLI_MIN(int, kBrotliMaxWindowBits,
|
||||
const int lg_win = BROTLI_MIN(int, BROTLI_MAX_WINDOW_BITS,
|
||||
BROTLI_MAX(int, 16, lgwin));
|
||||
int ok = BrotliCompressBufferQuality10(lg_win, input_size, input_buffer,
|
||||
encoded_size, encoded_buffer);
|
||||
@ -1326,7 +1345,7 @@ static void InjectBytePaddingBlock(BrotliEncoderState* s) {
|
||||
uint8_t* destination;
|
||||
s->last_byte_ = 0;
|
||||
s->last_byte_bits_ = 0;
|
||||
/* is_last = 0, data_nibbles = 11, reseved = 0, meta_nibbles = 00 */
|
||||
/* is_last = 0, data_nibbles = 11, reserved = 0, meta_nibbles = 00 */
|
||||
seal |= 0x6u << seal_bits;
|
||||
seal_bits += 6;
|
||||
/* If we have already created storage, then append to it.
|
||||
@ -1605,7 +1624,7 @@ BROTLI_BOOL BrotliEncoderCompressStream(
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Compress data only when internal outpuf buffer is empty, stream is not
|
||||
/* Compress data only when internal output buffer is empty, stream is not
|
||||
finished and there is no pending flush request. */
|
||||
if (s->available_out_ == 0 &&
|
||||
s->stream_state_ == BROTLI_STREAM_PROCESSING) {
|
||||
|
@ -246,7 +246,7 @@ void BrotliOptimizeHuffmanCountsForRle(size_t length, uint32_t* counts,
|
||||
size_t limit;
|
||||
size_t sum;
|
||||
const size_t streak_limit = 1240;
|
||||
/* Let's make the Huffman code more compatible with rle encoding. */
|
||||
/* Let's make the Huffman code more compatible with RLE encoding. */
|
||||
size_t i;
|
||||
for (i = 0; i < length; i++) {
|
||||
if (counts[i]) {
|
||||
@ -293,10 +293,10 @@ void BrotliOptimizeHuffmanCountsForRle(size_t length, uint32_t* counts,
|
||||
}
|
||||
}
|
||||
/* 2) Let's mark all population counts that already can be encoded
|
||||
with an rle code. */
|
||||
with an RLE code. */
|
||||
memset(good_for_rle, 0, length);
|
||||
{
|
||||
/* Let's not spoil any of the existing good rle codes.
|
||||
/* Let's not spoil any of the existing good RLE codes.
|
||||
Mark any seq of 0's that is longer as 5 as a good_for_rle.
|
||||
Mark any seq of non-0's that is longer as 7 as a good_for_rle. */
|
||||
uint32_t symbol = counts[0];
|
||||
@ -319,7 +319,7 @@ void BrotliOptimizeHuffmanCountsForRle(size_t length, uint32_t* counts,
|
||||
}
|
||||
}
|
||||
}
|
||||
/* 3) Let's replace those population counts that lead to more rle codes.
|
||||
/* 3) Let's replace those population counts that lead to more RLE codes.
|
||||
Math here is in 24.8 fixed point representation. */
|
||||
stride = 0;
|
||||
limit = 256 * (counts[0] + counts[1] + counts[2]) / 3 + 420;
|
||||
@ -420,15 +420,15 @@ void BrotliWriteHuffmanTree(const uint8_t* depth,
|
||||
}
|
||||
}
|
||||
|
||||
/* First gather statistics on if it is a good idea to do rle. */
|
||||
/* First gather statistics on if it is a good idea to do RLE. */
|
||||
if (length > 50) {
|
||||
/* Find rle coding for longer codes.
|
||||
Shorter codes seem not to benefit from rle. */
|
||||
/* Find RLE coding for longer codes.
|
||||
Shorter codes seem not to benefit from RLE. */
|
||||
DecideOverRleUse(depth, new_length,
|
||||
&use_rle_for_non_zero, &use_rle_for_zero);
|
||||
}
|
||||
|
||||
/* Actual rle coding. */
|
||||
/* Actual RLE coding. */
|
||||
for (i = 0; i < new_length;) {
|
||||
const uint8_t value = depth[i];
|
||||
size_t reps = 1;
|
||||
|
@ -30,7 +30,7 @@ static BROTLI_INLINE void InitHuffmanTree(HuffmanTree* self, uint32_t count,
|
||||
self->index_right_or_value_ = right;
|
||||
}
|
||||
|
||||
/* Returns 1 is assignment of depths succeded, otherwise 0. */
|
||||
/* Returns 1 is assignment of depths succeeded, otherwise 0. */
|
||||
BROTLI_INTERNAL BROTLI_BOOL BrotliSetDepth(
|
||||
int p, HuffmanTree* pool, uint8_t* depth, int max_depth);
|
||||
|
||||
@ -53,7 +53,7 @@ BROTLI_INTERNAL void BrotliCreateHuffmanTree(const uint32_t *data,
|
||||
uint8_t *depth);
|
||||
|
||||
/* Change the population counts in a way that the consequent
|
||||
Huffman tree compression, especially its rle-part will be more
|
||||
Huffman tree compression, especially its RLE-part will be more
|
||||
likely to compress this data more efficiently.
|
||||
|
||||
length contains the size of the histogram.
|
||||
@ -62,7 +62,7 @@ BROTLI_INTERNAL void BrotliCreateHuffmanTree(const uint32_t *data,
|
||||
BROTLI_INTERNAL void BrotliOptimizeHuffmanCountsForRle(
|
||||
size_t length, uint32_t* counts, uint8_t* good_for_rle);
|
||||
|
||||
/* Write a Huffman tree from bit depths into the bitstream representation
|
||||
/* Write a Huffman tree from bit depths into the bit-stream representation
|
||||
of a Huffman tree. The generated Huffman tree is to be compressed once
|
||||
more using a Huffman tree */
|
||||
BROTLI_INTERNAL void BrotliWriteHuffmanTree(const uint8_t* depth,
|
||||
|
16
enc/hash.h
16
enc/hash.h
@ -57,7 +57,7 @@ typedef struct DictionarySearchStatictics {
|
||||
|
||||
/* kHashMul32 multiplier has these properties:
|
||||
* The multiplier must be odd. Otherwise we may lose the highest bit.
|
||||
* No long streaks of 1s or 0s.
|
||||
* No long streaks of ones or zeros.
|
||||
* There is no effort to ensure that it is a prime, the oddity is enough
|
||||
for this use.
|
||||
* The number has been tuned heuristically against compression benchmarks. */
|
||||
@ -342,16 +342,16 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
|
||||
const uint32_t key = FN(HashBytes)(&data[cur_ix_masked]);
|
||||
size_t prev_ix = self->buckets_[key];
|
||||
/* The forest index of the rightmost node of the left subtree of the new
|
||||
root, updated as we traverse and reroot the tree of the hash bucket. */
|
||||
root, updated as we traverse and re-root the tree of the hash bucket. */
|
||||
size_t node_left = FN(LeftChildIndex)(self, cur_ix);
|
||||
/* The forest index of the leftmost node of the right subtree of the new
|
||||
root, updated as we traverse and reroot the tree of the hash bucket. */
|
||||
root, updated as we traverse and re-root the tree of the hash bucket. */
|
||||
size_t node_right = FN(RightChildIndex)(self, cur_ix);
|
||||
/* The match length of the rightmost node of the left subtree of the new
|
||||
root, updated as we traverse and reroot the tree of the hash bucket. */
|
||||
root, updated as we traverse and re-root the tree of the hash bucket. */
|
||||
size_t best_len_left = 0;
|
||||
/* The match length of the leftmost node of the right subtree of the new
|
||||
root, updated as we traverse and reroot the tree of the hash bucket. */
|
||||
root, updated as we traverse and re-root the tree of the hash bucket. */
|
||||
size_t best_len_right = 0;
|
||||
size_t depth_remaining;
|
||||
if (should_reroot_tree) {
|
||||
@ -511,14 +511,14 @@ static BROTLI_INLINE void FN(StitchToPreviousBlock)(HashToBinaryTree* self,
|
||||
/* Maximum distance is window size - 16, see section 9.1. of the spec.
|
||||
Furthermore, we have to make sure that we don't look further back
|
||||
from the start of the next block than the window size, otherwise we
|
||||
could access already overwritten areas of the ringbuffer. */
|
||||
could access already overwritten areas of the ring-buffer. */
|
||||
const size_t max_backward =
|
||||
self->window_mask_ - BROTLI_MAX(size_t,
|
||||
BROTLI_WINDOW_GAP - 1,
|
||||
position - i);
|
||||
/* We know that i + MAX_TREE_COMP_LENGTH <= position + num_bytes, i.e. the
|
||||
end of the current block and that we have at least
|
||||
MAX_TREE_COMP_LENGTH tail in the ringbuffer. */
|
||||
MAX_TREE_COMP_LENGTH tail in the ring-buffer. */
|
||||
FN(StoreAndFindMatches)(self, ringbuffer, i, ringbuffer_mask,
|
||||
MAX_TREE_COMP_LENGTH, max_backward, NULL, NULL);
|
||||
}
|
||||
@ -532,7 +532,7 @@ static BROTLI_INLINE void FN(StitchToPreviousBlock)(HashToBinaryTree* self,
|
||||
|
||||
/* For BUCKET_SWEEP == 1, enabling the dictionary lookup makes compression
|
||||
a little faster (0.5% - 1%) and it compresses 0.15% better on small text
|
||||
and html inputs. */
|
||||
and HTML inputs. */
|
||||
|
||||
#define HASHER() H2
|
||||
#define BUCKET_BITS 16
|
||||
|
@ -94,7 +94,7 @@ static void FN(Init)(
|
||||
MemoryManager* m, HashForgetfulChain* self, const uint8_t* data,
|
||||
const BrotliEncoderParams* params, size_t position, size_t bytes,
|
||||
BROTLI_BOOL is_last) {
|
||||
/* Choose which init method is faster.
|
||||
/* Choose which initialization method is faster.
|
||||
Init() is about 100 times faster than InitForData(). */
|
||||
const size_t kMaxBytesForPartialHashInit = BUCKET_SIZE >> 6;
|
||||
BROTLI_UNUSED(m);
|
||||
|
@ -24,7 +24,7 @@
|
||||
and the older are forgotten. */
|
||||
#define BLOCK_SIZE (1u << BLOCK_BITS)
|
||||
|
||||
/* Mask for accessing entries in a block (in a ringbuffer manner). */
|
||||
/* Mask for accessing entries in a block (in a ring-buffer manner). */
|
||||
#define BLOCK_MASK ((1 << BLOCK_BITS) - 1)
|
||||
|
||||
#define HASH_MAP_SIZE (2 << BUCKET_BITS)
|
||||
@ -83,7 +83,7 @@ static void FN(Init)(
|
||||
MemoryManager* m, HashLongestMatch* self, const uint8_t* data,
|
||||
const BrotliEncoderParams* params, size_t position, size_t bytes,
|
||||
BROTLI_BOOL is_last) {
|
||||
/* Choose which init method is faster.
|
||||
/* Choose which initialization method is faster.
|
||||
Init() is about 100 times faster than InitForData(). */
|
||||
const size_t kMaxBytesForPartialHashInit = HASH_MAP_SIZE >> 7;
|
||||
BROTLI_UNUSED(m);
|
||||
|
@ -72,7 +72,7 @@ static void FN(Init)(
|
||||
MemoryManager* m, HashLongestMatchQuickly* self, const uint8_t* data,
|
||||
const BrotliEncoderParams* params, size_t position, size_t bytes,
|
||||
BROTLI_BOOL is_last) {
|
||||
/* Choose which init method is faster.
|
||||
/* Choose which initialization method is faster.
|
||||
Init() is about 100 times faster than InitForData(). */
|
||||
const size_t kMaxBytesForPartialHashInit = HASH_MAP_SIZE >> 7;
|
||||
BROTLI_UNUSED(m);
|
||||
|
@ -57,8 +57,8 @@ static size_t DecideMultiByteStatsLevel(size_t pos, size_t len, size_t mask,
|
||||
|
||||
static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
|
||||
const uint8_t *data, float *cost) {
|
||||
/* max_utf8 is 0 (normal ascii single byte modeling),
|
||||
1 (for 2-byte utf-8 modeling), or 2 (for 3-byte utf-8 modeling). */
|
||||
/* max_utf8 is 0 (normal ASCII single byte modeling),
|
||||
1 (for 2-byte UTF-8 modeling), or 2 (for 3-byte UTF-8 modeling). */
|
||||
const size_t max_utf8 = DecideMultiByteStatsLevel(pos, len, mask, data);
|
||||
size_t histogram[3][256] = { { 0 } };
|
||||
size_t window_half = 495;
|
||||
|
@ -18,7 +18,7 @@ extern "C" {
|
||||
#endif
|
||||
|
||||
/* Estimates how many bits the literals in the interval [pos, pos + len) in the
|
||||
ringbuffer (data, mask) will take entropy coded and writes these estimates
|
||||
ring-buffer (data, mask) will take entropy coded and writes these estimates
|
||||
to the cost[0..len) array. */
|
||||
BROTLI_INTERNAL void BrotliEstimateBitCostsForLiterals(
|
||||
size_t pos, size_t len, size_t mask, const uint8_t *data, float *cost);
|
||||
|
@ -257,7 +257,7 @@ static void InitContextBlockSplitter(
|
||||
*histograms = BROTLI_ALLOC(m, HistogramLiteral, *histograms_size);
|
||||
self->histograms_ = *histograms;
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
/* Clear only current historgram. */
|
||||
/* Clear only current histogram. */
|
||||
ClearHistogramsLiteral(&self->histograms_[0], num_contexts);
|
||||
self->last_histogram_ix_[0] = self->last_histogram_ix_[1] = 0;
|
||||
}
|
||||
|
@ -39,7 +39,7 @@ typedef struct BrotliEncoderParams {
|
||||
int lgblock;
|
||||
} BrotliEncoderParams;
|
||||
|
||||
/* Returns hashtable size for quality levels 0 and 1. */
|
||||
/* Returns hash-table size for quality levels 0 and 1. */
|
||||
static BROTLI_INLINE size_t MaxHashTableSize(int quality) {
|
||||
return quality == FAST_ONE_PASS_COMPRESSION_QUALITY ? 1 << 15 : 1 << 17;
|
||||
}
|
||||
@ -54,7 +54,7 @@ static BROTLI_INLINE size_t MaxZopfliLen(const BrotliEncoderParams* params) {
|
||||
MAX_ZOPFLI_LEN_QUALITY_11;
|
||||
}
|
||||
|
||||
/* Number of best candidates to evaluate to expand zopfli chain. */
|
||||
/* Number of best candidates to evaluate to expand Zopfli chain. */
|
||||
static BROTLI_INLINE size_t MaxZopfliCandidates(
|
||||
const BrotliEncoderParams* params) {
|
||||
return params->quality <= 10 ? 1 : 5;
|
||||
@ -63,10 +63,10 @@ static BROTLI_INLINE size_t MaxZopfliCandidates(
|
||||
static BROTLI_INLINE void SanitizeParams(BrotliEncoderParams* params) {
|
||||
params->quality = BROTLI_MIN(int, BROTLI_MAX_QUALITY,
|
||||
BROTLI_MAX(int, BROTLI_MIN_QUALITY, params->quality));
|
||||
if (params->lgwin < kBrotliMinWindowBits) {
|
||||
params->lgwin = kBrotliMinWindowBits;
|
||||
} else if (params->lgwin > kBrotliMaxWindowBits) {
|
||||
params->lgwin = kBrotliMaxWindowBits;
|
||||
if (params->lgwin < BROTLI_MIN_WINDOW_BITS) {
|
||||
params->lgwin = BROTLI_MIN_WINDOW_BITS;
|
||||
} else if (params->lgwin > BROTLI_MAX_WINDOW_BITS) {
|
||||
params->lgwin = BROTLI_MAX_WINDOW_BITS;
|
||||
}
|
||||
}
|
||||
|
||||
@ -84,8 +84,8 @@ static BROTLI_INLINE int ComputeLgBlock(const BrotliEncoderParams* params) {
|
||||
lgblock = BROTLI_MIN(int, 18, params->lgwin);
|
||||
}
|
||||
} else {
|
||||
lgblock = BROTLI_MIN(int, kBrotliMaxInputBlockBits,
|
||||
BROTLI_MAX(int, kBrotliMinInputBlockBits, lgblock));
|
||||
lgblock = BROTLI_MIN(int, BROTLI_MAX_INPUT_BLOCK_BITS,
|
||||
BROTLI_MAX(int, BROTLI_MIN_INPUT_BLOCK_BITS, lgblock));
|
||||
}
|
||||
return lgblock;
|
||||
}
|
||||
@ -94,14 +94,15 @@ static BROTLI_INLINE int ComputeLgBlock(const BrotliEncoderParams* params) {
|
||||
Allocate at least lgwin + 1 bits for the ring buffer so that the newly
|
||||
added block fits there completely and we still get lgwin bits and at least
|
||||
read_block_size_bits + 1 bits because the copy tail length needs to be
|
||||
smaller than ringbuffer size. */
|
||||
smaller than ring-buffer size. */
|
||||
static BROTLI_INLINE int ComputeRbBits(const BrotliEncoderParams* params) {
|
||||
return 1 + BROTLI_MAX(int, params->lgwin, params->lgblock);
|
||||
}
|
||||
|
||||
static BROTLI_INLINE size_t MaxMetablockSize(
|
||||
const BrotliEncoderParams* params) {
|
||||
int bits = BROTLI_MIN(int, ComputeRbBits(params), kBrotliMaxInputBlockBits);
|
||||
int bits =
|
||||
BROTLI_MIN(int, ComputeRbBits(params), BROTLI_MAX_INPUT_BLOCK_BITS);
|
||||
return (size_t)1 << bits;
|
||||
}
|
||||
|
||||
|
@ -30,7 +30,7 @@ extern "C" {
|
||||
buffer_[-1] == buffer_[(1 << window_bits) - 1] and
|
||||
buffer_[-2] == buffer_[(1 << window_bits) - 2]. */
|
||||
typedef struct RingBuffer {
|
||||
/* Size of the ringbuffer is (1 << window_bits) + tail_size_. */
|
||||
/* Size of the ring-buffer is (1 << window_bits) + tail_size_. */
|
||||
const uint32_t size_;
|
||||
const uint32_t mask_;
|
||||
const uint32_t tail_size_;
|
||||
@ -42,7 +42,7 @@ typedef struct RingBuffer {
|
||||
/* The actual ring buffer containing the copy of the last two bytes, the data,
|
||||
and the copy of the beginning as a tail. */
|
||||
uint8_t *data_;
|
||||
/* The start of the ringbuffer. */
|
||||
/* The start of the ring-buffer. */
|
||||
uint8_t *buffer_;
|
||||
} RingBuffer;
|
||||
|
||||
@ -106,7 +106,7 @@ static BROTLI_INLINE void RingBufferWrite(
|
||||
MemoryManager* m, const uint8_t *bytes, size_t n, RingBuffer* rb) {
|
||||
if (rb->pos_ == 0 && n < rb->tail_size_) {
|
||||
/* Special case for the first write: to process the first block, we don't
|
||||
need to allocate the whole ringbuffer and we don't need the tail
|
||||
need to allocate the whole ring-buffer and we don't need the tail
|
||||
either. However, we do this memory usage optimization only if the
|
||||
first write is less than the tail size, which is also the input block
|
||||
size, otherwise it is likely that other blocks will follow and we
|
||||
|
@ -19,7 +19,7 @@ extern "C" {
|
||||
static const double kMinUTF8Ratio = 0.75;
|
||||
|
||||
/* Returns 1 if at least min_fraction of the bytes between pos and
|
||||
pos + length in the (data, mask) ringbuffer is UTF8-encoded, otherwise
|
||||
pos + length in the (data, mask) ring-buffer is UTF8-encoded, otherwise
|
||||
returns 0. */
|
||||
BROTLI_INTERNAL BROTLI_BOOL BrotliIsMostlyUTF8(
|
||||
const uint8_t* data, const size_t pos, const size_t mask,
|
||||
|
@ -4,7 +4,10 @@
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* API for Brotli decompression */
|
||||
/**
|
||||
* @file
|
||||
* API for Brotli decompression.
|
||||
*/
|
||||
|
||||
#ifndef BROTLI_DEC_DECODE_H_
|
||||
#define BROTLI_DEC_DECODE_H_
|
||||
@ -16,19 +19,47 @@
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Opaque structure that holds decoder state.
|
||||
*
|
||||
* Allocated and initialized with ::BrotliDecoderCreateInstance.
|
||||
* Cleaned up and deallocated with ::BrotliDecoderDestroyInstance.
|
||||
*/
|
||||
typedef struct BrotliDecoderStateStruct BrotliDecoderState;
|
||||
|
||||
/**
|
||||
* Result type for ::BrotliDecoderDecompress and
|
||||
* ::BrotliDecoderDecompressStream functions.
|
||||
*/
|
||||
typedef enum {
|
||||
/* Decoding error, e.g. corrupt input or memory allocation problem */
|
||||
/** Decoding error, e.g. corrupted input or memory allocation problem. */
|
||||
BROTLI_DECODER_RESULT_ERROR = 0,
|
||||
/* Decoding successfully completed */
|
||||
/** Decoding successfully completed */
|
||||
BROTLI_DECODER_RESULT_SUCCESS = 1,
|
||||
/* Partially done; should be called again with more input */
|
||||
/** Partially done; should be called again with more input */
|
||||
BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT = 2,
|
||||
/* Partially done; should be called again with more output */
|
||||
/** Partially done; should be called again with more output */
|
||||
BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT = 3
|
||||
} BrotliDecoderResult;
|
||||
|
||||
/**
|
||||
* Template that evaluates items of ::BrotliDecoderErrorCode.
|
||||
*
|
||||
* Example: @code {.cpp}
|
||||
* // Log Brotli error code.
|
||||
* switch (brotliDecoderErrorCode) {
|
||||
* #define CASE_(PREFIX, NAME, CODE) \
|
||||
* case BROTLI_DECODER ## PREFIX ## NAME: \
|
||||
* LOG(INFO) << "error code:" << #NAME; \
|
||||
* break;
|
||||
* #define NEWLINE_
|
||||
* BROTLI_DECODER_ERROR_CODES_LIST(CASE_, NEWLINE_)
|
||||
* #undef CASE_
|
||||
* #undef NEWLINE_
|
||||
* default: LOG(FATAL) << "unknown brotli error code";
|
||||
* }
|
||||
* @endcode
|
||||
*/
|
||||
#define BROTLI_DECODER_ERROR_CODES_LIST(BROTLI_ERROR_CODE, SEPARATOR) \
|
||||
BROTLI_ERROR_CODE(_, NO_ERROR, 0) SEPARATOR \
|
||||
/* Same as BrotliDecoderResult values */ \
|
||||
@ -65,112 +96,235 @@ typedef enum {
|
||||
BROTLI_ERROR_CODE(_ERROR_ALLOC_, CONTEXT_MAP, -25) SEPARATOR \
|
||||
BROTLI_ERROR_CODE(_ERROR_ALLOC_, RING_BUFFER_1, -26) SEPARATOR \
|
||||
BROTLI_ERROR_CODE(_ERROR_ALLOC_, RING_BUFFER_2, -27) SEPARATOR \
|
||||
/* -28..-29 codes are reserved for dynamic ringbuffer allocation */ \
|
||||
/* -28..-29 codes are reserved for dynamic ring-buffer allocation */ \
|
||||
BROTLI_ERROR_CODE(_ERROR_ALLOC_, BLOCK_TYPE_TREES, -30) SEPARATOR \
|
||||
\
|
||||
/* "Impossible" states */ \
|
||||
BROTLI_ERROR_CODE(_ERROR_, UNREACHABLE, -31)
|
||||
|
||||
/**
|
||||
* Error code for detailed logging / production debugging.
|
||||
*
|
||||
* See ::BrotliDecoderGetErrorCode and ::BROTLI_LAST_ERROR_CODE.
|
||||
*/
|
||||
typedef enum {
|
||||
#define BROTLI_COMMA_ ,
|
||||
#define BROTLI_ERROR_CODE_ENUM_ITEM_(PREFIX, NAME, CODE) \
|
||||
BROTLI_DECODER ## PREFIX ## NAME = CODE
|
||||
BROTLI_DECODER_ERROR_CODES_LIST(BROTLI_ERROR_CODE_ENUM_ITEM_, BROTLI_COMMA_)
|
||||
} BrotliDecoderErrorCode;
|
||||
#undef BROTLI_ERROR_CODE_ENUM_ITEM_
|
||||
#undef BROTLI_COMMA_
|
||||
} BrotliDecoderErrorCode;
|
||||
|
||||
/**
|
||||
* The value of the last error code, negative integer.
|
||||
*
|
||||
* All other error code values are in the range from ::BROTLI_LAST_ERROR_CODE
|
||||
* to @c -1. There are also 4 other possible non-error codes @c 0 .. @c 3 in
|
||||
* ::BrotliDecoderErrorCode enumeration.
|
||||
*/
|
||||
#define BROTLI_LAST_ERROR_CODE BROTLI_DECODER_ERROR_UNREACHABLE
|
||||
|
||||
/* Creates the instance of BrotliDecoderState and initializes it. |alloc_func|
|
||||
and |free_func| MUST be both zero or both non-zero. In the case they are both
|
||||
zero, default memory allocators are used. |opaque| is passed to |alloc_func|
|
||||
and |free_func| when they are called. */
|
||||
/**
|
||||
* Creates an instance of ::BrotliDecoderState and initializes it.
|
||||
*
|
||||
* @p alloc_func and @p free_func @b MUST be both zero or both non-zero. In the
|
||||
* case they are both zero, default memory allocators are used. @p opaque is
|
||||
* passed to @p alloc_func and @p free_func when they are called.
|
||||
*
|
||||
* @param alloc_func custom memory allocation function
|
||||
* @param free_func custom memory fee function
|
||||
* @param opaque custom memory manager handle
|
||||
* @returns @c 0 if instance can not be allocated or initialized
|
||||
* @returns pointer to initialized ::BrotliDecoderState otherwise
|
||||
*/
|
||||
BROTLI_DEC_API BrotliDecoderState* BrotliDecoderCreateInstance(
|
||||
brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
|
||||
|
||||
/* Deinitializes and frees BrotliDecoderState instance. */
|
||||
/**
|
||||
* Deinitializes and frees ::BrotliDecoderState instance.
|
||||
*
|
||||
* @param state decoder instance to be cleaned up and deallocated
|
||||
*/
|
||||
BROTLI_DEC_API void BrotliDecoderDestroyInstance(BrotliDecoderState* state);
|
||||
|
||||
/* Decompresses the data in |encoded_buffer| into |decoded_buffer|, and sets
|
||||
|*decoded_size| to the decompressed length. */
|
||||
/**
|
||||
* Performs one-shot memory-to-memory decompression.
|
||||
*
|
||||
* Decompresses the data in @p encoded_buffer into @p decoded_buffer, and sets
|
||||
* @p *decoded_size to the decompressed length.
|
||||
*
|
||||
* @param encoded_size size of @p encoded_buffer
|
||||
* @param encoded_buffer compressed data buffer with at least @p encoded_size
|
||||
* addressable bytes
|
||||
* @param[in, out] decoded_size @b in: size of @p decoded_buffer; \n
|
||||
* @b out: length of decompressed data written to
|
||||
* @p decoded_buffer
|
||||
* @param decoded_buffer decompressed data destination buffer
|
||||
* @returns ::BROTLI_DECODER_RESULT_ERROR if input is corrupted, memory
|
||||
* allocation failed, or @p decoded_buffer is not large enough;
|
||||
* @returns ::BROTLI_DECODER_RESULT_SUCCESS otherwise
|
||||
*/
|
||||
BROTLI_DEC_API BrotliDecoderResult BrotliDecoderDecompress(
|
||||
size_t encoded_size,
|
||||
const uint8_t encoded_buffer[BROTLI_ARRAY_PARAM(encoded_size)],
|
||||
size_t* decoded_size,
|
||||
uint8_t decoded_buffer[BROTLI_ARRAY_PARAM(*decoded_size)]);
|
||||
|
||||
/* Decompresses the data. Supports partial input and output.
|
||||
|
||||
Must be called with an allocated input buffer in |*next_in| and an allocated
|
||||
output buffer in |*next_out|. The values |*available_in| and |*available_out|
|
||||
must specify the allocated size in |*next_in| and |*next_out| respectively;
|
||||
when |*available_out| is 0, |next_out| is allowed to be NULL.
|
||||
|
||||
After each call, |*available_in| will be decremented by the amount of input
|
||||
bytes consumed, and the |*next_in| pointer will be incremented by that
|
||||
amount. Similarly, |*available_out| will be decremented by the amount of
|
||||
output bytes written, and the |*next_out| pointer will be incremented by that
|
||||
amount. |total_out|, if it is not a null-pointer, will be set to the number
|
||||
of bytes decompressed since the last state initialization.
|
||||
|
||||
Input is never overconsumed, so |next_in| and |available_in| could be passed
|
||||
to the next consumer after decoding is complete. */
|
||||
/**
|
||||
* Decompresses the input stream to the output stream.
|
||||
*
|
||||
* The values @p *available_in and @p *available_out must specify the number of
|
||||
* bytes addressable at @p *next_in and @p *next_out respectively.
|
||||
* When @p *available_out is @c 0, @p next_out is allowed to be @c NULL.
|
||||
*
|
||||
* After each call, @p *available_in will be decremented by the amount of input
|
||||
* bytes consumed, and the @p *next_in pointer will be incremented by that
|
||||
* amount. Similarly, @p *available_out will be decremented by the amount of
|
||||
* output bytes written, and the @p *next_out pointer will be incremented by
|
||||
* that amount.
|
||||
*
|
||||
* @p total_out, if it is not a null-pointer, will be set to the number
|
||||
* of bytes decompressed since the last @p state initialization.
|
||||
*
|
||||
* @note Input is never overconsumed, so @p next_in and @p available_in could be
|
||||
* passed to the next consumer after decoding is complete.
|
||||
*
|
||||
* @param state decoder instance
|
||||
* @param[in, out] available_in @b in: amount of available input; \n
|
||||
* @b out: amount of unused input
|
||||
* @param[in, out] next_in pointer to the next compressed byte
|
||||
* @param[in, out] available_out @b in: length of output buffer; \n
|
||||
* @b out: remaining size of output buffer
|
||||
* @param[in, out] next_out output buffer cursor;
|
||||
* can be @c NULL if @p available_out is @c 0
|
||||
* @param[out] total_out number of bytes decompressed so far; can be @c NULL
|
||||
* @returns ::BROTLI_DECODER_RESULT_ERROR if input is corrupted, memory
|
||||
* allocation failed, arguments were invalid, etc.;
|
||||
* use ::BrotliDecoderGetErrorCode to get detailed error code
|
||||
* @returns ::BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT decoding is blocked until
|
||||
* more output space is provided
|
||||
* @returns ::BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT decoding is blocked until
|
||||
* more input data is provided
|
||||
* @returns ::BROTLI_DECODER_RESULT_SUCCESS decoding is finished, no more
|
||||
* input might be consumed and no more output will be produced
|
||||
*/
|
||||
BROTLI_DEC_API BrotliDecoderResult BrotliDecoderDecompressStream(
|
||||
BrotliDecoderState* s, size_t* available_in, const uint8_t** next_in,
|
||||
BrotliDecoderState* state, size_t* available_in, const uint8_t** next_in,
|
||||
size_t* available_out, uint8_t** next_out, size_t* total_out);
|
||||
|
||||
/* Fills the new state with a dictionary for LZ77, warming up the ringbuffer,
|
||||
e.g. for custom static dictionaries for data formats.
|
||||
Not to be confused with the built-in transformable dictionary of Brotli.
|
||||
|size| should be less or equal to 2^24 (16MiB), otherwise the dictionary will
|
||||
be ignored. The dictionary must exist in memory until decoding is done and
|
||||
is owned by the caller. To use:
|
||||
1) Allocate and initialize state with BrotliCreateInstance
|
||||
2) Use BrotliDecoderSetCustomDictionary
|
||||
3) Use BrotliDecoderDecompressStream
|
||||
4) Clean up and free state with BrotliDestroyState
|
||||
*/
|
||||
/**
|
||||
* Prepends LZ77 dictionary.
|
||||
*
|
||||
* Fills the fresh ::BrotliDecoderState with additional data corpus for LZ77
|
||||
* backward references.
|
||||
*
|
||||
* @note Not to be confused with the static dictionary (see RFC7932 section 8).
|
||||
* @warning The dictionary must exist in memory until decoding is done and
|
||||
* is owned by the caller.
|
||||
*
|
||||
* Workflow:
|
||||
* -# Allocate and initialize state with ::BrotliDecoderCreateInstance
|
||||
* -# Invoke ::BrotliDecoderSetCustomDictionary
|
||||
* -# Use ::BrotliDecoderDecompressStream
|
||||
* -# Clean up and free state with ::BrotliDecoderDestroyInstance
|
||||
*
|
||||
* @param state decoder instance
|
||||
* @param size length of @p dict; should be less or equal to 2^24 (16MiB),
|
||||
* otherwise the dictionary will be ignored
|
||||
* @param dict "dictionary"; @b MUST be the same as used during compression
|
||||
*/
|
||||
BROTLI_DEC_API void BrotliDecoderSetCustomDictionary(
|
||||
BrotliDecoderState* s, size_t size,
|
||||
BrotliDecoderState* state, size_t size,
|
||||
const uint8_t dict[BROTLI_ARRAY_PARAM(size)]);
|
||||
|
||||
/* Returns BROTLI_TRUE, if decoder has some unconsumed output.
|
||||
Otherwise returns BROTLI_FALSE. */
|
||||
/**
|
||||
* Checks if decoder has more output.
|
||||
*
|
||||
* @param state decoder instance
|
||||
* @returns ::BROTLI_TRUE, if decoder has some unconsumed output
|
||||
* @returns ::BROTLI_FALSE otherwise
|
||||
*/
|
||||
BROTLI_DEC_API BROTLI_BOOL BrotliDecoderHasMoreOutput(
|
||||
const BrotliDecoderState* s);
|
||||
const BrotliDecoderState* state);
|
||||
|
||||
/* Returns pointer to internal output buffer.
|
||||
Set |size| to zero, to request all the continous output produced by decoder
|
||||
so far. Otherwise no more than |size| bytes output will be provided.
|
||||
After return |size| contains the size of output buffer region available for
|
||||
reading. |size| bytes of output are considered consumed for all consecutive
|
||||
calls to the instance methods; returned pointer becomes invalidated as well.
|
||||
|
||||
This method is created to make language bindings easier and more efficient:
|
||||
1. push data to DecompressStream, until "needs more output" is reported
|
||||
2. use TakeOutput to peek bytes and copy to language-specific entity
|
||||
Also this could be useful if there is an output stream that is able to
|
||||
consume all the provided data (e.g. when data is saved to file system).
|
||||
/**
|
||||
* Acquires pointer to internal output buffer.
|
||||
*
|
||||
* This method is used to make language bindings easier and more efficient:
|
||||
* -# push data to ::BrotliDecoderDecompressStream,
|
||||
* until ::BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT is reported
|
||||
* -# use ::BrotliDecoderTakeOutput to peek bytes and copy to language-specific
|
||||
* entity
|
||||
*
|
||||
* Also this could be useful if there is an output stream that is able to
|
||||
* consume all the provided data (e.g. when data is saved to file system).
|
||||
*
|
||||
* @attention After every call to ::BrotliDecoderTakeOutput @p *size bytes of
|
||||
* output are considered consumed for all consecutive calls to the
|
||||
* instance methods; returned pointer becomes invalidated as well.
|
||||
*
|
||||
* @note Decoder output is not guaranteed to be contiguous. This means that
|
||||
* after the size-unrestricted call to ::BrotliDecoderTakeOutput,
|
||||
* immediate next call to ::BrotliDecoderTakeOutput may return more data.
|
||||
*
|
||||
* @param state decoder instance
|
||||
* @param[in, out] size @b in: number of bytes caller is ready to take, @c 0 if
|
||||
* any amount could be handled; \n
|
||||
* @b out: amount of data pointed by returned pointer and
|
||||
* considered consumed; \n
|
||||
* out value is never greater than in value, unless it is @c 0
|
||||
* @returns pointer to output data
|
||||
*/
|
||||
BROTLI_DEC_API const uint8_t* BrotliDecoderTakeOutput(
|
||||
BrotliDecoderState* s, size_t* size);
|
||||
BrotliDecoderState* state, size_t* size);
|
||||
|
||||
/* Returns BROTLI_TRUE, if decoder has already received some input bytes.
|
||||
Otherwise returns BROTLI_FALSE. */
|
||||
BROTLI_DEC_API BROTLI_BOOL BrotliDecoderIsUsed(const BrotliDecoderState* s);
|
||||
/**
|
||||
* Checks if instance has already consumed input.
|
||||
*
|
||||
* Instance that returns ::BROTLI_FALSE is considered "fresh" and could be
|
||||
* reused.
|
||||
*
|
||||
* @param state decoder instance
|
||||
* @returns ::BROTLI_TRUE if decoder has already used some input bytes
|
||||
* @returns ::BROTLI_FALSE otherwise
|
||||
*/
|
||||
BROTLI_DEC_API BROTLI_BOOL BrotliDecoderIsUsed(const BrotliDecoderState* state);
|
||||
|
||||
/* Returns BROTLI_TRUE, if decoder is in a state where we reached the end of the
|
||||
input and produced all of the output; returns BROTLI_FALSE otherwise. */
|
||||
BROTLI_BOOL BrotliDecoderIsFinished(const BrotliDecoderState* s);
|
||||
/**
|
||||
* Checks if decoder instance reached the final state.
|
||||
*
|
||||
* @param state decoder instance
|
||||
* @returns ::BROTLI_TRUE if decoder is in a state where it reached the end of
|
||||
* the input and produced all of the output
|
||||
* @returns ::BROTLI_FALSE otherwise
|
||||
*/
|
||||
BROTLI_BOOL BrotliDecoderIsFinished(const BrotliDecoderState* state);
|
||||
|
||||
/* Returns detailed error code after BrotliDecoderDecompressStream returns
|
||||
BROTLI_DECODER_RESULT_ERROR. */
|
||||
BrotliDecoderErrorCode BrotliDecoderGetErrorCode(const BrotliDecoderState* s);
|
||||
/**
|
||||
* Acquires a detailed error code.
|
||||
*
|
||||
* Should be used only after ::BrotliDecoderDecompressStream returns
|
||||
* ::BROTLI_DECODER_RESULT_ERROR.
|
||||
*
|
||||
* See also ::BrotliDecoderErrorString
|
||||
*
|
||||
* @param state decoder instance
|
||||
* @returns last saved error code
|
||||
*/
|
||||
BrotliDecoderErrorCode BrotliDecoderGetErrorCode(
|
||||
const BrotliDecoderState* state);
|
||||
|
||||
/**
|
||||
* Converts error code to a c-string.
|
||||
*/
|
||||
BROTLI_DEC_API const char* BrotliDecoderErrorString(BrotliDecoderErrorCode c);
|
||||
|
||||
/* Decoder version. Look at BROTLI_VERSION for more information. */
|
||||
/**
|
||||
* Gets a decoder library version.
|
||||
*
|
||||
* Look at BROTLI_VERSION for more information.
|
||||
*/
|
||||
BROTLI_DEC_API uint32_t BrotliDecoderVersion(void);
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
|
@ -4,7 +4,10 @@
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* API for Brotli compression. */
|
||||
/**
|
||||
* @file
|
||||
* API for Brotli compression.
|
||||
*/
|
||||
|
||||
#ifndef BROTLI_ENC_ENCODE_H_
|
||||
#define BROTLI_ENC_ENCODE_H_
|
||||
@ -16,212 +19,405 @@
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
static const int kBrotliMinWindowBits = 10;
|
||||
static const int kBrotliMaxWindowBits = 24; /* == BROTLI_MAX_DISTANCE_BITS */
|
||||
static const int kBrotliMinInputBlockBits = 16;
|
||||
static const int kBrotliMaxInputBlockBits = 24;
|
||||
|
||||
/** Minimal value for ::BROTLI_PARAM_LGWIN parameter. */
|
||||
#define BROTLI_MIN_WINDOW_BITS 10
|
||||
/**
|
||||
* Maximal value for ::BROTLI_PARAM_LGWIN parameter.
|
||||
*
|
||||
* @note equal to @c BROTLI_MAX_DISTANCE_BITS constant.
|
||||
*/
|
||||
#define BROTLI_MAX_WINDOW_BITS 24
|
||||
/** Minimal value for ::BROTLI_PARAM_LGBLOCK parameter. */
|
||||
#define BROTLI_MIN_INPUT_BLOCK_BITS 16
|
||||
/** Maximal value for ::BROTLI_PARAM_LGBLOCK parameter. */
|
||||
#define BROTLI_MAX_INPUT_BLOCK_BITS 24
|
||||
/** Minimal value for ::BROTLI_PARAM_QUALITY parameter. */
|
||||
#define BROTLI_MIN_QUALITY 0
|
||||
/** Maximal value for ::BROTLI_PARAM_QUALITY parameter. */
|
||||
#define BROTLI_MAX_QUALITY 11
|
||||
|
||||
BROTLI_DEPRECATED static const int kBrotliMinWindowBits =
|
||||
BROTLI_MIN_WINDOW_BITS;
|
||||
BROTLI_DEPRECATED static const int kBrotliMaxWindowBits =
|
||||
BROTLI_MAX_WINDOW_BITS;
|
||||
|
||||
/** Options for ::BROTLI_PARAM_MODE parameter. */
|
||||
typedef enum BrotliEncoderMode {
|
||||
/* Default compression mode. The compressor does not know anything in
|
||||
advance about the properties of the input. */
|
||||
/**
|
||||
* Default compression mode.
|
||||
*
|
||||
* In this mode compressor does not know anything in advance about the
|
||||
* properties of the input.
|
||||
*/
|
||||
BROTLI_MODE_GENERIC = 0,
|
||||
/* Compression mode for UTF-8 format text input. */
|
||||
/** Compression mode for UTF-8 formated text input. */
|
||||
BROTLI_MODE_TEXT = 1,
|
||||
/* Compression mode used in WOFF 2.0. */
|
||||
/** Compression mode used in WOFF 2.0. */
|
||||
BROTLI_MODE_FONT = 2
|
||||
} BrotliEncoderMode;
|
||||
|
||||
/** Default value for ::BROTLI_PARAM_QUALITY parameter. */
|
||||
#define BROTLI_DEFAULT_QUALITY 11
|
||||
/** Default value for ::BROTLI_PARAM_LGWIN parameter. */
|
||||
#define BROTLI_DEFAULT_WINDOW 22
|
||||
/** Default value for ::BROTLI_PARAM_MODE parameter. */
|
||||
#define BROTLI_DEFAULT_MODE BROTLI_MODE_GENERIC
|
||||
|
||||
/** Operations that can be performed by streaming encoder. */
|
||||
typedef enum BrotliEncoderOperation {
|
||||
/**
|
||||
* Process input.
|
||||
*
|
||||
* Encoder may postpone producing output, until it has processed enough input.
|
||||
*/
|
||||
BROTLI_OPERATION_PROCESS = 0,
|
||||
/* Request output stream to flush. Performed when input stream is depleted
|
||||
and there is enough space in output stream. */
|
||||
/**
|
||||
* Produce output for all processed input.
|
||||
*
|
||||
* Actual flush is performed when input stream is depleted and there is enough
|
||||
* space in output stream. This means that client should repeat
|
||||
* ::BROTLI_OPERATION_FLUSH operation until @p available_in becomes @c 0, and
|
||||
* ::BrotliEncoderHasMoreOutput returns ::BROTLI_FALSE.
|
||||
*
|
||||
* @warning Until flush is complete, client @b SHOULD @b NOT swap,
|
||||
* reduce or extend input stream.
|
||||
*
|
||||
* When flush is complete, output data will be sufficient for decoder to
|
||||
* reproduce all the given input.
|
||||
*/
|
||||
BROTLI_OPERATION_FLUSH = 1,
|
||||
/* Request output stream to finish. Performed when input stream is depleted
|
||||
and there is enough space in output stream. */
|
||||
/**
|
||||
* Finalize the stream.
|
||||
*
|
||||
* Actual finalization is performed when input stream is depleted and there is
|
||||
* enough space in output stream. This means that client should repeat
|
||||
* ::BROTLI_OPERATION_FLUSH operation until @p available_in becomes @c 0, and
|
||||
* ::BrotliEncoderHasMoreOutput returns ::BROTLI_FALSE.
|
||||
*
|
||||
* @warning Until finalization is complete, client @b SHOULD @b NOT swap,
|
||||
* reduce or extend input stream.
|
||||
*
|
||||
* Helper function ::BrotliEncoderIsFinished checks if stream is finalized and
|
||||
* output fully dumped.
|
||||
*
|
||||
* Adding more input data to finalized stream is impossible.
|
||||
*/
|
||||
BROTLI_OPERATION_FINISH = 2,
|
||||
/* Emits metadata block to stream. Stream is soft-flushed before metadata
|
||||
block is emitted. CAUTION: when operation is started, length of the input
|
||||
buffer is interpreted as length of a metadata block; changing operation,
|
||||
expanding or truncating input before metadata block is completely emitted
|
||||
will cause an error; metadata block must not be greater than 16MiB. */
|
||||
/**
|
||||
* Emit metadata block to stream.
|
||||
*
|
||||
* Metadata is opaque to Brotli: neither encoder, nor decoder processes this
|
||||
* data or relies on it. It may be used to pass some extra information from
|
||||
* encoder client to decoder client without interfering with main data stream.
|
||||
*
|
||||
* @note Encoder may emit empty metadata blocks internally, to pad encoded
|
||||
* stream to byte boundary.
|
||||
*
|
||||
* @warning Until emitting metadata is complete client @b SHOULD @b NOT swap,
|
||||
* reduce or extend input stream.
|
||||
*
|
||||
* @warning The whole content of input buffer is considered to be the content
|
||||
* of metadata block. Do @b NOT @e append metadata to input stream,
|
||||
* before it is depleted with other operations.
|
||||
*
|
||||
* Stream is soft-flushed before metadata block is emitted. Metadata block
|
||||
* @b MUST be no longer than than 16MiB.
|
||||
*/
|
||||
BROTLI_OPERATION_EMIT_METADATA = 3
|
||||
} BrotliEncoderOperation;
|
||||
|
||||
/** Options to be used with ::BrotliEncoderSetParameter. */
|
||||
typedef enum BrotliEncoderParameter {
|
||||
/**
|
||||
* Tune encoder for specific input.
|
||||
*
|
||||
* ::BrotliEncoderMode enumerates all available values.
|
||||
*/
|
||||
BROTLI_PARAM_MODE = 0,
|
||||
/* Controls the compression-speed vs compression-density tradeoffs. The higher
|
||||
the quality, the slower the compression. Range is 0 to 11. */
|
||||
/**
|
||||
* The main compression speed-density lever.
|
||||
*
|
||||
* The higher the quality, the slower the compression. Range is
|
||||
* from ::BROTLI_MIN_QUALITY to ::BROTLI_MAX_QUALITY.
|
||||
*/
|
||||
BROTLI_PARAM_QUALITY = 1,
|
||||
/* Base 2 logarithm of the sliding window size. Range is 10 to 24. */
|
||||
/**
|
||||
* Recommended sliding LZ77 window size.
|
||||
*
|
||||
* Encoder may reduce this value, e.g. if input is much smaller than
|
||||
* window size.
|
||||
*
|
||||
* Window size is `(1 << value) - 16`.
|
||||
*
|
||||
* Range is from ::BROTLI_MIN_WINDOW_BITS to ::BROTLI_MAX_WINDOW_BITS.
|
||||
*/
|
||||
BROTLI_PARAM_LGWIN = 2,
|
||||
/* Base 2 logarithm of the maximum input block size. Range is 16 to 24.
|
||||
If set to 0, the value will be set based on the quality. */
|
||||
/**
|
||||
* Recommended input block size.
|
||||
*
|
||||
* Encoder may reduce this value, e.g. if input is much smaller than input
|
||||
* block size.
|
||||
*
|
||||
* Range is from ::BROTLI_MIN_INPUT_BLOCK_BITS to
|
||||
* ::BROTLI_MAX_INPUT_BLOCK_BITS.
|
||||
*
|
||||
* @note Bigger input block size allows better compression, but consumes more
|
||||
* memory. \n The rough formula of memory used for temporary input
|
||||
* storage is `3 << lgBlock`.
|
||||
*/
|
||||
BROTLI_PARAM_LGBLOCK = 3
|
||||
} BrotliEncoderParameter;
|
||||
|
||||
/* A state can not be reused for multiple brotli streams. */
|
||||
/**
|
||||
* Opaque structure that holds encoder state.
|
||||
*
|
||||
* Allocated and initialized with ::BrotliEncoderCreateInstance.
|
||||
* Cleaned up and deallocated with ::BrotliEncoderDestroyInstance.
|
||||
*/
|
||||
typedef struct BrotliEncoderStateStruct BrotliEncoderState;
|
||||
|
||||
/* Sets the specified parameter to the given encoder instance.
|
||||
Returns BROTLI_FALSE if parameter is unrecognized, or value is invalid.
|
||||
Returns BROTLI_FALSE if value of parameter can not be changed at current
|
||||
encoder state (e.g. when encoding is started, window size might be already
|
||||
encoded and therefore it is impossible to change it).
|
||||
Returns BROTLI_TRUE if value is accepted. CAUTION: invalid values might be
|
||||
accepted in case they would not break encoding process. */
|
||||
/**
|
||||
* Sets the specified parameter to the given encoder instance.
|
||||
*
|
||||
* @param state encoder instance
|
||||
* @param param parameter to set
|
||||
* @param value new parameter value
|
||||
* @returns ::BROTLI_FALSE if parameter is unrecognized, or value is invalid
|
||||
* @returns ::BROTLI_FALSE if value of parameter can not be changed at current
|
||||
* encoder state (e.g. when encoding is started, window size might be
|
||||
* already encoded and therefore it is impossible to change it)
|
||||
* @returns ::BROTLI_TRUE if value is accepted
|
||||
* @warning invalid values might be accepted in case they would not break
|
||||
* encoding process.
|
||||
*/
|
||||
BROTLI_ENC_API BROTLI_BOOL BrotliEncoderSetParameter(
|
||||
BrotliEncoderState* state, BrotliEncoderParameter p, uint32_t value);
|
||||
BrotliEncoderState* state, BrotliEncoderParameter param, uint32_t value);
|
||||
|
||||
/* Creates the instance of BrotliEncoderState and initializes it.
|
||||
|alloc_func| and |free_func| MUST be both zero or both non-zero. In the case
|
||||
they are both zero, default memory allocators are used. |opaque| is passed to
|
||||
|alloc_func| and |free_func| when they are called. */
|
||||
/**
|
||||
* Creates an instance of ::BrotliEncoderState and initializes it.
|
||||
*
|
||||
* @p alloc_func and @p free_func @b MUST be both zero or both non-zero. In the
|
||||
* case they are both zero, default memory allocators are used. @p opaque is
|
||||
* passed to @p alloc_func and @p free_func when they are called.
|
||||
*
|
||||
* @param alloc_func custom memory allocation function
|
||||
* @param free_func custom memory fee function
|
||||
* @param opaque custom memory manager handle
|
||||
* @returns @c 0 if instance can not be allocated or initialized
|
||||
* @returns pointer to initialized ::BrotliEncoderState otherwise
|
||||
*/
|
||||
BROTLI_ENC_API BrotliEncoderState* BrotliEncoderCreateInstance(
|
||||
brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
|
||||
|
||||
/* Deinitializes and frees BrotliEncoderState instance. */
|
||||
/**
|
||||
* Deinitializes and frees ::BrotliEncoderState instance.
|
||||
*
|
||||
* @param state decoder instance to be cleaned up and deallocated
|
||||
*/
|
||||
BROTLI_ENC_API void BrotliEncoderDestroyInstance(BrotliEncoderState* state);
|
||||
|
||||
/* The maximum input size that can be processed at once. */
|
||||
/** @deprecated Calculates maximum input size that can be processed at once. */
|
||||
BROTLI_DEPRECATED BROTLI_ENC_API size_t BrotliEncoderInputBlockSize(
|
||||
BrotliEncoderState* state);
|
||||
|
||||
/* Copies the given input data to the internal ring buffer of the compressor.
|
||||
No processing of the data occurs at this time and this function can be
|
||||
called multiple times before calling WriteBrotliData() to process the
|
||||
accumulated input. At most input_block_size() bytes of input data can be
|
||||
copied to the ring buffer, otherwise the next WriteBrotliData() will fail.
|
||||
*/
|
||||
/** @deprecated Copies the given input data to the internal ring buffer. */
|
||||
BROTLI_DEPRECATED BROTLI_ENC_API void BrotliEncoderCopyInputToRingBuffer(
|
||||
BrotliEncoderState* state, const size_t input_size,
|
||||
const uint8_t* input_buffer);
|
||||
|
||||
/* Processes the accumulated input data and sets |*out_size| to the length of
|
||||
the new output meta-block, or to zero if no new output meta-block has been
|
||||
created (in this case the processed input data is buffered internally).
|
||||
If |*out_size| is positive, |*output| points to the start of the output
|
||||
data. If |is_last| or |force_flush| is BROTLI_TRUE, an output meta-block is
|
||||
always created. However, until |is_last| is BROTLI_TRUE encoder may retain up
|
||||
to 7 bits of the last byte of output. To force encoder to dump the remaining
|
||||
bits use WriteMetadata() to append an empty meta-data block.
|
||||
Returns BROTLI_FALSE if the size of the input data is larger than
|
||||
input_block_size(). */
|
||||
/** @deprecated Processes the accumulated input. */
|
||||
BROTLI_DEPRECATED BROTLI_ENC_API BROTLI_BOOL BrotliEncoderWriteData(
|
||||
BrotliEncoderState* state, const BROTLI_BOOL is_last,
|
||||
const BROTLI_BOOL force_flush, size_t* out_size, uint8_t** output);
|
||||
|
||||
/* Fills the new state with a dictionary for LZ77, warming up the ringbuffer,
|
||||
e.g. for custom static dictionaries for data formats.
|
||||
Not to be confused with the built-in transformable dictionary of Brotli.
|
||||
To decode, use BrotliDecoderSetCustomDictionary() of the decoder with the
|
||||
same dictionary. */
|
||||
/**
|
||||
* Prepends imaginary LZ77 dictionary.
|
||||
*
|
||||
* Fills the fresh ::BrotliEncoderState with additional data corpus for LZ77
|
||||
* backward references.
|
||||
*
|
||||
* @note Not to be confused with the static dictionary (see RFC7932 section 8).
|
||||
*
|
||||
* Workflow:
|
||||
* -# Allocate and initialize state with ::BrotliEncoderCreateInstance
|
||||
* -# Set ::BROTLI_PARAM_LGWIN parameter
|
||||
* -# Invoke ::BrotliEncoderSetCustomDictionary
|
||||
* -# Use ::BrotliEncoderCompressStream
|
||||
* -# Clean up and free state with ::BrotliEncoderDestroyInstance
|
||||
*
|
||||
* @param state encoder instance
|
||||
* @param size length of @p dict; at most "window size" bytes are used
|
||||
* @param dict "dictionary"; @b MUST use same dictionary during decompression
|
||||
*/
|
||||
BROTLI_ENC_API void BrotliEncoderSetCustomDictionary(
|
||||
BrotliEncoderState* state, size_t size,
|
||||
const uint8_t dict[BROTLI_ARRAY_PARAM(size)]);
|
||||
|
||||
/* Returns buffer size that is large enough to contain BrotliEncoderCompress
|
||||
output for any input.
|
||||
CAUTION: result is not applicable to BrotliEncoderCompressStream output,
|
||||
because every "flush" adds extra overhead bytes, and some encoder settings
|
||||
(e.g. quality 0 and 1) might imply a "soft flush" after every chunk of input.
|
||||
Returns 0 if result does not fit size_t. */
|
||||
/**
|
||||
* Calculates the output size bound for the given @p input_size.
|
||||
*
|
||||
* @warning Result is not applicable to ::BrotliEncoderCompressStream output,
|
||||
* because every "flush" adds extra overhead bytes, and some encoder
|
||||
* settings (e.g. quality @c 0 and @c 1) might imply a "soft flush"
|
||||
* after every chunk of input.
|
||||
*
|
||||
* @param input_size size of projected input
|
||||
* @returns @c 0 if result does not fit @c size_t
|
||||
*/
|
||||
BROTLI_ENC_API size_t BrotliEncoderMaxCompressedSize(size_t input_size);
|
||||
|
||||
/* Compresses the data in |input_buffer| into |encoded_buffer|, and sets
|
||||
|*encoded_size| to the compressed length.
|
||||
BROTLI_DEFAULT_QUALITY, BROTLI_DEFAULT_WINDOW and BROTLI_DEFAULT_MODE should
|
||||
be used as |quality|, |lgwin| and |mode| if there are no specific
|
||||
requirements to encoder speed and compression ratio.
|
||||
If compression fails, |*encoded_size| is set to 0.
|
||||
If BrotliEncoderMaxCompressedSize(|input_size|) is not zero, then
|
||||
|*encoded_size| is never set to the bigger value.
|
||||
Returns BROTLI_FALSE if there was an error and BROTLI_TRUE otherwise. */
|
||||
/**
|
||||
* Performs one-shot memory-to-memory compression.
|
||||
*
|
||||
* Compresses the data in @p input_buffer into @p encoded_buffer, and sets
|
||||
* @p *encoded_size to the compressed length.
|
||||
*
|
||||
* @note If ::BrotliEncoderMaxCompressedSize(@p input_size) returns non-zero
|
||||
* value, then output is guaranteed to be no longer than that.
|
||||
*
|
||||
* @param quality quality parameter value, e.g. ::BROTLI_DEFAULT_QUALITY
|
||||
* @param lgwin lgwin parameter value, e.g. ::BROTLI_DEFAULT_WINDOW
|
||||
* @param mode mode parameter value, e.g. ::BROTLI_DEFAULT_MODE
|
||||
* @param input_size size of @p input_buffer
|
||||
* @param input_buffer input data buffer with at least @p input_size
|
||||
* addressable bytes
|
||||
* @param[in, out] encoded_size @b in: size of @p encoded_buffer; \n
|
||||
* @b out: length of compressed data written to
|
||||
* @p encoded_buffer, or @c 0 if compression fails
|
||||
* @param encoded_buffer compressed data destination buffer
|
||||
* @returns ::BROTLI_FALSE in case of compression error
|
||||
* @returns ::BROTLI_FALSE if output buffer is too small
|
||||
* @returns ::BROTLI_TRUE otherwise
|
||||
*/
|
||||
BROTLI_ENC_API BROTLI_BOOL BrotliEncoderCompress(
|
||||
int quality, int lgwin, BrotliEncoderMode mode, size_t input_size,
|
||||
const uint8_t input_buffer[BROTLI_ARRAY_PARAM(input_size)],
|
||||
size_t* encoded_size,
|
||||
uint8_t encoded_buffer[BROTLI_ARRAY_PARAM(*encoded_size)]);
|
||||
|
||||
/* Progressively compress input stream and push produced bytes to output stream.
|
||||
Internally workflow consists of 3 tasks:
|
||||
* (optional) copy input data to internal buffer
|
||||
* actually compress data and (optionally) store it to internal buffer
|
||||
* (optional) copy compressed bytes from internal buffer to output stream
|
||||
Whenever all 3 tasks can't move forward anymore, or error occurs, this
|
||||
method returns.
|
||||
|
||||
|available_in| and |next_in| represent input stream; when X bytes of input
|
||||
are consumed, X is subtracted from |available_in| and added to |next_in|.
|
||||
|available_out| and |next_out| represent output stream; when Y bytes are
|
||||
pushed to output, Y is subtracted from |available_out| and added to
|
||||
|next_out|. |total_out|, if it is not a null-pointer, is assigned to the
|
||||
total amount of bytes pushed by the instance of encoder to output.
|
||||
|
||||
|op| is used to perform flush or finish the stream.
|
||||
|
||||
Flushing the stream means forcing encoding of all input passed to encoder and
|
||||
completing the current output block, so it could be fully decoded by stream
|
||||
decoder. To perform flush |op| must be set to BROTLI_OPERATION_FLUSH. Under
|
||||
some circumstances (e.g. lack of output stream capacity) this operation would
|
||||
require several calls to BrotliEncoderCompressStream. The method must be
|
||||
called again until both input stream is depleted and encoder has no more
|
||||
output (see BrotliEncoderHasMoreOutput) after the method is called.
|
||||
|
||||
Finishing the stream means encoding of all input passed to encoder and
|
||||
adding specific "final" marks, so stream decoder could determine that stream
|
||||
is complete. To perform finish |op| must be set to BROTLI_OPERATION_FINISH.
|
||||
Under some circumstances (e.g. lack of output stream capacity) this operation
|
||||
would require several calls to BrotliEncoderCompressStream. The method must
|
||||
be called again until both input stream is depleted and encoder has no more
|
||||
output (see BrotliEncoderHasMoreOutput) after the method is called.
|
||||
|
||||
WARNING: when flushing and finishing, |op| should not change until operation
|
||||
is complete; input stream should not be refilled as well.
|
||||
|
||||
Returns BROTLI_FALSE if there was an error and BROTLI_TRUE otherwise.
|
||||
*/
|
||||
/**
|
||||
* Compresses input stream to output stream.
|
||||
*
|
||||
* The values @p *available_in and @p *available_out must specify the number of
|
||||
* bytes addressable at @p *next_in and @p *next_out respectively.
|
||||
* When @p *available_out is @c 0, @p next_out is allowed to be @c NULL.
|
||||
*
|
||||
* After each call, @p *available_in will be decremented by the amount of input
|
||||
* bytes consumed, and the @p *next_in pointer will be incremented by that
|
||||
* amount. Similarly, @p *available_out will be decremented by the amount of
|
||||
* output bytes written, and the @p *next_out pointer will be incremented by
|
||||
* that amount.
|
||||
*
|
||||
* @p total_out, if it is not a null-pointer, will be set to the number
|
||||
* of bytes decompressed since the last @p state initialization.
|
||||
*
|
||||
*
|
||||
*
|
||||
* Internally workflow consists of 3 tasks:
|
||||
* -# (optionally) copy input data to internal buffer
|
||||
* -# actually compress data and (optionally) store it to internal buffer
|
||||
* -# (optionally) copy compressed bytes from internal buffer to output stream
|
||||
* Whenever all 3 tasks can't move forward anymore, or error occurs, this
|
||||
* method returns the control flow to caller.
|
||||
*
|
||||
* @p op is used to perform flush, finish the stream, or inject metadata block.
|
||||
* See ::BrotliEncoderOperation for more information.
|
||||
*
|
||||
* Flushing the stream means forcing encoding of all input passed to encoder and
|
||||
* completing the current output block, so it could be fully decoded by stream
|
||||
* decoder. To perform flush set @p op to ::BROTLI_OPERATION_FLUSH.
|
||||
* Under some circumstances (e.g. lack of output stream capacity) this operation
|
||||
* would require several calls to ::BrotliEncoderCompressStream. The method must
|
||||
* be called again until both input stream is depleted and encoder has no more
|
||||
* output (see ::BrotliEncoderHasMoreOutput) after the method is called.
|
||||
*
|
||||
* Finishing the stream means encoding of all input passed to encoder and
|
||||
* adding specific "final" marks, so stream decoder could determine that stream
|
||||
* is complete. To perform finish set @p op to ::BROTLI_OPERATION_FINISH.
|
||||
* Under some circumstances (e.g. lack of output stream capacity) this operation
|
||||
* would require several calls to ::BrotliEncoderCompressStream. The method must
|
||||
* be called again until both input stream is depleted and encoder has no more
|
||||
* output (see ::BrotliEncoderHasMoreOutput) after the method is called.
|
||||
*
|
||||
* @warning When flushing and finishing, @p op should not change until operation
|
||||
* is complete; input stream should not be swapped, reduced or
|
||||
* extended as well.
|
||||
*
|
||||
* @param state encoder instance
|
||||
* @param op requested operation
|
||||
* @param[in, out] available_in @b in: amount of available input; \n
|
||||
* @b out: amount of unused input
|
||||
* @param[in, out] next_in pointer to the next input byte
|
||||
* @param[in, out] available_out @b in: length of output buffer; \n
|
||||
* @b out: remaining size of output buffer
|
||||
* @param[in, out] next_out compressed output buffer cursor;
|
||||
* can be @c NULL if @p available_out is @c 0
|
||||
* @param[out] total_out number of bytes produced so far; can be @c NULL
|
||||
* @returns ::BROTLI_FALSE if there was an error
|
||||
* @returns ::BROTLI_TRUE otherwise
|
||||
*/
|
||||
BROTLI_ENC_API BROTLI_BOOL BrotliEncoderCompressStream(
|
||||
BrotliEncoderState* s, BrotliEncoderOperation op, size_t* available_in,
|
||||
BrotliEncoderState* state, BrotliEncoderOperation op, size_t* available_in,
|
||||
const uint8_t** next_in, size_t* available_out, uint8_t** next_out,
|
||||
size_t* total_out);
|
||||
|
||||
/* Check if encoder is in "finished" state, i.e. no more input is acceptable and
|
||||
no more output will be produced.
|
||||
Works only with BrotliEncoderCompressStream workflow.
|
||||
Returns BROTLI_TRUE if stream is finished and BROTLI_FALSE otherwise. */
|
||||
BROTLI_ENC_API BROTLI_BOOL BrotliEncoderIsFinished(BrotliEncoderState* s);
|
||||
/**
|
||||
* Checks if encoder instance reached the final state.
|
||||
*
|
||||
* @param state encoder instance
|
||||
* @returns ::BROTLI_TRUE if encoder is in a state where it reached the end of
|
||||
* the input and produced all of the output
|
||||
* @returns ::BROTLI_FALSE otherwise
|
||||
*/
|
||||
BROTLI_ENC_API BROTLI_BOOL BrotliEncoderIsFinished(BrotliEncoderState* state);
|
||||
|
||||
/* Check if encoder has more output bytes in internal buffer.
|
||||
Works only with BrotliEncoderCompressStream workflow.
|
||||
Returns BROTLI_TRUE if has more output (in internal buffer) and BROTLI_FALSE
|
||||
otherwise. */
|
||||
BROTLI_ENC_API BROTLI_BOOL BrotliEncoderHasMoreOutput(BrotliEncoderState* s);
|
||||
/**
|
||||
* Checks if encoder has more output.
|
||||
*
|
||||
* @param state encoder instance
|
||||
* @returns ::BROTLI_TRUE, if encoder has some unconsumed output
|
||||
* @returns ::BROTLI_FALSE otherwise
|
||||
*/
|
||||
BROTLI_ENC_API BROTLI_BOOL BrotliEncoderHasMoreOutput(
|
||||
BrotliEncoderState* state);
|
||||
|
||||
/* Returns pointer to internal output buffer.
|
||||
Set |size| to zero, to request all the continous output produced by encoder
|
||||
so far. Otherwise no more than |size| bytes output will be provided.
|
||||
After return |size| contains the size of output buffer region available for
|
||||
reading. |size| bytes of output are considered consumed for all consecutive
|
||||
calls to the instance methods; returned pointer becomes invalidated as well.
|
||||
|
||||
This method is created to make language bindings easier and more efficient:
|
||||
1. push input data to CompressStream, until HasMoreOutput becomes true
|
||||
2. use TakeOutput to peek bytes and copy to language-specific entity
|
||||
Also this could be useful if there is an output stream that is able to
|
||||
consume all the provided data (e.g. when data is saved to file system).
|
||||
/**
|
||||
* Acquires pointer to internal output buffer.
|
||||
*
|
||||
* This method is used to make language bindings easier and more efficient:
|
||||
* -# push data to ::BrotliEncoderCompressStream,
|
||||
* until ::BrotliEncoderHasMoreOutput returns BROTL_TRUE
|
||||
* -# use ::BrotliEncoderTakeOutput to peek bytes and copy to language-specific
|
||||
* entity
|
||||
*
|
||||
* Also this could be useful if there is an output stream that is able to
|
||||
* consume all the provided data (e.g. when data is saved to file system).
|
||||
*
|
||||
* @attention After every call to ::BrotliEncoderTakeOutput @p *size bytes of
|
||||
* output are considered consumed for all consecutive calls to the
|
||||
* instance methods; returned pointer becomes invalidated as well.
|
||||
*
|
||||
* @note Encoder output is not guaranteed to be contiguous. This means that
|
||||
* after the size-unrestricted call to ::BrotliEncoderTakeOutput,
|
||||
* immediate next call to ::BrotliEncoderTakeOutput may return more data.
|
||||
*
|
||||
* @param state encoder instance
|
||||
* @param[in, out] size @b in: number of bytes caller is ready to take, @c 0 if
|
||||
* any amount could be handled; \n
|
||||
* @b out: amount of data pointed by returned pointer and
|
||||
* considered consumed; \n
|
||||
* out value is never greater than in value, unless it is @c 0
|
||||
* @returns pointer to output data
|
||||
*/
|
||||
BROTLI_ENC_API const uint8_t* BrotliEncoderTakeOutput(
|
||||
BrotliEncoderState* s, size_t* size);
|
||||
BrotliEncoderState* state, size_t* size);
|
||||
|
||||
|
||||
/* Encoder version. Look at BROTLI_VERSION for more information. */
|
||||
/**
|
||||
* Gets an encoder library version.
|
||||
*
|
||||
* Look at BROTLI_VERSION for more information.
|
||||
*/
|
||||
BROTLI_ENC_API uint32_t BrotliEncoderVersion(void);
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
|
@ -4,7 +4,10 @@
|
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/
|
||||
|
||||
/* Common types */
|
||||
/**
|
||||
* @file
|
||||
* Common types used in decoder and encoder API.
|
||||
*/
|
||||
|
||||
#ifndef BROTLI_COMMON_TYPES_H_
|
||||
#define BROTLI_COMMON_TYPES_H_
|
||||
@ -24,15 +27,31 @@ typedef __int64 int64_t;
|
||||
#include <stdint.h>
|
||||
#endif /* defined(_MSC_VER) && (_MSC_VER < 1600) */
|
||||
|
||||
/* BROTLI_BOOL is a portable "bool" replacement. For input parameters it is
|
||||
preferrable either use BROTLI_TRUE and BROTLI_FALSE macros, or convert
|
||||
boolean expression with TO_BROTLI_BOOL macros.
|
||||
Return values should not be tested for equality with "true", "false",
|
||||
"BROTLI_TRUE", "BROTLI_FALSE", but rather be evaluated, for example:
|
||||
`if (foo(enc) && !bar(dec) { bool x = !!baz(enc); }` */
|
||||
/**
|
||||
* A portable @c bool replacement.
|
||||
*
|
||||
* ::BROTLI_BOOL is a "documentation" type: actually it is @c int, but in API it
|
||||
* denotes a type, whose only values are ::BROTLI_TRUE and ::BROTLI_FALSE.
|
||||
*
|
||||
* ::BROTLI_BOOL values passed to Brotli should either be ::BROTLI_TRUE or
|
||||
* ::BROTLI_FALSE, or be a result of ::TO_BROTLI_BOOL macros.
|
||||
*
|
||||
* ::BROTLI_BOOL values returned by Brotli should not be tested for equality
|
||||
* with @c true, @c false, ::BROTLI_TRUE, ::BROTLI_FALSE, but rather should be
|
||||
* evaluated, for example: @code{.cpp}
|
||||
* if (SomeBrotliFunction(encoder, BROTLI_TRUE) &&
|
||||
* !OtherBrotliFunction(decoder, BROTLI_FALSE)) {
|
||||
* bool x = !!YetAnotherBrotliFunction(encoder, TO_BROLTI_BOOL(2 * 2 == 4));
|
||||
* DoSometing(x);
|
||||
* }
|
||||
* @endcode
|
||||
*/
|
||||
#define BROTLI_BOOL int
|
||||
/** Portable @c true replacement. */
|
||||
#define BROTLI_TRUE 1
|
||||
/** Portable @c false replacement. */
|
||||
#define BROTLI_FALSE 0
|
||||
/** @c bool to ::BROTLI_BOOL conversion macros. */
|
||||
#define TO_BROTLI_BOOL(X) (!!(X) ? BROTLI_TRUE : BROTLI_FALSE)
|
||||
|
||||
#define BROTLI_MAKE_UINT64_T(high, low) ((((uint64_t)(high)) << 32) | low)
|
||||
@ -40,15 +59,25 @@ typedef __int64 int64_t;
|
||||
#define BROTLI_UINT32_MAX (~((uint32_t)0))
|
||||
#define BROTLI_SIZE_MAX (~((size_t)0))
|
||||
|
||||
/* Allocating function pointer. Function MUST return 0 in the case of failure.
|
||||
Otherwise it MUST return a valid pointer to a memory region of at least
|
||||
size length. Neither items nor size are allowed to be 0.
|
||||
opaque argument is a pointer provided by client and could be used to bind
|
||||
function to specific object (memory pool). */
|
||||
/**
|
||||
* Allocating function pointer type.
|
||||
*
|
||||
* @param opaque custom memory manager handle provided by client
|
||||
* @param size requested memory region size; can not be @c 0
|
||||
* @returns @c 0 in the case of failure
|
||||
* @returns a valid pointer to a memory region of at least @p size bytes
|
||||
* long otherwise
|
||||
*/
|
||||
typedef void* (*brotli_alloc_func)(void* opaque, size_t size);
|
||||
|
||||
/* Deallocating function pointer. Function SHOULD be no-op in the case the
|
||||
address is 0. */
|
||||
/**
|
||||
* Deallocating function pointer type.
|
||||
*
|
||||
* This function @b SHOULD do nothing if @p address is @c 0.
|
||||
*
|
||||
* @param opaque custom memory manager handle provided by client
|
||||
* @param address memory region pointer returned by ::brotli_alloc_func, or @c 0
|
||||
*/
|
||||
typedef void (*brotli_free_func)(void* opaque, void* address);
|
||||
|
||||
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \
|
||||
|
@ -14,10 +14,22 @@ java_binary(
|
||||
)
|
||||
|
||||
java_test(
|
||||
name = "bundle_checker_test",
|
||||
name = "bundle_checker_data_test",
|
||||
args = ["java/integration/test_data.zip"],
|
||||
data = ["test_data.zip"],
|
||||
main_class = "org.brotli.integration.BundleChecker",
|
||||
use_testrunner = 0,
|
||||
runtime_deps = [":bundle_checker_lib"],
|
||||
)
|
||||
|
||||
java_test(
|
||||
name = "bundle_checker_fuzz_test",
|
||||
args = [
|
||||
"-s",
|
||||
"java/integration/fuzz_data.zip"
|
||||
],
|
||||
data = ["fuzz_data.zip"],
|
||||
main_class = "org.brotli.integration.BundleChecker",
|
||||
use_testrunner = 0,
|
||||
runtime_deps = [":bundle_checker_lib"],
|
||||
)
|
||||
|
@ -18,7 +18,7 @@ import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
|
||||
/**
|
||||
* Decompress files and checks thier checksums.
|
||||
* Decompress files and (optionally) checks their checksums.
|
||||
*
|
||||
* <p> File are read from ZIP archive passed as an array of bytes. Multiple checkers negotiate about
|
||||
* task distribution via shared AtomicInteger counter.
|
||||
@ -28,10 +28,15 @@ import java.util.zip.ZipInputStream;
|
||||
public class BundleChecker implements Runnable {
|
||||
final AtomicInteger nextJob;
|
||||
final InputStream input;
|
||||
final boolean sanityCheck;
|
||||
|
||||
public BundleChecker(InputStream input, AtomicInteger nextJob) {
|
||||
/**
|
||||
* @param sanityCheck do not calculate checksum and ignore {@link IOException}.
|
||||
*/
|
||||
public BundleChecker(InputStream input, AtomicInteger nextJob, boolean sanityCheck) {
|
||||
this.input = input;
|
||||
this.nextJob = nextJob;
|
||||
this.sanityCheck = sanityCheck;
|
||||
}
|
||||
|
||||
/** ECMA CRC64 polynomial. */
|
||||
@ -93,11 +98,18 @@ public class BundleChecker implements Runnable {
|
||||
continue;
|
||||
}
|
||||
entryName = entry.getName();
|
||||
String entryCrcString = entryName.substring(0, entryName.indexOf('.'));
|
||||
int dotIndex = entryName.indexOf('.');
|
||||
String entryCrcString = (dotIndex == -1) ? entryName : entryName.substring(0, dotIndex);
|
||||
long entryCrc = new BigInteger(entryCrcString, 16).longValue();
|
||||
if (entryCrc != decompressAndCalculateCrc(zis)) {
|
||||
try {
|
||||
if (entryCrc != decompressAndCalculateCrc(zis) && !sanityCheck) {
|
||||
throw new RuntimeException("CRC mismatch");
|
||||
}
|
||||
} catch (IOException iox) {
|
||||
if (!sanityCheck) {
|
||||
throw new RuntimeException("Decompression failed", iox);
|
||||
}
|
||||
}
|
||||
zis.closeEntry();
|
||||
entryName = "";
|
||||
jobIndex = nextJob.getAndIncrement();
|
||||
@ -110,11 +122,19 @@ public class BundleChecker implements Runnable {
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws FileNotFoundException {
|
||||
if (args.length == 0) {
|
||||
throw new RuntimeException("Usage: BundleChecker <fileX.zip> ...");
|
||||
int argsOffset = 0;
|
||||
boolean sanityCheck = false;
|
||||
if (args.length != 0) {
|
||||
if (args[0].equals("-s")) {
|
||||
sanityCheck = true;
|
||||
argsOffset = 1;
|
||||
}
|
||||
for (int i = 0; i < args.length; ++i) {
|
||||
new BundleChecker(new FileInputStream(args[i]), new AtomicInteger(0)).run();
|
||||
}
|
||||
if (args.length == argsOffset) {
|
||||
throw new RuntimeException("Usage: BundleChecker [-s] <fileX.zip> ...");
|
||||
}
|
||||
for (int i = argsOffset; i < args.length; ++i) {
|
||||
new BundleChecker(new FileInputStream(args[i]), new AtomicInteger(0), sanityCheck).run();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
BIN
java/integration/fuzz_data.zip
Executable file
BIN
java/integration/fuzz_data.zip
Executable file
Binary file not shown.
@ -39,6 +39,7 @@
|
||||
<artifactId>exec-maven-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>data</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>java</goal>
|
||||
@ -51,6 +52,21 @@
|
||||
</arguments>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>fuzz</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>java</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<executable>java</executable>
|
||||
<mainClass>org.brotli.integration.BundleChecker</mainClass>
|
||||
<arguments>
|
||||
<argument>-s</argument>
|
||||
<argument>fuzz_data.zip</argument>
|
||||
</arguments>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
|
Loading…
Reference in New Issue
Block a user