AuroraMiddleware/brotli
1
0
Fork 0
mirror of https://github.com/google/brotli.git synced 2024-11-24 20:40:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update API, and more (#581)

Browse Source 
Update API, and more:
 * remove "custom dictionary" support
 * c/encoder: fix #580: big-endian build
 * Java: reduce jar size
 * Java: speedup decoding
 * Java: add 32-bit CPU support
 * Java: make source code JS transpiler-ready
This commit is contained in:
Eugene Kliuchnikov 2017-08-04 10:02:56 +02:00 committed by GitHub
parent 0608253110
commit d63e8f75f5
65 changed files with 1061 additions and 2086 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
c/common/constants.h
View File

@ -38,6 +38,8 @@
BROTLI_MAX_NDIRECT + \
(BROTLI_MAX_DISTANCE_BITS << \
(BROTLI_MAX_NPOSTFIX + 1)))
/* Distance that is guaranteed to be representable in any stream. */
#define BROTLI_MAX_DISTANCE 0x3FFFFFC
/* 7.1. Context modes and context ID lookup for literals */
/* "context IDs for literals are in the range of 0..63" */

35
c/dec/decode.c
View File

@ -1226,7 +1226,9 @@ static BrotliDecoderErrorCode BROTLI_NOINLINE WriteRingBuffer(
BROTLI_LOG_UINT(to_write);
BROTLI_LOG_UINT(num_written);
s->partial_pos_out += num_written;
if (total_out) *total_out = s->partial_pos_out - (size_t)s->custom_dict_size;
if (total_out) {
*total_out = s->partial_pos_out;
}
if (num_written < to_write) {
if (s->ringbuffer_size == (1 << s->window_bits) || force) {
return BROTLI_DECODER_NEEDS_MORE_OUTPUT;
@ -1278,13 +1280,7 @@ static BROTLI_BOOL BROTLI_NOINLINE BrotliEnsureRingBuffer(
s->ringbuffer[s->new_ringbuffer_size - 2] = 0;
s->ringbuffer[s->new_ringbuffer_size - 1] = 0;
if (!old_ringbuffer) {
if (s->custom_dict) {
memcpy(s->ringbuffer, s->custom_dict, (size_t)s->custom_dict_size);
s->partial_pos_out = (size_t)s->custom_dict_size;
s->pos = s->custom_dict_size;
}
} else {
if (!!old_ringbuffer) {
memcpy(s->ringbuffer, old_ringbuffer, (size_t)s->pos);
BROTLI_FREE(s, old_ringbuffer);
}
@ -1373,8 +1369,7 @@ static void BROTLI_NOINLINE BrotliCalculateRingBufferSize(
}
if (!s->ringbuffer) {
/* Custom dictionary counts as a "virtual" output. */
output_size = s->custom_dict_size;
output_size = 0;
} else {
output_size = s->pos;
}
@ -1752,14 +1747,14 @@ CommandPostDecodeLiterals:
/* Apply copy of LZ77 back-reference, or static dictionary reference if
the distance is larger than the max LZ77 distance */
if (s->distance_code > s->max_distance) {
int address = s->distance_code - s->max_distance - 1;
if (i >= BROTLI_MIN_DICTIONARY_WORD_LENGTH &&
i <= BROTLI_MAX_DICTIONARY_WORD_LENGTH) {
int offset = (int)s->dictionary->offsets_by_length[i];
int word_id = s->distance_code - s->max_distance - 1;
uint32_t shift = s->dictionary->size_bits_by_length[i];
int mask = (int)BitMask(shift);
int word_idx = word_id & mask;
int transform_idx = word_id >> shift;
int word_idx = address & mask;
int transform_idx = address >> shift;
/* Compensate double distance-ring-buffer roll. */
s->dist_rb_idx += s->distance_context;
offset += word_idx * i;
@ -2022,11 +2017,6 @@ BrotliDecoderResult BrotliDecoderDecompressStream(
}
/* Maximum distance, see section 9.1. of the spec. */
s->max_backward_distance = (1 << s->window_bits) - BROTLI_WINDOW_GAP;
/* Limit custom dictionary size. */
if (s->custom_dict_size >= s->max_backward_distance) {
s->custom_dict += s->custom_dict_size - s->max_backward_distance;
s->custom_dict_size = s->max_backward_distance;
}
/* Allocate memory for both block_type_trees and block_len_trees. */
s->block_type_trees = (HuffmanCode*)BROTLI_ALLOC(s,
@ -2323,15 +2313,6 @@ BrotliDecoderResult BrotliDecoderDecompressStream(
return SaveErrorCode(s, result);
}
void BrotliDecoderSetCustomDictionary(
BrotliDecoderState* s, size_t size, const uint8_t* dict) {
if (size > (1u << 24)) {
return;
}
s->custom_dict = dict;
s->custom_dict_size = (int)size;
}
BROTLI_BOOL BrotliDecoderHasMoreOutput(const BrotliDecoderState* s) {
/* After unrecoverable error remaining output is considered nonsensical. */
if ((int)s->error_code < 0) {

3
c/dec/state.c
View File

@ -83,9 +83,6 @@ void BrotliDecoderStateInitWithCustomAllocators(BrotliDecoderState* s,
s->distance_hgroup.codes = NULL;
s->distance_hgroup.htrees = NULL;
s->custom_dict = NULL;
s->custom_dict_size = 0;
s->is_last_metablock = 0;
s->is_uncompressed = 0;
s->is_metadata = 0;

4
c/dec/state.h
View File

@ -197,10 +197,6 @@ struct BrotliDecoderStateStruct {
uint32_t mtf_upper_bound;
uint32_t mtf[64 + 1];
/* For custom dictionaries */
const uint8_t* custom_dict;
int custom_dict_size;
/* less used attributes are in the end of this struct */
/* States inside function calls */
BrotliRunningMetablockHeaderState substate_metablock_header;

8
c/enc/backward_references.c
View File

@ -48,6 +48,8 @@ static BROTLI_INLINE size_t ComputeDistanceCode(size_t distance,
#define EXPAND_CAT(a, b) CAT(a, b)
#define CAT(a, b) a ## b
#define FN(X) EXPAND_CAT(X, HASHER())
#define EXPORT_FN(X) EXPAND_CAT(X, EXPAND_CAT(PREFIX(), HASHER()))
#define PREFIX() N
#define HASHER() H2
/* NOLINTNEXTLINE(build/include) */
@ -94,6 +96,8 @@ static BROTLI_INLINE size_t ComputeDistanceCode(size_t distance,
#include "./backward_references_inc.h"
#undef HASHER
#undef PREFIX
#undef EXPORT_FN
#undef FN
#undef CAT
#undef EXPAND_CAT
@ -113,11 +117,11 @@ void BrotliCreateBackwardReferences(const BrotliDictionary* dictionary,
switch (params->hasher.type) {
#define CASE_(N) \
case N: \
CreateBackwardReferencesH ## N(dictionary, \
CreateBackwardReferencesNH ## N(dictionary, \
kStaticDictionaryHash, num_bytes, position, ringbuffer, \
ringbuffer_mask, params, hasher, dist_cache, \
last_insert_len, commands, num_commands, num_literals); \
break;
return;
FOR_GENERIC_HASHERS(CASE_)
#undef CASE_
default:

7
c/enc/backward_references_hq.c
View File

@ -629,6 +629,7 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m,
const size_t store_end = num_bytes >= StoreLookaheadH10() ?
position + num_bytes - StoreLookaheadH10() + 1 : position;
size_t i;
size_t gap = 0;
nodes[0].length = 0;
nodes[0].u.cost = 0;
InitZopfliCostModel(m, &model, num_bytes);
@ -640,7 +641,8 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m,
const size_t pos = position + i;
const size_t max_distance = BROTLI_MIN(size_t, pos, max_backward_limit);
size_t num_matches = FindAllMatchesH10(hasher, dictionary, ringbuffer,
ringbuffer_mask, pos, num_bytes - i, max_distance, params, matches);
ringbuffer_mask, pos, num_bytes - i, max_distance, gap, params,
matches);
size_t skip;
if (num_matches > 0 &&
BackwardMatchLength(&matches[num_matches - 1]) > max_zopfli_len) {
@ -712,6 +714,7 @@ void BrotliCreateHqZopfliBackwardReferences(
ZopfliCostModel model;
ZopfliNode* nodes;
BackwardMatch* matches = BROTLI_ALLOC(m, BackwardMatch, matches_size);
size_t gap = 0;
if (BROTLI_IS_OOM(m)) return;
for (i = 0; i + HashTypeLengthH10() - 1 < num_bytes; ++i) {
const size_t pos = position + i;
@ -725,7 +728,7 @@ void BrotliCreateHqZopfliBackwardReferences(
cur_match_pos + MAX_NUM_MATCHES_H10);
if (BROTLI_IS_OOM(m)) return;
num_found_matches = FindAllMatchesH10(hasher, dictionary, ringbuffer,
ringbuffer_mask, pos, max_length, max_distance, params,
ringbuffer_mask, pos, max_length, max_distance, gap, params,
&matches[cur_match_pos]);
cur_match_end = cur_match_pos + num_found_matches;
for (j = cur_match_pos; j + 1 < cur_match_end; ++j) {

17
c/enc/backward_references_inc.h
View File

@ -5,11 +5,11 @@
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* template parameters: FN */
/* template parameters: EXPORT_FN, FN */
static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
const BrotliDictionary* dictionary, const uint16_t* dictionary_hash,
size_t num_bytes, size_t position,
static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
const BrotliDictionary* dictionary,
const uint16_t* dictionary_hash, size_t num_bytes, size_t position,
const uint8_t* ringbuffer, size_t ringbuffer_mask,
const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
size_t* last_insert_len, Command* commands, size_t* num_commands,
@ -27,6 +27,7 @@ static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
const size_t random_heuristics_window_size =
LiteralSpreeLengthForSparseSearch(params);
size_t apply_random_heuristics = position + random_heuristics_window_size;
const size_t gap = 0;
/* Minimum score to accept a backward reference. */
const score_t kMinScore = BROTLI_SCORE_BASE + 100;
@ -43,7 +44,7 @@ static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
sr.score = kMinScore;
FN(FindLongestMatch)(hasher, dictionary, dictionary_hash, ringbuffer,
ringbuffer_mask, dist_cache, position,
max_length, max_distance, &sr);
max_length, max_distance, gap, &sr);
if (sr.score > kMinScore) {
/* Found a match. Let's look for something even better ahead. */
int delayed_backward_references_in_row = 0;
@ -59,7 +60,7 @@ static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
max_distance = BROTLI_MIN(size_t, position + 1, max_backward_limit);
FN(FindLongestMatch)(hasher, dictionary, dictionary_hash, ringbuffer,
ringbuffer_mask, dist_cache, position + 1,
max_length, max_distance, &sr2);
max_length, max_distance, gap, &sr2);
if (sr2.score >= sr.score + cost_diff_lazy) {
/* Ok, let's just write one byte for now and start a match from the
next byte. */
@ -80,8 +81,8 @@ static BROTLI_NOINLINE void FN(CreateBackwardReferences)(
/* The first 16 codes are special short-codes,
and the minimum offset is 1. */
size_t distance_code =
ComputeDistanceCode(sr.distance, max_distance, dist_cache);
if (sr.distance <= max_distance && distance_code > 0) {
ComputeDistanceCode(sr.distance, max_distance + gap, dist_cache);
if ((sr.distance <= (max_distance + gap)) && distance_code > 0) {
dist_cache[3] = dist_cache[2];
dist_cache[2] = dist_cache[1];
dist_cache[1] = dist_cache[0];

6
c/enc/compress_fragment.c
View File

@ -42,7 +42,7 @@ extern "C" {
static const uint32_t kHashMul32 = 0x1e35a7bd;
static BROTLI_INLINE uint32_t Hash(const uint8_t* p, size_t shift) {
const uint64_t h = (BROTLI_UNALIGNED_LOAD64(p) << 24) * kHashMul32;
const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(p) << 24) * kHashMul32;
return (uint32_t)(h >> shift);
}
@ -603,7 +603,7 @@ trawl:
compression we first update "table" with the hashes of some positions
within the last copy. */
{
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 3);
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 3);
uint32_t prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
uint32_t cur_hash = HashBytesAtOffset(input_bytes, 3, shift);
table[prev_hash] = (int)(ip - base_ip - 3);
@ -640,7 +640,7 @@ trawl:
compression we first update "table" with the hashes of some positions
within the last copy. */
{
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 3);
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 3);
uint32_t prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
uint32_t cur_hash = HashBytesAtOffset(input_bytes, 3, shift);
table[prev_hash] = (int)(ip - base_ip - 3);

10
c/enc/compress_fragment_two_pass.c
View File

@ -41,7 +41,7 @@ extern "C" {
static const uint32_t kHashMul32 = 0x1e35a7bd;
static BROTLI_INLINE uint32_t Hash(const uint8_t* p, size_t shift) {
const uint64_t h = (BROTLI_UNALIGNED_LOAD64(p) << 16) * kHashMul32;
const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(p) << 16) * kHashMul32;
return (uint32_t)(h >> shift);
}
@ -346,7 +346,7 @@ trawl:
/* We could immediately start working at ip now, but to improve
compression we first update "table" with the hashes of some
positions within the last copy. */
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 5);
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 5);
uint32_t prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
uint32_t cur_hash;
table[prev_hash] = (int)(ip - base_ip - 5);
@ -354,7 +354,7 @@ trawl:
table[prev_hash] = (int)(ip - base_ip - 4);
prev_hash = HashBytesAtOffset(input_bytes, 2, shift);
table[prev_hash] = (int)(ip - base_ip - 3);
input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 2);
input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 2);
cur_hash = HashBytesAtOffset(input_bytes, 2, shift);
prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
table[prev_hash] = (int)(ip - base_ip - 2);
@ -386,7 +386,7 @@ trawl:
/* We could immediately start working at ip now, but to improve
compression we first update "table" with the hashes of some
positions within the last copy. */
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 5);
uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 5);
uint32_t prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
uint32_t cur_hash;
table[prev_hash] = (int)(ip - base_ip - 5);
@ -394,7 +394,7 @@ trawl:
table[prev_hash] = (int)(ip - base_ip - 4);
prev_hash = HashBytesAtOffset(input_bytes, 2, shift);
table[prev_hash] = (int)(ip - base_ip - 3);
input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 2);
input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 2);
cur_hash = HashBytesAtOffset(input_bytes, 2, shift);
prev_hash = HashBytesAtOffset(input_bytes, 0, shift);
table[prev_hash] = (int)(ip - base_ip - 2);

30
c/enc/encode.c
View File

@ -832,36 +832,6 @@ static void CopyInputToRingBuffer(BrotliEncoderState* s,
}
}
void BrotliEncoderSetCustomDictionary(BrotliEncoderState* s, size_t size,
const uint8_t* dict) {
size_t max_dict_size = BROTLI_MAX_BACKWARD_LIMIT(s->params.lgwin);
size_t dict_size = size;
MemoryManager* m = &s->memory_manager_;
if (!EnsureInitialized(s)) return;
if (dict_size == 0 ||
s->params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY ||
s->params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY) {
return;
}
if (size > max_dict_size) {
dict += size - max_dict_size;
dict_size = max_dict_size;
}
CopyInputToRingBuffer(s, dict_size, dict);
s->last_flush_pos_ = dict_size;
s->last_processed_pos_ = dict_size;
if (dict_size > 0) {
s->prev_byte_ = dict[dict_size - 1];
}
if (dict_size > 1) {
s->prev_byte2_ = dict[dict_size - 2];
}
HasherPrependCustomDictionary(m, &s->hasher_, &s->params, dict_size, dict);
if (BROTLI_IS_OOM(m)) return;
}
/* Marks all input as processed.
Returns true if position wrapping occurs. */
static BROTLI_BOOL UpdateLastProcessedPos(BrotliEncoderState* s) {

10
c/enc/find_match_length.h
View File

@ -17,7 +17,7 @@ extern "C" {
#endif
/* Separate implementation for little-endian 64-bit targets, for speed. */
#if defined(__GNUC__) && defined(_LP64) && defined(IS_LITTLE_ENDIAN)
#if defined(__GNUC__) && defined(_LP64) && defined(BROTLI_LITTLE_ENDIAN)
static BROTLI_INLINE size_t FindMatchLengthWithLimit(const uint8_t* s1,
const uint8_t* s2,
@ -25,13 +25,13 @@ static BROTLI_INLINE size_t FindMatchLengthWithLimit(const uint8_t* s1,
size_t matched = 0;
size_t limit2 = (limit >> 3) + 1; /* + 1 is for pre-decrement in while */
while (BROTLI_PREDICT_TRUE(--limit2)) {
if (BROTLI_PREDICT_FALSE(BROTLI_UNALIGNED_LOAD64(s2) ==
BROTLI_UNALIGNED_LOAD64(s1 + matched))) {
if (BROTLI_PREDICT_FALSE(BROTLI_UNALIGNED_LOAD64LE(s2) ==
BROTLI_UNALIGNED_LOAD64LE(s1 + matched))) {
s2 += 8;
matched += 8;
} else {
uint64_t x =
BROTLI_UNALIGNED_LOAD64(s2) ^ BROTLI_UNALIGNED_LOAD64(s1 + matched);
uint64_t x = BROTLI_UNALIGNED_LOAD64LE(s2) ^
BROTLI_UNALIGNED_LOAD64LE(s1 + matched);
size_t matching_bits = (size_t)__builtin_ctzll(x);
matched += matching_bits >> 3;
return matched;

27
c/enc/hash.h
View File

@ -173,6 +173,9 @@ static BROTLI_INLINE BROTLI_BOOL TestStaticDictionaryItem(
backward = max_backward + dist + 1 +
(transform_id << dictionary->size_bits_by_length[len]);
}
if (backward >= BROTLI_MAX_DISTANCE) {
return BROTLI_FALSE;
}
score = BackwardReferenceScore(matchlen, backward);
if (score < out->score) {
return BROTLI_FALSE;
@ -417,30 +420,6 @@ static BROTLI_INLINE void HasherSetup(MemoryManager* m, HasherHandle* handle,
}
}
/* Custom LZ77 window. */
static BROTLI_INLINE void HasherPrependCustomDictionary(
MemoryManager* m, HasherHandle* handle, BrotliEncoderParams* params,
const size_t size, const uint8_t* dict) {
size_t overlap;
size_t i;
HasherHandle self;
HasherSetup(m, handle, params, dict, 0, size, BROTLI_FALSE);
if (BROTLI_IS_OOM(m)) return;
self = *handle;
switch (GetHasherCommon(self)->params.type) {
#define PREPEND_(N) \
case N: \
overlap = (StoreLookaheadH ## N()) - 1; \
for (i = 0; i + overlap < size; i++) { \
StoreH ## N(self, dict, ~(size_t)0, i); \
} \
break;
FOR_ALL_HASHERS(PREPEND_)
#undef PREPEND_
default: break;
}
}
static BROTLI_INLINE void InitOrStitchToPreviousBlock(
MemoryManager* m, HasherHandle* handle, const uint8_t* data, size_t mask,
BrotliEncoderParams* params, size_t position, size_t input_size,

4
c/enc/hash_forgetful_chain_inc.h
View File

@ -158,7 +158,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache,
const size_t cur_ix, const size_t max_length, const size_t max_backward,
HasherSearchResult* BROTLI_RESTRICT out) {
const size_t gap, HasherSearchResult* BROTLI_RESTRICT out) {
HashForgetfulChain* self = FN(Self)(handle);
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
/* Don't accept a short copy from far away. */
@ -241,7 +241,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
}
if (out->score == min_score) {
SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward, out,
handle, &data[cur_ix_masked], max_length, max_backward + gap, out,
BROTLI_FALSE);
}
}

6
c/enc/hash_longest_match64_inc.h
View File

@ -23,7 +23,7 @@ static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 8; }
static BROTLI_INLINE uint32_t FN(HashBytes)(const uint8_t *data,
const uint64_t mask,
const int shift) {
const uint64_t h = (BROTLI_UNALIGNED_LOAD64(data) & mask) * kHashMul64Long;
const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(data) & mask) * kHashMul64Long;
/* The higher bits contain more mixture from the multiplication,
so we take our results from there. */
return (uint32_t)(h >> shift);
@ -161,7 +161,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
const BrotliDictionary* dictionary, const uint16_t* dictionary_hash,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
const size_t max_length, const size_t max_backward,
const size_t max_length, const size_t max_backward, const size_t gap,
HasherSearchResult* BROTLI_RESTRICT out) {
HasherCommon* common = GetHasherCommon(handle);
HashLongestMatch* self = FN(Self)(handle);
@ -258,7 +258,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
}
if (min_score == out->score) {
SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward, out,
handle, &data[cur_ix_masked], max_length, max_backward + gap, out,
BROTLI_FALSE);
}
}

4
c/enc/hash_longest_match_inc.h
View File

@ -154,7 +154,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
const BrotliDictionary* dictionary, const uint16_t* dictionary_hash,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
const size_t max_length, const size_t max_backward,
const size_t max_length, const size_t max_backward, const size_t gap,
HasherSearchResult* BROTLI_RESTRICT out) {
HasherCommon* common = GetHasherCommon(handle);
HashLongestMatch* self = FN(Self)(handle);
@ -250,7 +250,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
}
if (min_score == out->score) {
SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward, out,
handle, &data[cur_ix_masked], max_length, max_backward + gap, out,
BROTLI_FALSE);
}
}

6
c/enc/hash_longest_match_quickly_inc.h
View File

@ -22,7 +22,7 @@ static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 8; }
the address in. The HashLongestMatch and HashLongestMatchQuickly
classes have separate, different implementations of hashing. */
static uint32_t FN(HashBytes)(const uint8_t* data) {
const uint64_t h = ((BROTLI_UNALIGNED_LOAD64(data) << (64 - 8 * HASH_LEN)) *
const uint64_t h = ((BROTLI_UNALIGNED_LOAD64LE(data) << (64 - 8 * HASH_LEN)) *
kHashMul64);
/* The higher bits contain more mixture from the multiplication,
so we take our results from there. */
@ -129,7 +129,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
const uint16_t* dictionary_hash, const uint8_t* BROTLI_RESTRICT data,
const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache,
const size_t cur_ix, const size_t max_length, const size_t max_backward,
HasherSearchResult* BROTLI_RESTRICT out) {
const size_t gap, HasherSearchResult* BROTLI_RESTRICT out) {
HashLongestMatchQuickly* self = FN(Self)(handle);
const size_t best_len_in = out->len;
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
@ -222,7 +222,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
}
if (USE_DICTIONARY && min_score == out->score) {
SearchInStaticDictionary(dictionary, dictionary_hash,
handle, &data[cur_ix_masked], max_length, max_backward, out,
handle, &data[cur_ix_masked], max_length, max_backward + gap, out,
BROTLI_TRUE);
}
self->buckets_[key + ((cur_ix >> 3) % BUCKET_SWEEP)] = (uint32_t)cur_ix;

12
c/enc/hash_to_binary_tree_inc.h
View File

@ -19,8 +19,8 @@
#define BUCKET_SIZE (1 << BUCKET_BITS)
static size_t FN(HashTypeLength)(void) { return 4; }
static size_t FN(StoreLookahead)(void) { return MAX_TREE_COMP_LENGTH; }
static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 4; }
static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return MAX_TREE_COMP_LENGTH; }
static uint32_t FN(HashBytes)(const uint8_t *data) {
uint32_t h = BROTLI_UNALIGNED_LOAD32(data) * kHashMul32;
@ -199,7 +199,7 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
const BrotliDictionary* dictionary, const uint8_t* data,
const size_t ring_buffer_mask, const size_t cur_ix,
const size_t max_length, const size_t max_backward,
const size_t max_length, const size_t max_backward, const size_t gap,
const BrotliEncoderParams* params, BackwardMatch* matches) {
BackwardMatch* const orig_matches = matches;
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
@ -248,8 +248,10 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
for (l = minlen; l <= maxlen; ++l) {
uint32_t dict_id = dict_matches[l];
if (dict_id < kInvalidMatch) {
InitDictionaryBackwardMatch(matches++,
max_backward + (dict_id >> 5) + 1, l, dict_id & 31);
size_t distance = max_backward + gap + (dict_id >> 5) + 1;
if (distance < BROTLI_MAX_DISTANCE) {
InitDictionaryBackwardMatch(matches++, distance, l, dict_id & 31);
}
}
}
}

64
c/enc/port.h
View File

@ -26,36 +26,37 @@
#define __LITTLE_ENDIAN LITTLE_ENDIAN
#endif
/* define the macro IS_LITTLE_ENDIAN
/* define the macro BROTLI_LITTLE_ENDIAN
using the above endian definitions from endian.h if
endian.h was included */
#ifdef __BYTE_ORDER
#if __BYTE_ORDER == __LITTLE_ENDIAN
#define IS_LITTLE_ENDIAN
#define BROTLI_LITTLE_ENDIAN
#endif
#else
#if defined(__LITTLE_ENDIAN__)
#define IS_LITTLE_ENDIAN
#define BROTLI_LITTLE_ENDIAN
#endif
#endif /* __BYTE_ORDER */
#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
#define IS_LITTLE_ENDIAN
#define BROTLI_LITTLE_ENDIAN
#endif
/* Enable little-endian optimization for x64 architecture on Windows. */
#if (defined(_WIN32) || defined(_WIN64)) && defined(_M_X64)
#define IS_LITTLE_ENDIAN
#define BROTLI_LITTLE_ENDIAN
#endif
/* Portable handling of unaligned loads, stores, and copies.
On some platforms, like ARM, the copy functions can be more efficient
then a load and a store. */
#if defined(ARCH_PIII) || \
defined(ARCH_ATHLON) || defined(ARCH_K8) || defined(_ARCH_PPC)
#if defined(BROTLI_LITTLE_ENDIAN) && (\
defined(ARCH_PIII) || defined(ARCH_ATHLON) || \
defined(ARCH_K8) || defined(_ARCH_PPC))
/* x86 and x86-64 can perform unaligned loads/stores directly;
modern PowerPC hardware can also do unaligned integer loads and stores;
@ -63,14 +64,12 @@
*/
#define BROTLI_UNALIGNED_LOAD32(_p) (*(const uint32_t *)(_p))
#define BROTLI_UNALIGNED_LOAD64(_p) (*(const uint64_t *)(_p))
#define BROTLI_UNALIGNED_LOAD64LE(_p) (*(const uint64_t *)(_p))
#define BROTLI_UNALIGNED_STORE32(_p, _val) \
(*(uint32_t *)(_p) = (_val))
#define BROTLI_UNALIGNED_STORE64(_p, _val) \
#define BROTLI_UNALIGNED_STORE64LE(_p, _val) \
(*(uint64_t *)(_p) = (_val))
#elif defined(__arm__) && \
#elif defined(BROTLI_LITTLE_ENDIAN) && defined(__arm__) && \
!defined(__ARM_ARCH_5__) && \
!defined(__ARM_ARCH_5T__) && \
!defined(__ARM_ARCH_5TE__) && \
@ -88,16 +87,14 @@
slowly (trip through kernel mode). */
#define BROTLI_UNALIGNED_LOAD32(_p) (*(const uint32_t *)(_p))
#define BROTLI_UNALIGNED_STORE32(_p, _val) \
(*(uint32_t *)(_p) = (_val))
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64(const void *p) {
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void *p) {
uint64_t t;
memcpy(&t, p, sizeof t);
return t;
}
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64(void *p, uint64_t v) {
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void *p, uint64_t v) {
memcpy(p, &v, sizeof v);
}
@ -112,20 +109,47 @@ static BROTLI_INLINE uint32_t BROTLI_UNALIGNED_LOAD32(const void *p) {
return t;
}
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64(const void *p) {
#if defined(BROTLI_LITTLE_ENDIAN)
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void *p) {
uint64_t t;
memcpy(&t, p, sizeof t);
return t;
}
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE32(void *p, uint32_t v) {
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void *p, uint64_t v) {
memcpy(p, &v, sizeof v);
}
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64(void *p, uint64_t v) {
memcpy(p, &v, sizeof v);
#else /* BROTLI_LITTLE_ENDIAN */
static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void *p) {
const uint8_t* in = (const uint8_t*)p;
uint64_t value = (uint64_t)(in[0]);
value |= (uint64_t)(in[1]) << 8;
value |= (uint64_t)(in[2]) << 16;
value |= (uint64_t)(in[3]) << 24;
value |= (uint64_t)(in[4]) << 32;
value |= (uint64_t)(in[5]) << 40;
value |= (uint64_t)(in[6]) << 48;
value |= (uint64_t)(in[7]) << 56;
return value;
}
static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void *p, uint64_t v) {
uint8_t* out = (uint8_t*)p;
out[0] = (uint8_t)v;
out[1] = (uint8_t)(v >> 8);
out[2] = (uint8_t)(v >> 16);
out[3] = (uint8_t)(v >> 24);
out[4] = (uint8_t)(v >> 32);
out[5] = (uint8_t)(v >> 40);
out[6] = (uint8_t)(v >> 48);
out[7] = (uint8_t)(v >> 56);
}
#endif /* BROTLI_LITTLE_ENDIAN */
#endif
#define TEMPLATE_(T) \

4
c/enc/write_bits.h
View File

@ -40,7 +40,7 @@ static BROTLI_INLINE void BrotliWriteBits(size_t n_bits,
uint64_t bits,
size_t * BROTLI_RESTRICT pos,
uint8_t * BROTLI_RESTRICT array) {
#ifdef IS_LITTLE_ENDIAN
#ifdef BROTLI_LITTLE_ENDIAN
/* This branch of the code can write up to 56 bits at a time,
7 bits are lost by being perhaps already in *p and at least
1 bit is needed to initialize the bit-stream ahead (i.e. if 7
@ -54,7 +54,7 @@ static BROTLI_INLINE void BrotliWriteBits(size_t n_bits,
assert((bits >> n_bits) == 0);
assert(n_bits <= 56);
v |= bits << (*pos & 7);
BROTLI_UNALIGNED_STORE64(p, v); /* Set some bits. */
BROTLI_UNALIGNED_STORE64LE(p, v); /* Set some bits. */
*pos += n_bits;
#else
/* implicit & 0xff is assumed for uint8_t arithmetics */

31
c/include/brotli/decode.h
View File

@ -84,8 +84,9 @@ typedef enum {
BROTLI_ERROR_CODE(_ERROR_FORMAT_, PADDING_1, -14) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_FORMAT_, PADDING_2, -15) SEPARATOR \
\
/* -16..-18 codes are reserved */ \
/* -16..-17 codes are reserved */ \
\
BROTLI_ERROR_CODE(_ERROR_, COMPOUND_DICTIONARY, -18) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_, DICTIONARY_NOT_SET, -19) SEPARATOR \
BROTLI_ERROR_CODE(_ERROR_, INVALID_ARGUMENTS, -20) SEPARATOR \
\
@ -241,31 +242,6 @@ BROTLI_DEC_API BrotliDecoderResult BrotliDecoderDecompressStream(
BrotliDecoderState* state, size_t* available_in, const uint8_t** next_in,
size_t* available_out, uint8_t** next_out, size_t* total_out);
/**
* Prepends LZ77 dictionary.
*
* Fills the fresh ::BrotliDecoderState with additional data corpus for LZ77
* backward references.
*
* @note Not to be confused with the static dictionary (see RFC7932 section 8).
* @warning The dictionary must exist in memory until decoding is done and
* is owned by the caller.
*
* Workflow:
* -# Allocate and initialize state with ::BrotliDecoderCreateInstance
* -# Invoke ::BrotliDecoderSetCustomDictionary
* -# Use ::BrotliDecoderDecompressStream
* -# Clean up and free state with ::BrotliDecoderDestroyInstance
*
* @param state decoder instance
* @param size length of @p dict; should be less or equal to 2^24 (16MiB),
* otherwise the dictionary will be ignored
* @param dict "dictionary"; @b MUST be the same as used during compression
*/
BROTLI_DEC_API void BrotliDecoderSetCustomDictionary(
BrotliDecoderState* state, size_t size,
const uint8_t dict[BROTLI_ARRAY_PARAM(size)]);
/**
* Checks if decoder has more output.
*
@ -327,7 +303,8 @@ BROTLI_DEC_API BROTLI_BOOL BrotliDecoderIsUsed(const BrotliDecoderState* state);
* the input and produced all of the output
* @returns ::BROTLI_FALSE otherwise
*/
BROTLI_DEC_API BROTLI_BOOL BrotliDecoderIsFinished(const BrotliDecoderState* state);
BROTLI_DEC_API BROTLI_BOOL BrotliDecoderIsFinished(
const BrotliDecoderState* state);
/**
* Acquires a detailed error code.

23
c/include/brotli/encode.h
View File

@ -223,29 +223,6 @@ BROTLI_ENC_API BrotliEncoderState* BrotliEncoderCreateInstance(
*/
BROTLI_ENC_API void BrotliEncoderDestroyInstance(BrotliEncoderState* state);
/**
* Prepends imaginary LZ77 dictionary.
*
* Fills the fresh ::BrotliEncoderState with additional data corpus for LZ77
* backward references.
*
* @note Not to be confused with the static dictionary (see RFC7932 section 8).
*
* Workflow:
* -# Allocate and initialize state with ::BrotliEncoderCreateInstance
* -# Set ::BROTLI_PARAM_LGWIN parameter
* -# Invoke ::BrotliEncoderSetCustomDictionary
* -# Use ::BrotliEncoderCompressStream
* -# Clean up and free state with ::BrotliEncoderDestroyInstance
*
* @param state encoder instance
* @param size length of @p dict; at most "window size" bytes are used
* @param dict "dictionary"; @b MUST use same dictionary during decompression
*/
BROTLI_ENC_API void BrotliEncoderSetCustomDictionary(
BrotliEncoderState* state, size_t size,
const uint8_t dict[BROTLI_ARRAY_PARAM(size)]);
/**
* Calculates the output size bound for the given @p input_size.
*

95
c/tools/brotli.c
View File

@ -91,7 +91,6 @@ typedef struct {
BROTLI_BOOL test_integrity;
BROTLI_BOOL decompress;
const char* output_path;
const char* dictionary_path;
const char* suffix;
int not_input_indices[MAX_OPTIONS];
size_t longest_path_len;
@ -100,8 +99,6 @@ typedef struct {
/* Inner state */
int argc;
char** argv;
uint8_t* dictionary;
size_t dictionary_size;
char* modified_path; /* Storage for path with appended / cut suffix */
int iterator;
int ignore;
@ -285,9 +282,6 @@ static Command ParseParams(Context* params) {
if (params->lgwin != 0 && params->lgwin < BROTLI_MIN_WINDOW_BITS) {
return COMMAND_INVALID;
}
} else if (c == 'D') {
if (params->dictionary_path) return COMMAND_INVALID;
params->dictionary_path = argv[i];
} else if (c == 'S') {
if (suffix_set) return COMMAND_INVALID;
suffix_set = BROTLI_TRUE;
@ -342,10 +336,7 @@ static Command ParseParams(Context* params) {
if (!value || value[1] == 0) return COMMAND_INVALID;
key_len = (size_t)(value - arg);
value++;
if (strncmp("dictionary", arg, key_len) == 0) {
if (params->dictionary_path) return COMMAND_INVALID;
params->dictionary_path = value;
} else if (strncmp("lgwin", arg, key_len) == 0) {
if (strncmp("lgwin", arg, key_len) == 0) {
if (lgwin_set) return COMMAND_INVALID;
lgwin_set = ParseInt(value, 0,
BROTLI_MAX_WINDOW_BITS, &params->lgwin);
@ -427,7 +418,7 @@ Options:\n\
fprintf(stdout, "\
window size = 2**NUM - 16\n\
0 lets compressor decide over the optimal value\n\
-D FILE, --dictionary=FILE use FILE as LZ77 dictionary\n");
");
fprintf(stdout, "\
-S SUF, --suffix=SUF output file suffix (default:'%s')\n",
DEFAULT_SUFFIX);
@ -482,23 +473,6 @@ static BROTLI_BOOL OpenOutputFile(const char* output_path, FILE** f,
return BROTLI_TRUE;
}
static int64_t FileSize(const char* path) {
FILE* f = fopen(path, "rb");
int64_t retval;
if (f == NULL) {
return -1;
}
if (fseek(f, 0L, SEEK_END) != 0) {
fclose(f);
return -1;
}
retval = ftell(f);
if (fclose(f) != 0) {
return -1;
}
return retval;
}
/* Copy file times and permissions.
TODO(eustas): this is a "best effort" implementation; honest cross-platform
fully featured implementation is way too hacky; add more hacks by request. */
@ -532,58 +506,6 @@ static void CopyStat(const char* input_path, const char* output_path) {
}
}
/* Result ownership is passed to caller.
|*dictionary_size| is set to resulting buffer size. */
static BROTLI_BOOL ReadDictionary(Context* context) {
static const int kMaxDictionarySize = (1 << 24) - 16;
FILE* f;
int64_t file_size_64;
uint8_t* buffer;
size_t bytes_read;
if (context->dictionary_path == NULL) return BROTLI_TRUE;
f = fopen(context->dictionary_path, "rb");
if (f == NULL) {
fprintf(stderr, "failed to open dictionary file [%s]: %s\n",
PrintablePath(context->dictionary_path), strerror(errno));
return BROTLI_FALSE;
}
file_size_64 = FileSize(context->dictionary_path);
if (file_size_64 == -1) {
fprintf(stderr, "could not get size of dictionary file [%s]",
PrintablePath(context->dictionary_path));
fclose(f);
return BROTLI_FALSE;
}
if (file_size_64 > kMaxDictionarySize) {
fprintf(stderr, "dictionary [%s] is larger than maximum allowed: %d\n",
PrintablePath(context->dictionary_path), kMaxDictionarySize);
fclose(f);
return BROTLI_FALSE;
}
context->dictionary_size = (size_t)file_size_64;
buffer = (uint8_t*)malloc(context->dictionary_size);
if (!buffer) {
fprintf(stderr, "could not read dictionary: out of memory\n");
fclose(f);
return BROTLI_FALSE;
}
bytes_read = fread(buffer, sizeof(uint8_t), context->dictionary_size, f);
if (bytes_read != context->dictionary_size) {
free(buffer);
fprintf(stderr, "failed to read dictionary [%s]: %s\n",
PrintablePath(context->dictionary_path), strerror(errno));
fclose(f);
return BROTLI_FALSE;
}
fclose(f);
context->dictionary = buffer;
return BROTLI_TRUE;
}
static BROTLI_BOOL NextFile(Context* context) {
const char* arg;
size_t arg_len;
@ -764,10 +686,6 @@ static BROTLI_BOOL DecompressFiles(Context* context) {
fprintf(stderr, "out of memory\n");
return BROTLI_FALSE;
}
if (context->dictionary) {
BrotliDecoderSetCustomDictionary(s,
context->dictionary_size, context->dictionary);
}
is_ok = OpenFiles(context);
if (is_ok) is_ok = DecompressFile(context, s);
BrotliDecoderDestroyInstance(s);
@ -833,10 +751,6 @@ static BROTLI_BOOL CompressFiles(Context* context) {
BROTLI_PARAM_QUALITY, (uint32_t)context->quality);
BrotliEncoderSetParameter(s,
BROTLI_PARAM_LGWIN, (uint32_t)context->lgwin);
if (context->dictionary) {
BrotliEncoderSetCustomDictionary(s,
context->dictionary_size, context->dictionary);
}
is_ok = OpenFiles(context);
if (is_ok) is_ok = CompressFile(context, s);
BrotliEncoderDestroyInstance(s);
@ -862,7 +776,6 @@ int main(int argc, char** argv) {
context.write_to_stdout = BROTLI_FALSE;
context.decompress = BROTLI_FALSE;
context.output_path = NULL;
context.dictionary_path = NULL;
context.suffix = DEFAULT_SUFFIX;
for (i = 0; i < MAX_OPTIONS; ++i) context.not_input_indices[i] = 0;
context.longest_path_len = 1;
@ -870,8 +783,6 @@ int main(int argc, char** argv) {
context.argc = argc;
context.argv = argv;
context.dictionary = NULL;
context.dictionary_size = 0;
context.modified_path = NULL;
context.iterator = 0;
context.ignore = 0;
@ -886,7 +797,6 @@ int main(int argc, char** argv) {
if (command == COMMAND_COMPRESS || command == COMMAND_DECOMPRESS ||
command == COMMAND_TEST_INTEGRITY) {
if (!ReadDictionary(&context)) is_ok = BROTLI_FALSE;
if (is_ok) {
size_t modified_path_len =
context.longest_path_len + strlen(context.suffix) + 1;
@ -931,7 +841,6 @@ int main(int argc, char** argv) {
if (context.iterator_error) is_ok = BROTLI_FALSE;
free(context.dictionary);
free(context.modified_path);
free(context.buffer);

3
c/tools/brotli.md
View File

@ -84,9 +84,6 @@ OPTIONS
`(2**NUM - 16)`; 0 lets compressor decide over the optimal value; bigger
windows size improve density; decoder might require up to window size
memory to operate
* `-D FILE`, `--dictionary=FILE`:
use FILE as LZ77 dictionary; same dictionary MUST be used both for
compression and decompression
* `-S SUF`, `--suffix=SUF`:
output file suffix (default: `.br`)
* `-V`, `--version`:

6
docs/brotli.1
View File

@ -1,4 +1,4 @@
.TH "BROTLI" "1" "May 2017" "brotli 1.0.0" "User commands"
.TH "BROTLI" "1" "August 2017" "brotli 1.0.0" "User commands"
.SH "NAME"
\fBbrotli\fR \- brotli, unbrotli \- compress or decompress files
.SH SYNOPSIS
@ -108,10 +108,6 @@ Conflicting or duplicate \fIoptions\fR are not allowed\.
windows size improve density; decoder might require up to window size
memory to operate
.IP \(bu 2
\fB\-D FILE\fP, \fB\-\-dictionary=FILE\fP:
use FILE as LZ77 dictionary; same dictionary MUST be used both for
compression and decompression
.IP \(bu 2
\fB\-S SUF\fP, \fB\-\-suffix=SUF\fP:
output file suffix (default: \fB\|\.br\fP)
.IP \(bu 2

42
docs/decode.h.3
View File

@ -1,4 +1,4 @@
.TH "decode.h" 3 "Tue Jun 13 2017" "Brotli" \" -*- nroff -*-
.TH "decode.h" 3 "Wed Aug 2 2017" "Brotli" \" -*- nroff -*-
.ad l
.nh
.SH NAME
@ -72,10 +72,6 @@ decode.h \- API for Brotli decompression\&.
.br
.RI "\fIChecks if instance has already consumed input\&. \fP"
.ti -1c
.RI "void \fBBrotliDecoderSetCustomDictionary\fP (\fBBrotliDecoderState\fP *state, size_t size, const uint8_t dict[size])"
.br
.RI "\fIPrepends LZ77 dictionary\&. \fP"
.ti -1c
.RI "\fBBROTLI_BOOL\fP \fBBrotliDecoderSetParameter\fP (\fBBrotliDecoderState\fP *state, \fBBrotliDecoderParameter\fP param, uint32_t value)"
.br
.RI "\fISets the specified parameter to the given decoder instance\&. \fP"
@ -346,42 +342,6 @@ Checks if instance has already consumed input\&. Instance that returns \fBBROTLI
.RE
.PP
.SS "void BrotliDecoderSetCustomDictionary (\fBBrotliDecoderState\fP * state, size_t size, const uint8_t dict[size])"
.PP
Prepends LZ77 dictionary\&. Fills the fresh \fBBrotliDecoderState\fP with additional data corpus for LZ77 backward references\&.
.PP
\fBNote:\fP
.RS 4
Not to be confused with the static dictionary (see RFC7932 section 8)\&.
.RE
.PP
\fBWarning:\fP
.RS 4
The dictionary must exist in memory until decoding is done and is owned by the caller\&.
.RE
.PP
Workflow:
.IP "1." 4
Allocate and initialize state with \fBBrotliDecoderCreateInstance\fP
.IP "2." 4
Invoke \fBBrotliDecoderSetCustomDictionary\fP
.IP "3." 4
Use \fBBrotliDecoderDecompressStream\fP
.IP "4." 4
Clean up and free state with \fBBrotliDecoderDestroyInstance\fP
.PP
.PP
\fBParameters:\fP
.RS 4
\fIstate\fP decoder instance
.br
\fIsize\fP length of \fCdict\fP; should be less or equal to 2^24 (16MiB), otherwise the dictionary will be ignored
.br
\fIdict\fP 'dictionary'; \fBMUST\fP be the same as used during compression
.RE
.PP
.SS "\fBBROTLI_BOOL\fP BrotliDecoderSetParameter (\fBBrotliDecoderState\fP * state, \fBBrotliDecoderParameter\fP param, uint32_t value)"
.PP

39
docs/encode.h.3
View File

@ -1,4 +1,4 @@
.TH "encode.h" 3 "Tue Feb 28 2017" "Brotli" \" -*- nroff -*-
.TH "encode.h" 3 "Wed Aug 2 2017" "Brotli" \" -*- nroff -*-
.ad l
.nh
.SH NAME
@ -100,10 +100,6 @@ encode.h \- API for Brotli compression\&.
.br
.RI "\fICalculates the output size bound for the given \fCinput_size\fP\&. \fP"
.ti -1c
.RI "void \fBBrotliEncoderSetCustomDictionary\fP (\fBBrotliEncoderState\fP *state, size_t size, const uint8_t dict[size])"
.br
.RI "\fIPrepends imaginary LZ77 dictionary\&. \fP"
.ti -1c
.RI "\fBBROTLI_BOOL\fP \fBBrotliEncoderSetParameter\fP (\fBBrotliEncoderState\fP *state, \fBBrotliEncoderParameter\fP param, uint32_t value)"
.br
.RI "\fISets the specified parameter to the given encoder instance\&. \fP"
@ -482,39 +478,6 @@ Result is not applicable to \fBBrotliEncoderCompressStream\fP output, because ev
.RE
.PP
.SS "void BrotliEncoderSetCustomDictionary (\fBBrotliEncoderState\fP * state, size_t size, const uint8_t dict[size])"
.PP
Prepends imaginary LZ77 dictionary\&. Fills the fresh \fBBrotliEncoderState\fP with additional data corpus for LZ77 backward references\&.
.PP
\fBNote:\fP
.RS 4
Not to be confused with the static dictionary (see RFC7932 section 8)\&.
.RE
.PP
Workflow:
.IP "1." 4
Allocate and initialize state with \fBBrotliEncoderCreateInstance\fP
.IP "2." 4
Set \fBBROTLI_PARAM_LGWIN\fP parameter
.IP "3." 4
Invoke \fBBrotliEncoderSetCustomDictionary\fP
.IP "4." 4
Use \fBBrotliEncoderCompressStream\fP
.IP "5." 4
Clean up and free state with \fBBrotliEncoderDestroyInstance\fP
.PP
.PP
\fBParameters:\fP
.RS 4
\fIstate\fP encoder instance
.br
\fIsize\fP length of \fCdict\fP; at most 'window size' bytes are used
.br
\fIdict\fP 'dictionary'; \fBMUST\fP use same dictionary during decompression
.RE
.PP
.SS "\fBBROTLI_BOOL\fP BrotliEncoderSetParameter (\fBBrotliEncoderState\fP * state, \fBBrotliEncoderParameter\fP param, uint32_t value)"
.PP

2
docs/types.h.3
View File

@ -1,4 +1,4 @@
.TH "types.h" 3 "Tue Feb 28 2017" "Brotli" \" -*- nroff -*-
.TH "types.h" 3 "Wed Aug 2 2017" "Brotli" \" -*- nroff -*-
.ad l
.nh
.SH NAME

6
java/org/brotli/dec/BUILD
View File

@ -42,12 +42,6 @@ java_test(
runtime_deps = [":test_lib"],
)
java_test(
name = "EnumTest",
test_class = "org.brotli.dec.EnumTest",
runtime_deps = [":test_lib"],
)
java_test(
name = "SynthTest",
test_class = "org.brotli.dec.SynthTest",

291
java/org/brotli/dec/BitReader.java
View File

@ -6,52 +6,34 @@
package org.brotli.dec;
import java.io.IOException;
import java.io.InputStream;
/**
* Bit reading helpers.
*/
final class BitReader {
/**
* Input byte buffer, consist of a ring-buffer and a "slack" region where bytes from the start of
* the ring-buffer are copied.
*/
private static final int CAPACITY = 1024;
private static final int SLACK = 16;
private static final int INT_BUFFER_SIZE = CAPACITY + SLACK;
private static final int BYTE_READ_SIZE = CAPACITY << 2;
private static final int BYTE_BUFFER_SIZE = INT_BUFFER_SIZE << 2;
// Possible values: {5, 6}. 5 corresponds to 32-bit build, 6 to 64-bit. This value is used for
// conditional compilation -> produced artifacts might be binary INCOMPATIBLE (JLS 13.2).
private static final int LOG_BITNESS = 6;
private static final int BITNESS = 1 << LOG_BITNESS;
private final byte[] byteBuffer = new byte[BYTE_BUFFER_SIZE];
private final int[] intBuffer = new int[INT_BUFFER_SIZE];
private final IntReader intReader = new IntReader();
private static final int BYTENESS = BITNESS / 8;
private static final int CAPACITY = 4096;
// After encountering the end of the input stream, this amount of zero bytes will be appended.
private static final int SLACK = 64;
private static final int BUFFER_SIZE = CAPACITY + SLACK;
// Don't bother to replenish the buffer while this number of bytes is available.
private static final int SAFEGUARD = 36;
private static final int WATERLINE = CAPACITY - SAFEGUARD;
private InputStream input;
// "Half" refers to "half of native integer type", i.e. on 64-bit machines it is 32-bit type,
// on 32-bit machines it is 16-bit.
private static final int HALF_BITNESS = BITNESS / 2;
private static final int HALF_SIZE = BYTENESS / 2;
private static final int HALVES_CAPACITY = CAPACITY / HALF_SIZE;
private static final int HALF_BUFFER_SIZE = BUFFER_SIZE / HALF_SIZE;
private static final int HALF_WATERLINE = WATERLINE / HALF_SIZE;
/**
* Input stream is finished.
*/
private boolean endOfStreamReached;
/**
* Pre-fetched bits.
*/
long accumulator;
/**
* Current bit-reading position in accumulator.
*/
int bitOffset;
/**
* Offset of next item in intBuffer.
*/
private int intOffset;
/* Number of bytes in unfinished "int" item. */
private int tailBytes = 0;
private static final int LOG_HALF_SIZE = LOG_BITNESS - 4;
/**
* Fills up the input buffer.
@ -62,141 +44,148 @@ final class BitReader {
* buffer.
*/
// TODO: Split to check and read; move read outside of decoding loop.
static void readMoreInput(BitReader br) {
if (br.intOffset <= CAPACITY - 9) {
static void readMoreInput(State s) {
if (s.halfOffset <= HALF_WATERLINE) {
return;
}
if (br.endOfStreamReached) {
if (intAvailable(br) >= -2) {
if (s.endOfStreamReached != 0) {
if (halfAvailable(s) >= -2) {
return;
}
throw new BrotliRuntimeException("No more input");
}
int readOffset = br.intOffset << 2;
int bytesRead = BYTE_READ_SIZE - readOffset;
System.arraycopy(br.byteBuffer, readOffset, br.byteBuffer, 0, bytesRead);
br.intOffset = 0;
try {
while (bytesRead < BYTE_READ_SIZE) {
int len = br.input.read(br.byteBuffer, bytesRead, BYTE_READ_SIZE - bytesRead);
int readOffset = s.halfOffset << LOG_HALF_SIZE;
int bytesInBuffer = CAPACITY - readOffset;
// Move unused bytes to the head of the buffer.
Utils.copyBytesWithin(s.byteBuffer, 0, readOffset, CAPACITY);
s.halfOffset = 0;
while (bytesInBuffer < CAPACITY) {
int spaceLeft = CAPACITY - bytesInBuffer;
int len = Utils.readInput(s.input, s.byteBuffer, bytesInBuffer, spaceLeft);
// EOF is -1 in Java, but 0 in C#.
if (len <= 0) {
br.endOfStreamReached = true;
br.tailBytes = bytesRead;
bytesRead += 3;
s.endOfStreamReached = 1;
s.tailBytes = bytesInBuffer;
bytesInBuffer += HALF_SIZE - 1;
break;
}
bytesRead += len;
bytesInBuffer += len;
}
} catch (IOException e) {
throw new BrotliRuntimeException("Failed to read input", e);
}
IntReader.convert(br.intReader, bytesRead >> 2);
bytesToNibbles(s, bytesInBuffer);
}
static void checkHealth(BitReader br, boolean endOfStream) {
if (!br.endOfStreamReached) {
static void checkHealth(State s, int endOfStream) {
if (s.endOfStreamReached == 0) {
return;
}
int byteOffset = (br.intOffset << 2) + ((br.bitOffset + 7) >> 3) - 8;
if (byteOffset > br.tailBytes) {
int byteOffset = (s.halfOffset << LOG_HALF_SIZE) + ((s.bitOffset + 7) >> 3) - BYTENESS;
if (byteOffset > s.tailBytes) {
throw new BrotliRuntimeException("Read after end");
}
if (endOfStream && (byteOffset != br.tailBytes)) {
if ((endOfStream != 0) && (byteOffset != s.tailBytes)) {
throw new BrotliRuntimeException("Unused bytes after end");
}
}
/**
* Advances the Read buffer by 5 bytes to make room for reading next 24 bits.
*/
static void fillBitWindow(BitReader br) {
if (br.bitOffset >= 32) {
br.accumulator = ((long) br.intBuffer[br.intOffset++] << 32) | (br.accumulator >>> 32);
br.bitOffset -= 32;
static void fillBitWindow(State s) {
if (s.bitOffset >= HALF_BITNESS) {
if (BITNESS == 64) {
s.accumulator64 = ((long) s.intBuffer[s.halfOffset++] << HALF_BITNESS)
| (s.accumulator64 >>> HALF_BITNESS);
} else {
s.accumulator32 = ((int) s.shortBuffer[s.halfOffset++] << HALF_BITNESS)
| (s.accumulator32 >>> HALF_BITNESS);
}
s.bitOffset -= HALF_BITNESS;
}
}
/**
* Reads the specified number of bits from Read Buffer.
*/
static int readBits(BitReader br, int n) {
fillBitWindow(br);
int val = (int) (br.accumulator >>> br.bitOffset) & ((1 << n) - 1);
br.bitOffset += n;
static int peekBits(State s) {
if (BITNESS == 64) {
return (int) (s.accumulator64 >>> s.bitOffset);
} else {
return s.accumulator32 >>> s.bitOffset;
}
}
static int readFewBits(State s, int n) {
fillBitWindow(s);
int val = peekBits(s) & ((1 << n) - 1);
s.bitOffset += n;
return val;
}
/**
* Initialize bit reader.
*
* <p> Initialisation turns bit reader to a ready state. Also a number of bytes is prefetched to
* accumulator. Because of that this method may block until enough data could be read from input.
*
* @param br BitReader POJO
* @param input data source
*/
static void init(BitReader br, InputStream input) {
if (br.input != null) {
throw new IllegalStateException("Bit reader already has associated input stream");
static int readBits(State s, int n) {
if (HALF_BITNESS >= 24) {
return readFewBits(s, n);
} else {
if (n <= 16) {
return readFewBits(s, n);
} else if (s.bitOffset + n <= BITNESS) {
return readFewBits(s, n);
} else {
int lo = readFewBits(s, 16);
int hi = readFewBits(s, n - 16);
return lo | (hi << 16);
}
IntReader.init(br.intReader, br.byteBuffer, br.intBuffer);
br.input = input;
br.accumulator = 0;
br.bitOffset = 64;
br.intOffset = CAPACITY;
br.endOfStreamReached = false;
prepare(br);
}
private static void prepare(BitReader br) {
readMoreInput(br);
checkHealth(br, false);
fillBitWindow(br);
fillBitWindow(br);
}
static void reload(BitReader br) {
if (br.bitOffset == 64) {
prepare(br);
}
}
static void close(BitReader br) throws IOException {
InputStream is = br.input;
br.input = null;
if (is != null) {
is.close();
static void initBitReader(State s) {
s.byteBuffer = new byte[BUFFER_SIZE];
if (BITNESS == 64) {
s.accumulator64 = 0;
s.intBuffer = new int[HALF_BUFFER_SIZE];
} else {
s.accumulator32 = 0;
s.shortBuffer = new short[HALF_BUFFER_SIZE];
}
s.bitOffset = BITNESS;
s.halfOffset = HALVES_CAPACITY;
s.endOfStreamReached = 0;
prepare(s);
}
static void prepare(State s) {
readMoreInput(s);
checkHealth(s, 0);
fillBitWindow(s);
fillBitWindow(s);
}
static void reload(State s) {
if (s.bitOffset == BITNESS) {
prepare(s);
}
}
static void jumpToByteBoundary(BitReader br) {
int padding = (64 - br.bitOffset) & 7;
static void jumpToByteBoundary(State s) {
int padding = (BITNESS - s.bitOffset) & 7;
if (padding != 0) {
int paddingBits = BitReader.readBits(br, padding);
int paddingBits = readBits(s, padding);
if (paddingBits != 0) {
throw new BrotliRuntimeException("Corrupted padding bits");
}
}
}
static int intAvailable(BitReader br) {
int limit = CAPACITY;
if (br.endOfStreamReached) {
limit = (br.tailBytes + 3) >> 2;
static int halfAvailable(State s) {
int limit = HALVES_CAPACITY;
if (s.endOfStreamReached != 0) {
limit = (s.tailBytes + (HALF_SIZE - 1)) >> LOG_HALF_SIZE;
}
return limit - br.intOffset;
return limit - s.halfOffset;
}
static void copyBytes(BitReader br, byte[] data, int offset, int length) {
if ((br.bitOffset & 7) != 0) {
static void copyBytes(State s, byte[] data, int offset, int length) {
if ((s.bitOffset & 7) != 0) {
throw new BrotliRuntimeException("Unaligned copyBytes");
}
// Drain accumulator.
while ((br.bitOffset != 64) && (length != 0)) {
data[offset++] = (byte) (br.accumulator >>> br.bitOffset);
br.bitOffset += 8;
while ((s.bitOffset != BITNESS) && (length != 0)) {
data[offset++] = (byte) peekBits(s);
s.bitOffset += 8;
length--;
}
if (length == 0) {
@ -204,43 +193,63 @@ final class BitReader {
}
// Get data from shadow buffer with "sizeof(int)" granularity.
int copyInts = Math.min(intAvailable(br), length >> 2);
if (copyInts > 0) {
int readOffset = br.intOffset << 2;
System.arraycopy(br.byteBuffer, readOffset, data, offset, copyInts << 2);
offset += copyInts << 2;
length -= copyInts << 2;
br.intOffset += copyInts;
int copyNibbles = Math.min(halfAvailable(s), length >> LOG_HALF_SIZE);
if (copyNibbles > 0) {
int readOffset = s.halfOffset << LOG_HALF_SIZE;
int delta = copyNibbles << LOG_HALF_SIZE;
System.arraycopy(s.byteBuffer, readOffset, data, offset, delta);
offset += delta;
length -= delta;
s.halfOffset += copyNibbles;
}
if (length == 0) {
return;
}
// Read tail bytes.
if (intAvailable(br) > 0) {
if (halfAvailable(s) > 0) {
// length = 1..3
fillBitWindow(br);
fillBitWindow(s);
while (length != 0) {
data[offset++] = (byte) (br.accumulator >>> br.bitOffset);
br.bitOffset += 8;
data[offset++] = (byte) peekBits(s);
s.bitOffset += 8;
length--;
}
checkHealth(br, false);
checkHealth(s, 0);
return;
}
// Now it is possible to copy bytes directly.
try {
while (length > 0) {
int len = br.input.read(data, offset, length);
int len = Utils.readInput(s.input, data, offset, length);
if (len == -1) {
throw new BrotliRuntimeException("Unexpected end of input");
}
offset += len;
length -= len;
}
} catch (IOException e) {
throw new BrotliRuntimeException("Failed to read input", e);
}
/**
* Translates bytes to halves (int/short).
*/
static void bytesToNibbles(State s, int byteLen) {
byte[] byteBuffer = s.byteBuffer;
int halfLen = byteLen >> LOG_HALF_SIZE;
if (BITNESS == 64) {
int[] intBuffer = s.intBuffer;
for (int i = 0; i < halfLen; ++i) {
intBuffer[i] = ((byteBuffer[i * 4] & 0xFF))
| ((byteBuffer[(i * 4) + 1] & 0xFF) << 8)
| ((byteBuffer[(i * 4) + 2] & 0xFF) << 16)
| ((byteBuffer[(i * 4) + 3] & 0xFF) << 24);
}
} else {
short[] shortBuffer = s.shortBuffer;
for (int i = 0; i < halfLen; ++i) {
shortBuffer[i] = (short) ((byteBuffer[i * 2] & 0xFF)
| ((byteBuffer[(i * 2) + 1] & 0xFF) << 8));
}
}
}
}

6
java/org/brotli/dec/BitReaderTest.java
View File

@ -21,11 +21,11 @@ public class BitReaderTest {
@Test
public void testReadAfterEos() {
BitReader reader = new BitReader();
BitReader.init(reader, new ByteArrayInputStream(new byte[1]));
State reader = new State();
Decode.initState(reader, new ByteArrayInputStream(new byte[1]));
BitReader.readBits(reader, 9);
try {
BitReader.checkHealth(reader, false);
BitReader.checkHealth(reader, 0);
} catch (BrotliRuntimeException ex) {
// This exception is expected.
return;

28
java/org/brotli/dec/BrotliInputStream.java
View File

@ -50,7 +50,7 @@ public class BrotliInputStream extends InputStream {
* @throws IOException in case of corrupted data or source stream problems
*/
public BrotliInputStream(InputStream source) throws IOException {
this(source, DEFAULT_INTERNAL_BUFFER_SIZE, null);
this(source, DEFAULT_INTERNAL_BUFFER_SIZE);
}
/**
@ -67,25 +67,6 @@ public class BrotliInputStream extends InputStream {
* @throws IOException in case of corrupted data or source stream problems
*/
public BrotliInputStream(InputStream source, int byteReadBufferSize) throws IOException {
this(source, byteReadBufferSize, null);
}
/**
* Creates a {@link InputStream} wrapper that decompresses brotli data.
*
* <p> For byte-by-byte reading ({@link #read()}) internal buffer of specified size is
* allocated and used.
*
* <p> Will block the thread until first kilobyte of data of source is available.
*
* @param source compressed data source
* @param byteReadBufferSize size of internal buffer used in case of
* byte-by-byte reading
* @param customDictionary custom dictionary data; {@code null} if not used
* @throws IOException in case of corrupted data or source stream problems
*/
public BrotliInputStream(InputStream source, int byteReadBufferSize,
byte[] customDictionary) throws IOException {
if (byteReadBufferSize <= 0) {
throw new IllegalArgumentException("Bad buffer size:" + byteReadBufferSize);
} else if (source == null) {
@ -95,13 +76,10 @@ public class BrotliInputStream extends InputStream {
this.remainingBufferBytes = 0;
this.bufferOffset = 0;
try {
State.setInput(state, source);
Decode.initState(state, source);
} catch (BrotliRuntimeException ex) {
throw new IOException("Brotli decoder initialization failed", ex);
}
if (customDictionary != null) {
Decode.setCustomDictionary(state, customDictionary);
}
}
/**
@ -109,7 +87,7 @@ public class BrotliInputStream extends InputStream {
*/
@Override
public void close() throws IOException {
State.close(state);
Decode.close(state);
}
/**

186
java/org/brotli/dec/Context.java
View File

@ -11,152 +11,48 @@ package org.brotli.dec;
*/
final class Context {
static final int[] LOOKUP = {
// CONTEXT_UTF8, last byte.
// ASCII range.
0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 4, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
8, 12, 16, 12, 12, 20, 12, 16, 24, 28, 12, 12, 32, 12, 36, 12,
44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 32, 32, 24, 40, 28, 12,
12, 48, 52, 52, 52, 48, 52, 52, 52, 48, 52, 52, 52, 52, 52, 48,
52, 52, 52, 52, 52, 48, 52, 52, 52, 52, 52, 24, 12, 28, 12, 12,
12, 56, 60, 60, 60, 56, 60, 60, 60, 56, 60, 60, 60, 60, 60, 56,
60, 60, 60, 60, 60, 56, 60, 60, 60, 60, 60, 24, 12, 28, 12, 0,
static final int[] LOOKUP = new int[2048];
// UTF8 continuation byte range.
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
private static final String UTF_MAP = " !! ! \"#$##%#$&'##(#)#+++++++++"
+ "+((&*'##,---,---,-----,-----,-----&#'###.///.///./////./////./////&#'# ";
private static final String UTF_RLE = "A/* ': & : $ \u0081 @";
// UTF8 lead byte range.
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3,
private static void unpackLookupTable(int[] lookup, String map, String rle) {
// LSB6, MSB6, SIGNED
for (int i = 0; i < 256; ++i) {
lookup[i] = i & 0x3F;
lookup[512 + i] = i >> 2;
lookup[1792 + i] = 2 + (i >> 6);
}
// UTF8
for (int i = 0; i < 128; ++i) {
lookup[1024 + i] = 4 * (map.charAt(i) - 32);
}
for (int i = 0; i < 64; ++i) {
lookup[1152 + i] = i & 1;
lookup[1216 + i] = 2 + (i & 1);
}
int offset = 1280;
for (int k = 0; k < 19; ++k) {
int value = k & 3;
int rep = rle.charAt(k) - 32;
for (int i = 0; i < rep; ++i) {
lookup[offset++] = value;
}
}
// SIGNED
for (int i = 0; i < 16; ++i) {
lookup[1792 + i] = 1;
lookup[2032 + i] = 6;
}
lookup[1792] = 0;
lookup[2047] = 7;
for (int i = 0; i < 256; ++i) {
lookup[1536 + i] = lookup[1792 + i] << 3;
}
}
// CONTEXT_UTF8 second last byte.
// ASCII range.
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1,
1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1,
1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 1, 0,
// UTF8 continuation byte range.
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
// UTF8 lead byte range.
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
// CONTEXT_SIGNED, second last byte.
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7,
// CONTEXT_SIGNED, last byte, same as the above values shifted by 3 bits.
0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 56,
// CONTEXT_LSB6, last byte.
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,
48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,
// CONTEXT_MSB6, last byte.
0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3,
4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7,
8, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 11, 11, 11, 11,
12, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15,
16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18, 19, 19, 19, 19,
20, 20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 23, 23, 23, 23,
24, 24, 24, 24, 25, 25, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27,
28, 28, 28, 28, 29, 29, 29, 29, 30, 30, 30, 30, 31, 31, 31, 31,
32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 35, 35, 35, 35,
36, 36, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38, 39, 39, 39, 39,
40, 40, 40, 40, 41, 41, 41, 41, 42, 42, 42, 42, 43, 43, 43, 43,
44, 44, 44, 44, 45, 45, 45, 45, 46, 46, 46, 46, 47, 47, 47, 47,
48, 48, 48, 48, 49, 49, 49, 49, 50, 50, 50, 50, 51, 51, 51, 51,
52, 52, 52, 52, 53, 53, 53, 53, 54, 54, 54, 54, 55, 55, 55, 55,
56, 56, 56, 56, 57, 57, 57, 57, 58, 58, 58, 58, 59, 59, 59, 59,
60, 60, 60, 60, 61, 61, 61, 61, 62, 62, 62, 62, 63, 63, 63, 63,
// CONTEXT_{M,L}SB6, second last byte,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
};
static final int[] LOOKUP_OFFSETS = {
// CONTEXT_LSB6
1024, 1536,
// CONTEXT_MSB6
1280, 1536,
// CONTEXT_UTF8
0, 256,
// CONTEXT_SIGNED
768, 512
};
static {
unpackLookupTable(LOOKUP, UTF_MAP, UTF_RLE);
}
}

919
java/org/brotli/dec/Decode.java
View File

File diff suppressed because it is too large Load Diff

47
java/org/brotli/dec/DecodeTest.java
View File

@ -21,6 +21,14 @@ import org.junit.runners.JUnit4;
@RunWith(JUnit4.class)
public class DecodeTest {
static byte[] readUniBytes(String uniBytes) {
byte[] result = new byte[uniBytes.length()];
for (int i = 0; i < result.length; ++i) {
result[i] = (byte) uniBytes.charAt(i);
}
return result;
}
private byte[] decompress(byte[] data, boolean byByte) throws IOException {
byte[] buffer = new byte[65536];
ByteArrayInputStream input = new ByteArrayInputStream(data);
@ -49,35 +57,9 @@ public class DecodeTest {
return output.toByteArray();
}
private byte[] decompressWithDictionary(byte[] data, byte[] dictionary) throws IOException {
byte[] buffer = new byte[65536];
ByteArrayInputStream input = new ByteArrayInputStream(data);
ByteArrayOutputStream output = new ByteArrayOutputStream();
BrotliInputStream brotliInput = new BrotliInputStream(
input, BrotliInputStream.DEFAULT_INTERNAL_BUFFER_SIZE, dictionary);
while (true) {
int len = brotliInput.read(buffer, 0, buffer.length);
if (len <= 0) {
break;
}
output.write(buffer, 0, len);
}
brotliInput.close();
return output.toByteArray();
}
private void checkDecodeResourceWithDictionary(String expected, String compressed,
String dictionary) throws IOException {
byte[] expectedBytes = Transform.readUniBytes(expected);
byte[] compressedBytes = Transform.readUniBytes(compressed);
byte[] dictionaryBytes = Transform.readUniBytes(dictionary);
byte[] actual = decompressWithDictionary(compressedBytes, dictionaryBytes);
assertArrayEquals(expectedBytes, actual);
}
private void checkDecodeResource(String expected, String compressed) throws IOException {
byte[] expectedBytes = Transform.readUniBytes(expected);
byte[] compressedBytes = Transform.readUniBytes(compressed);
byte[] expectedBytes = readUniBytes(expected);
byte[] compressedBytes = readUniBytes(compressed);
byte[] actual = decompress(compressedBytes, false);
assertArrayEquals(expectedBytes, actual);
byte[] actualByByte = decompress(compressedBytes, true);
@ -168,20 +150,11 @@ public class DecodeTest {
+ "\u0091\u00FF\u0087\u00E9\u001E");
}
@Test
public void testFoxFox() throws IOException {
checkDecodeResourceWithDictionary(
"The quick brown fox jumps over the lazy dog",
"\u001B*\u0000\u0000 \u0000\u00C2\u0098\u00B0\u00CA\u0001",
"The quick brown fox jumps over the lazy dog");
}
@Test
public void testUtils() {
new Context();
new Decode();
new Dictionary();
new Huffman();
new Prefix();
}
}

15
java/org/brotli/dec/Dictionary.java
View File

@ -48,19 +48,4 @@ public final class Dictionary {
/* Might have been set when {@link DictionaryData} was loaded.*/
return data;
}
static final int[] OFFSETS_BY_LENGTH = {
0, 0, 0, 0, 0, 4096, 9216, 21504, 35840, 44032, 53248, 63488, 74752, 87040, 93696, 100864,
104704, 106752, 108928, 113536, 115968, 118528, 119872, 121280, 122016
};
static final int[] SIZE_BITS_BY_LENGTH = {
0, 0, 0, 0, 10, 10, 11, 11, 10, 10, 10, 10, 10, 9, 9, 8, 7, 7, 8, 7, 7, 6, 6, 5, 5
};
static final int MIN_WORD_LENGTH = 4;
static final int MAX_WORD_LENGTH = 24;
static final int MAX_TRANSFORMED_WORD_LENGTH = 5 + MAX_WORD_LENGTH + 8;
}

40
java/org/brotli/dec/DictionaryData.java
View File

File diff suppressed because one or more lines are too long

53
java/org/brotli/dec/EnumTest.java
View File

@ -1,53 +0,0 @@
/* Copyright 2017 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
package org.brotli.dec;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.TreeSet;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/**
* Tests for Enum-like classes.
*/
@RunWith(JUnit4.class)
public class EnumTest {
private void checkEnumClass(Class<?> clazz) {
TreeSet<Integer> values = new TreeSet<Integer>();
for (Field f : clazz.getDeclaredFields()) {
assertEquals("int", f.getType().getName());
assertEquals(Modifier.FINAL | Modifier.STATIC, f.getModifiers());
Integer value = null;
try {
value = f.getInt(null);
} catch (IllegalAccessException ex) {
fail("Inaccessible field");
}
assertFalse(values.contains(value));
values.add(value);
}
assertEquals(0, values.first().intValue());
assertEquals(values.size(), values.last() + 1);
}
@Test
public void testRunningState() {
checkEnumClass(RunningState.class);
}
@Test
public void testWordTransformType() {
checkEnumClass(WordTransformType.class);
}
}

6
java/org/brotli/dec/Huffman.java
View File

@ -11,12 +11,6 @@ package org.brotli.dec;
*/
final class Huffman {
/**
* Maximum possible Huffman table size for an alphabet size of 704, max code length 15 and root
* table bits 8.
*/
static final int HUFFMAN_MAX_TABLE_SIZE = 1080;
private static final int MAX_LENGTH = 15;
/**

57
java/org/brotli/dec/HuffmanTreeGroup.java
View File

@ -1,57 +0,0 @@
/* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
package org.brotli.dec;
/**
* Contains a collection of huffman trees with the same alphabet size.
*/
final class HuffmanTreeGroup {
/**
* The maximal alphabet size in this group.
*/
private int alphabetSize;
/**
* Storage for Huffman lookup tables.
*/
int[] codes;
/**
* Offsets of distinct lookup tables in {@link #codes} storage.
*/
int[] trees;
/**
* Initializes the Huffman tree group.
*
* @param group POJO to be initialised
* @param alphabetSize the maximal alphabet size in this group
* @param n number of Huffman codes
*/
static void init(HuffmanTreeGroup group, int alphabetSize, int n) {
group.alphabetSize = alphabetSize;
group.codes = new int[n * Huffman.HUFFMAN_MAX_TABLE_SIZE];
group.trees = new int[n];
}
/**
* Decodes Huffman trees from input stream and constructs lookup tables.
*
* @param group target POJO
* @param br data source
*/
static void decode(HuffmanTreeGroup group, BitReader br) {
int next = 0;
int n = group.trees.length;
for (int i = 0; i < n; i++) {
group.trees[i] = next;
Decode.readHuffmanCode(group.alphabetSize, group.codes, next, br);
next += Huffman.HUFFMAN_MAX_TABLE_SIZE;
}
}
}

36
java/org/brotli/dec/IntReader.java
View File

@ -1,36 +0,0 @@
/* Copyright 2017 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
package org.brotli.dec;
/**
* Byte-to-int conversion magic.
*/
final class IntReader {
private byte[] byteBuffer;
private int[] intBuffer;
static void init(IntReader ir, byte[] byteBuffer, int[] intBuffer) {
ir.byteBuffer = byteBuffer;
ir.intBuffer = intBuffer;
}
/**
* Translates bytes to ints.
*
* NB: intLen == 4 * byteSize!
* NB: intLen should be less or equal to intBuffer length.
*/
static void convert(IntReader ir, int intLen) {
for (int i = 0; i < intLen; ++i) {
ir.intBuffer[i] = ((ir.byteBuffer[i * 4] & 0xFF))
| ((ir.byteBuffer[(i * 4) + 1] & 0xFF) << 8)
| ((ir.byteBuffer[(i * 4) + 2] & 0xFF) << 16)
| ((ir.byteBuffer[(i * 4) + 3] & 0xFF) << 24);
}
}
}

53
java/org/brotli/dec/Prefix.java
View File

@ -1,53 +0,0 @@
/* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
package org.brotli.dec;
/**
* Lookup tables to map prefix codes to value ranges.
*
* <p> This is used during decoding of the block lengths, literal insertion lengths and copy
* lengths.
*
* <p> Range represents values: [offset, offset + 2 ^ n_bits)
*/
final class Prefix {
static final int[] BLOCK_LENGTH_OFFSET = {
1, 5, 9, 13, 17, 25, 33, 41, 49, 65, 81, 97, 113, 145, 177, 209, 241, 305, 369, 497,
753, 1265, 2289, 4337, 8433, 16625
};
static final int[] BLOCK_LENGTH_N_BITS = {
2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 7, 8, 9, 10, 11, 12, 13, 24
};
static final int[] INSERT_LENGTH_OFFSET = {
0, 1, 2, 3, 4, 5, 6, 8, 10, 14, 18, 26, 34, 50, 66, 98, 130, 194, 322, 578, 1090, 2114, 6210,
22594
};
static final int[] INSERT_LENGTH_N_BITS = {
0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 12, 14, 24
};
static final int[] COPY_LENGTH_OFFSET = {
2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 18, 22, 30, 38, 54, 70, 102, 134, 198, 326, 582, 1094,
2118
};
static final int[] COPY_LENGTH_N_BITS = {
0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 24
};
static final int[] INSERT_RANGE_LUT = {
0, 0, 8, 8, 0, 16, 8, 16, 16
};
static final int[] COPY_RANGE_LUT = {
0, 8, 0, 8, 16, 0, 16, 8, 16
};
}

26
java/org/brotli/dec/RunningState.java
View File

@ -1,26 +0,0 @@
/* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
package org.brotli.dec;
/**
* Enumeration of decoding state-machine.
*/
final class RunningState {
static final int UNINITIALIZED = 0;
static final int BLOCK_START = 1;
static final int COMPRESSED_BLOCK_START = 2;
static final int MAIN_LOOP = 3;
static final int READ_METADATA = 4;
static final int COPY_UNCOMPRESSED = 5;
static final int INSERT_LOOP = 6;
static final int COPY_LOOP = 7;
static final int COPY_WRAP_BUFFER = 8;
static final int TRANSFORM = 9;
static final int FINISHED = 10;
static final int CLOSED = 11;
static final int WRITE = 12;
}

128
java/org/brotli/dec/State.java
View File

@ -6,51 +6,58 @@
package org.brotli.dec;
import static org.brotli.dec.RunningState.BLOCK_START;
import static org.brotli.dec.RunningState.CLOSED;
import static org.brotli.dec.RunningState.UNINITIALIZED;
import java.io.IOException;
import java.io.InputStream;
final class State {
int runningState = UNINITIALIZED;
int nextRunningState;
final BitReader br = new BitReader();
byte[] ringBuffer;
final int[] blockTypeTrees = new int[3 * Huffman.HUFFMAN_MAX_TABLE_SIZE];
final int[] blockLenTrees = new int[3 * Huffman.HUFFMAN_MAX_TABLE_SIZE];
byte[] contextModes;
byte[] contextMap;
byte[] distContextMap;
byte[] output;
byte[] byteBuffer; // BitReader
// Current meta-block header information.
short[] shortBuffer; // BitReader
int[] intBuffer; // BitReader
int[] rings;
int[] blockTrees;
int[] hGroup0;
int[] hGroup1;
int[] hGroup2;
long accumulator64; // BitReader: pre-fetched bits.
int runningState; // Default value is 0 == Decode.UNINITIALIZED
int nextRunningState;
int accumulator32; // BitReader: pre-fetched bits.
int bitOffset; // BitReader: bit-reading position in accumulator.
int halfOffset; // BitReader: offset of next item in intBuffer/shortBuffer.
int tailBytes; // BitReader: number of bytes in unfinished half.
int endOfStreamReached; // BitReader: input stream is finished.
int metaBlockLength;
boolean inputEnd;
boolean isUncompressed;
boolean isMetadata;
final HuffmanTreeGroup hGroup0 = new HuffmanTreeGroup();
final HuffmanTreeGroup hGroup1 = new HuffmanTreeGroup();
final HuffmanTreeGroup hGroup2 = new HuffmanTreeGroup();
final int[] blockLength = new int[3];
final int[] numBlockTypes = new int[3];
final int[] blockTypeRb = new int[6];
final int[] distRb = {16, 15, 11, 4};
int pos = 0;
int maxDistance = 0;
int distRbIdx = 0;
boolean trivialLiteralContext = false;
int literalTreeIndex = 0;
int inputEnd;
int isUncompressed;
int isMetadata;
int literalBlockLength;
int numLiteralBlockTypes;
int commandBlockLength;
int numCommandBlockTypes;
int distanceBlockLength;
int numDistanceBlockTypes;
int pos;
int maxDistance;
int distRbIdx;
int trivialLiteralContext;
int literalTreeIndex;
int literalTree;
int j;
int insertLength;
byte[] contextModes;
byte[] contextMap;
int contextMapSlice;
int distContextMapSlice;
int contextLookupOffset1;
int contextLookupOffset2;
int treeCommandOffset;
int distanceCode;
byte[] distContextMap;
int numDirectDistanceCodes;
int distancePostfixMask;
int distancePostfixBits;
@ -59,62 +66,23 @@ final class State {
int copyDst;
int maxBackwardDistance;
int maxRingBufferSize;
int ringBufferSize = 0;
long expectedTotalSize = 0;
byte[] customDictionary = new byte[0];
int bytesToIgnore = 0;
int ringBufferSize;
int expectedTotalSize;
int bytesToIgnore;
int outputOffset;
int outputLength;
int outputUsed;
int bytesWritten;
int bytesToWrite;
byte[] output;
// TODO: Update to current spec.
private static int decodeWindowBits(BitReader br) {
if (BitReader.readBits(br, 1) == 0) {
return 16;
}
int n = BitReader.readBits(br, 3);
if (n != 0) {
return 17 + n;
}
n = BitReader.readBits(br, 3);
if (n != 0) {
return 8 + n;
}
return 17;
}
InputStream input; // BitReader
/**
* Associate input with decoder state.
*
* @param state uninitialized state without associated input
* @param input compressed data source
*/
static void setInput(State state, InputStream input) {
if (state.runningState != UNINITIALIZED) {
throw new IllegalStateException("State MUST be uninitialized");
}
BitReader.init(state.br, input);
int windowBits = decodeWindowBits(state.br);
if (windowBits == 9) { /* Reserved case for future expansion. */
throw new BrotliRuntimeException("Invalid 'windowBits' code");
}
state.maxRingBufferSize = 1 << windowBits;
state.maxBackwardDistance = state.maxRingBufferSize - 16;
state.runningState = BLOCK_START;
}
static void close(State state) throws IOException {
if (state.runningState == UNINITIALIZED) {
throw new IllegalStateException("State MUST be initialized");
}
if (state.runningState == CLOSED) {
return;
}
state.runningState = CLOSED;
BitReader.close(state.br);
State() {
this.ringBuffer = new byte[0];
this.rings = new int[10];
this.rings[0] = 16;
this.rings[1] = 15;
this.rings[2] = 11;
this.rings[3] = 4;
}
}

10
java/org/brotli/dec/SynthTest.java
View File

@ -22,6 +22,14 @@ import org.junit.runners.JUnit4;
@RunWith(JUnit4.class)
public class SynthTest {
static byte[] readUniBytes(String uniBytes) {
byte[] result = new byte[uniBytes.length()];
for (int i = 0; i < result.length; ++i) {
result[i] = (byte) uniBytes.charAt(i);
}
return result;
}
private byte[] decompress(byte[] data) throws IOException {
byte[] buffer = new byte[65536];
ByteArrayInputStream input = new ByteArrayInputStream(data);
@ -40,7 +48,7 @@ public class SynthTest {
private void checkSynth(byte[] compressed, boolean expectSuccess,
String expectedOutput) {
byte[] expected = Transform.readUniBytes(expectedOutput);
byte[] expected = readUniBytes(expectedOutput);
try {
byte[] actual = decompress(compressed);
if (!expectSuccess) {

233
java/org/brotli/dec/Transform.java
View File

@ -6,27 +6,6 @@
package org.brotli.dec;
import static org.brotli.dec.WordTransformType.IDENTITY;
import static org.brotli.dec.WordTransformType.OMIT_FIRST_1;
import static org.brotli.dec.WordTransformType.OMIT_FIRST_2;
import static org.brotli.dec.WordTransformType.OMIT_FIRST_3;
import static org.brotli.dec.WordTransformType.OMIT_FIRST_4;
import static org.brotli.dec.WordTransformType.OMIT_FIRST_5;
import static org.brotli.dec.WordTransformType.OMIT_FIRST_6;
import static org.brotli.dec.WordTransformType.OMIT_FIRST_7;
import static org.brotli.dec.WordTransformType.OMIT_FIRST_9;
import static org.brotli.dec.WordTransformType.OMIT_LAST_1;
import static org.brotli.dec.WordTransformType.OMIT_LAST_2;
import static org.brotli.dec.WordTransformType.OMIT_LAST_3;
import static org.brotli.dec.WordTransformType.OMIT_LAST_4;
import static org.brotli.dec.WordTransformType.OMIT_LAST_5;
import static org.brotli.dec.WordTransformType.OMIT_LAST_6;
import static org.brotli.dec.WordTransformType.OMIT_LAST_7;
import static org.brotli.dec.WordTransformType.OMIT_LAST_8;
import static org.brotli.dec.WordTransformType.OMIT_LAST_9;
import static org.brotli.dec.WordTransformType.UPPERCASE_ALL;
import static org.brotli.dec.WordTransformType.UPPERCASE_FIRST;
import java.nio.ByteBuffer;
/**
@ -34,185 +13,82 @@ import java.nio.ByteBuffer;
*/
final class Transform {
private final byte[] prefix;
private final int type;
private final byte[] suffix;
static final int NUM_TRANSFORMS = 121;
private static final int[] TRANSFORMS = new int[NUM_TRANSFORMS * 3];
private static final byte[] PREFIX_SUFFIX = new byte[217];
private static final int[] PREFIX_SUFFIX_HEADS = new int[51];
Transform(String prefix, int type, String suffix) {
this.prefix = readUniBytes(prefix);
this.type = type;
this.suffix = readUniBytes(suffix);
// Bundle of 0-terminated strings.
private static final String PREFIX_SUFFIX_SRC = "# #s #, #e #.# the #.com/#\u00C2\u00A0# of # and"
+ " # in # to #\"#\">#\n#]# for # a # that #. # with #'# from # by #. The # on # as # is #ing"
+ " #\n\t#:#ed #(# at #ly #=\"# of the #. This #,# not #er #al #='#ful #ive #less #est #ize #"
+ "ous #";
private static final String TRANSFORMS_SRC = " !! ! , *! &! \" ! ) * * - ! # ! #!*! "
+ "+ ,$ ! - % . / # 0 1 . \" 2 3!* 4% ! # / 5 6 7 8 0 1 & $ 9 + : "
+ " ; < ' != > ?! 4 @ 4 2 & A *# ( B C& ) % ) !*# *-% A +! *. D! %' & E *6 F "
+ " G% ! *A *% H! D I!+! J!+ K +- *4! A L!*4 M N +6 O!*% +.! K *G P +%( ! G *D +D "
+ " Q +# *K!*G!+D!+# +G +A +4!+% +K!+4!*D!+K!*K";
private static void unpackTransforms(byte[] prefixSuffix, int[] prefixSuffixHeads,
int[] transforms, String prefixSuffixSrc, String transformsSrc) {
int n = prefixSuffixSrc.length();
int index = 1;
for (int i = 0; i < n; ++i) {
char c = prefixSuffixSrc.charAt(i);
prefixSuffix[i] = (byte) c;
if (c == 35) { // == #
prefixSuffixHeads[index++] = i + 1;
prefixSuffix[i] = 0;
}
}
static byte[] readUniBytes(String uniBytes) {
byte[] result = new byte[uniBytes.length()];
for (int i = 0; i < result.length; ++i) {
result[i] = (byte) uniBytes.charAt(i);
for (int i = 0; i < NUM_TRANSFORMS * 3; ++i) {
transforms[i] = transformsSrc.charAt(i) - 32;
}
return result;
}
static final Transform[] TRANSFORMS = {
new Transform("", IDENTITY, ""),
new Transform("", IDENTITY, " "),
new Transform(" ", IDENTITY, " "),
new Transform("", OMIT_FIRST_1, ""),
new Transform("", UPPERCASE_FIRST, " "),
new Transform("", IDENTITY, " the "),
new Transform(" ", IDENTITY, ""),
new Transform("s ", IDENTITY, " "),
new Transform("", IDENTITY, " of "),
new Transform("", UPPERCASE_FIRST, ""),
new Transform("", IDENTITY, " and "),
new Transform("", OMIT_FIRST_2, ""),
new Transform("", OMIT_LAST_1, ""),
new Transform(", ", IDENTITY, " "),
new Transform("", IDENTITY, ", "),
new Transform(" ", UPPERCASE_FIRST, " "),
new Transform("", IDENTITY, " in "),
new Transform("", IDENTITY, " to "),
new Transform("e ", IDENTITY, " "),
new Transform("", IDENTITY, "\""),
new Transform("", IDENTITY, "."),
new Transform("", IDENTITY, "\">"),
new Transform("", IDENTITY, "\n"),
new Transform("", OMIT_LAST_3, ""),
new Transform("", IDENTITY, "]"),
new Transform("", IDENTITY, " for "),
new Transform("", OMIT_FIRST_3, ""),
new Transform("", OMIT_LAST_2, ""),
new Transform("", IDENTITY, " a "),
new Transform("", IDENTITY, " that "),
new Transform(" ", UPPERCASE_FIRST, ""),
new Transform("", IDENTITY, ". "),
new Transform(".", IDENTITY, ""),
new Transform(" ", IDENTITY, ", "),
new Transform("", OMIT_FIRST_4, ""),
new Transform("", IDENTITY, " with "),
new Transform("", IDENTITY, "'"),
new Transform("", IDENTITY, " from "),
new Transform("", IDENTITY, " by "),
new Transform("", OMIT_FIRST_5, ""),
new Transform("", OMIT_FIRST_6, ""),
new Transform(" the ", IDENTITY, ""),
new Transform("", OMIT_LAST_4, ""),
new Transform("", IDENTITY, ". The "),
new Transform("", UPPERCASE_ALL, ""),
new Transform("", IDENTITY, " on "),
new Transform("", IDENTITY, " as "),
new Transform("", IDENTITY, " is "),
new Transform("", OMIT_LAST_7, ""),
new Transform("", OMIT_LAST_1, "ing "),
new Transform("", IDENTITY, "\n\t"),
new Transform("", IDENTITY, ":"),
new Transform(" ", IDENTITY, ". "),
new Transform("", IDENTITY, "ed "),
new Transform("", OMIT_FIRST_9, ""),
new Transform("", OMIT_FIRST_7, ""),
new Transform("", OMIT_LAST_6, ""),
new Transform("", IDENTITY, "("),
new Transform("", UPPERCASE_FIRST, ", "),
new Transform("", OMIT_LAST_8, ""),
new Transform("", IDENTITY, " at "),
new Transform("", IDENTITY, "ly "),
new Transform(" the ", IDENTITY, " of "),
new Transform("", OMIT_LAST_5, ""),
new Transform("", OMIT_LAST_9, ""),
new Transform(" ", UPPERCASE_FIRST, ", "),
new Transform("", UPPERCASE_FIRST, "\""),
new Transform(".", IDENTITY, "("),
new Transform("", UPPERCASE_ALL, " "),
new Transform("", UPPERCASE_FIRST, "\">"),
new Transform("", IDENTITY, "=\""),
new Transform(" ", IDENTITY, "."),
new Transform(".com/", IDENTITY, ""),
new Transform(" the ", IDENTITY, " of the "),
new Transform("", UPPERCASE_FIRST, "'"),
new Transform("", IDENTITY, ". This "),
new Transform("", IDENTITY, ","),
new Transform(".", IDENTITY, " "),
new Transform("", UPPERCASE_FIRST, "("),
new Transform("", UPPERCASE_FIRST, "."),
new Transform("", IDENTITY, " not "),
new Transform(" ", IDENTITY, "=\""),
new Transform("", IDENTITY, "er "),
new Transform(" ", UPPERCASE_ALL, " "),
new Transform("", IDENTITY, "al "),
new Transform(" ", UPPERCASE_ALL, ""),
new Transform("", IDENTITY, "='"),
new Transform("", UPPERCASE_ALL, "\""),
new Transform("", UPPERCASE_FIRST, ". "),
new Transform(" ", IDENTITY, "("),
new Transform("", IDENTITY, "ful "),
new Transform(" ", UPPERCASE_FIRST, ". "),
new Transform("", IDENTITY, "ive "),
new Transform("", IDENTITY, "less "),
new Transform("", UPPERCASE_ALL, "'"),
new Transform("", IDENTITY, "est "),
new Transform(" ", UPPERCASE_FIRST, "."),
new Transform("", UPPERCASE_ALL, "\">"),
new Transform(" ", IDENTITY, "='"),
new Transform("", UPPERCASE_FIRST, ","),
new Transform("", IDENTITY, "ize "),
new Transform("", UPPERCASE_ALL, "."),
new Transform("\u00c2\u00a0", IDENTITY, ""),
new Transform(" ", IDENTITY, ","),
new Transform("", UPPERCASE_FIRST, "=\""),
new Transform("", UPPERCASE_ALL, "=\""),
new Transform("", IDENTITY, "ous "),
new Transform("", UPPERCASE_ALL, ", "),
new Transform("", UPPERCASE_FIRST, "='"),
new Transform(" ", UPPERCASE_FIRST, ","),
new Transform(" ", UPPERCASE_ALL, "=\""),
new Transform(" ", UPPERCASE_ALL, ", "),
new Transform("", UPPERCASE_ALL, ","),
new Transform("", UPPERCASE_ALL, "("),
new Transform("", UPPERCASE_ALL, ". "),
new Transform(" ", UPPERCASE_ALL, "."),
new Transform("", UPPERCASE_ALL, "='"),
new Transform(" ", UPPERCASE_ALL, ". "),
new Transform(" ", UPPERCASE_FIRST, "=\""),
new Transform(" ", UPPERCASE_ALL, "='"),
new Transform(" ", UPPERCASE_FIRST, "='")
};
static {
unpackTransforms(PREFIX_SUFFIX, PREFIX_SUFFIX_HEADS, TRANSFORMS, PREFIX_SUFFIX_SRC,
TRANSFORMS_SRC);
}
static int transformDictionaryWord(byte[] dst, int dstOffset, ByteBuffer data, int wordOffset,
int len, Transform transform) {
int len, int transformIndex) {
int offset = dstOffset;
int transformOffset = 3 * transformIndex;
int transformPrefix = PREFIX_SUFFIX_HEADS[TRANSFORMS[transformOffset]];
int transformType = TRANSFORMS[transformOffset + 1];
int transformSuffix = PREFIX_SUFFIX_HEADS[TRANSFORMS[transformOffset + 2]];
// Copy prefix.
byte[] string = transform.prefix;
int tmp = string.length;
int i = 0;
// In most cases tmp < 10 -> no benefits from System.arrayCopy
while (i < tmp) {
dst[offset++] = string[i++];
while (PREFIX_SUFFIX[transformPrefix] != 0) {
dst[offset++] = PREFIX_SUFFIX[transformPrefix++];
}
// Copy trimmed word.
int op = transform.type;
tmp = WordTransformType.getOmitFirst(op);
if (tmp > len) {
tmp = len;
int omitFirst = transformType >= 12 ? (transformType - 11) : 0;
if (omitFirst > len) {
omitFirst = len;
}
wordOffset += tmp;
len -= tmp;
len -= WordTransformType.getOmitLast(op);
i = len;
wordOffset += omitFirst;
len -= omitFirst;
len -= transformType <= 9 ? transformType : 0; // Omit last.
int i = len;
while (i > 0) {
dst[offset++] = data.get(wordOffset++);
i--;
}
if (op == UPPERCASE_ALL || op == UPPERCASE_FIRST) {
// Ferment.
if (transformType == 11 || transformType == 10) {
int uppercaseOffset = offset - len;
if (op == UPPERCASE_FIRST) {
if (transformType == 10) {
len = 1;
}
while (len > 0) {
tmp = dst[uppercaseOffset] & 0xFF;
int tmp = dst[uppercaseOffset] & 0xFF;
if (tmp < 0xc0) {
if (tmp >= 'a' && tmp <= 'z') {
if (tmp >= 97 && tmp <= 122) { // in [a..z] range
dst[uppercaseOffset] ^= (byte) 32;
}
uppercaseOffset += 1;
@ -230,11 +106,8 @@ final class Transform {
}
// Copy suffix.
string = transform.suffix;
tmp = string.length;
i = 0;
while (i < tmp) {
dst[offset++] = string[i++];
while (PREFIX_SUFFIX[transformSuffix] != 0) {
dst[offset++] = PREFIX_SUFFIX[transformSuffix++];
}
return offset - dstOffset;

18
java/org/brotli/dec/TransformTest.java
View File

@ -34,23 +34,21 @@ public class TransformTest {
@Test
public void testTrimAll() {
byte[] output = new byte[2];
byte[] output = new byte[0];
byte[] input = {119, 111, 114, 100}; // "word"
Transform transform = new Transform("[", WordTransformType.OMIT_FIRST_5, "]");
Transform.transformDictionaryWord(
output, 0, ByteBuffer.wrap(input), 0, input.length, transform);
byte[] expectedOutput = {91, 93}; // "[]"
output, 0, ByteBuffer.wrap(input), 0, input.length, 39);
byte[] expectedOutput = new byte[0];
assertArrayEquals(expectedOutput, output);
}
@Test
public void testCapitalize() {
byte[] output = new byte[8];
byte[] output = new byte[6];
byte[] input = {113, -61, -90, -32, -92, -86}; // "qæप"
Transform transform = new Transform("[", WordTransformType.UPPERCASE_ALL, "]");
Transform.transformDictionaryWord(
output, 0, ByteBuffer.wrap(input), 0, input.length, transform);
byte[] expectedOutput = {91, 81, -61, -122, -32, -92, -81, 93}; // "[QÆय]"
output, 0, ByteBuffer.wrap(input), 0, input.length, 44);
byte[] expectedOutput = {81, -61, -122, -32, -92, -81}; // "QÆय"
assertArrayEquals(expectedOutput, output);
}
@ -62,9 +60,9 @@ public class TransformTest {
byte[] testWord = {111, 49, 50, 51, 52, 53, 54, 55, 56, 57, 97, 98, 99, 100, 101, 102};
byte[] output = new byte[2259];
int offset = 0;
for (int i = 0; i < Transform.TRANSFORMS.length; ++i) {
for (int i = 0; i < Transform.NUM_TRANSFORMS; ++i) {
offset += Transform.transformDictionaryWord(
output, offset, ByteBuffer.wrap(testWord), 0, testWord.length, Transform.TRANSFORMS[i]);
output, offset, ByteBuffer.wrap(testWord), 0, testWord.length, i);
output[offset++] = -1;
}
assertEquals(output.length, offset);

39
java/org/brotli/dec/Utils.java
View File

@ -6,6 +6,9 @@
package org.brotli.dec;
import java.io.IOException;
import java.io.InputStream;
/**
* A set of utility methods.
*/
@ -25,11 +28,11 @@ final class Utils {
* @param offset the first byte to fill
* @param length number of bytes to change
*/
static void fillWithZeroes(byte[] dest, int offset, int length) {
int cursor = 0;
while (cursor < length) {
int step = Math.min(cursor + 1024, length) - cursor;
System.arraycopy(BYTE_ZEROES, 0, dest, offset + cursor, step);
static void fillBytesWithZeroes(byte[] dest, int start, int end) {
int cursor = start;
while (cursor < end) {
int step = Math.min(cursor + 1024, end) - cursor;
System.arraycopy(BYTE_ZEROES, 0, dest, cursor, step);
cursor += step;
}
}
@ -44,12 +47,28 @@ final class Utils {
* @param offset the first item to fill
* @param length number of item to change
*/
static void fillWithZeroes(int[] dest, int offset, int length) {
int cursor = 0;
while (cursor < length) {
int step = Math.min(cursor + 1024, length) - cursor;
System.arraycopy(INT_ZEROES, 0, dest, offset + cursor, step);
static void fillIntsWithZeroes(int[] dest, int start, int end) {
int cursor = start;
while (cursor < end) {
int step = Math.min(cursor + 1024, end) - cursor;
System.arraycopy(INT_ZEROES, 0, dest, cursor, step);
cursor += step;
}
}
static void copyBytesWithin(byte[] bytes, int target, int start, int end) {
System.arraycopy(bytes, start, bytes, target, end - start);
}
static int readInput(InputStream src, byte[] dst, int offset, int length) {
try {
return src.read(dst, offset, length);
} catch (IOException e) {
throw new BrotliRuntimeException("Failed to read input", e);
}
}
static void closeInput(InputStream src) throws IOException {
src.close();
}
}

45
java/org/brotli/dec/WordTransformType.java
View File

@ -1,45 +0,0 @@
/* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
package org.brotli.dec;
/**
* Enumeration of all possible word transformations.
*
* <p>There are two simple types of transforms: omit X first/last symbols, two character-case
* transforms and the identity transform.
*/
final class WordTransformType {
static final int IDENTITY = 0;
static final int OMIT_LAST_1 = 1;
static final int OMIT_LAST_2 = 2;
static final int OMIT_LAST_3 = 3;
static final int OMIT_LAST_4 = 4;
static final int OMIT_LAST_5 = 5;
static final int OMIT_LAST_6 = 6;
static final int OMIT_LAST_7 = 7;
static final int OMIT_LAST_8 = 8;
static final int OMIT_LAST_9 = 9;
static final int UPPERCASE_FIRST = 10;
static final int UPPERCASE_ALL = 11;
static final int OMIT_FIRST_1 = 12;
static final int OMIT_FIRST_2 = 13;
static final int OMIT_FIRST_3 = 14;
static final int OMIT_FIRST_4 = 15;
static final int OMIT_FIRST_5 = 16;
static final int OMIT_FIRST_6 = 17;
static final int OMIT_FIRST_7 = 18;
static final int OMIT_FIRST_8 = 19;
static final int OMIT_FIRST_9 = 20;
static int getOmitFirst(int type) {
return type >= OMIT_FIRST_1 ? (type - OMIT_FIRST_1 + 1) : 0;
}
static int getOmitLast(int type) {
return type <= OMIT_LAST_9 ? (type - OMIT_LAST_1 + 1) : 0;
}
}

7
java/org/brotli/wrapper/dec/BrotliDecoderChannel.java
View File

@ -27,13 +27,8 @@ public class BrotliDecoderChannel extends Decoder implements ReadableByteChannel
* @param bufferSize intermediate buffer size
* @param customDictionary initial LZ77 dictionary
*/
public BrotliDecoderChannel(ReadableByteChannel source, int bufferSize,
ByteBuffer customDictionary) throws IOException {
super(source, bufferSize, customDictionary);
}
public BrotliDecoderChannel(ReadableByteChannel source, int bufferSize) throws IOException {
super(source, bufferSize, null);
super(source, bufferSize);
}
public BrotliDecoderChannel(ReadableByteChannel source) throws IOException {

10
java/org/brotli/wrapper/dec/BrotliInputStream.java
View File

@ -8,7 +8,6 @@ package org.brotli.wrapper.dec;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
/**
@ -25,15 +24,10 @@ public class BrotliInputStream extends InputStream {
*
* @param source underlying source
* @param bufferSize intermediate buffer size
* @param customDictionary initial LZ77 dictionary
*/
public BrotliInputStream(InputStream source, int bufferSize, ByteBuffer customDictionary)
public BrotliInputStream(InputStream source, int bufferSize)
throws IOException {
this.decoder = new Decoder(Channels.newChannel(source), bufferSize, customDictionary);
}
public BrotliInputStream(InputStream source, int bufferSize) throws IOException {
this.decoder = new Decoder(Channels.newChannel(source), bufferSize, null);
this.decoder = new Decoder(Channels.newChannel(source), bufferSize);
}
public BrotliInputStream(InputStream source) throws IOException {

8
java/org/brotli/wrapper/dec/Decoder.java
View File

@ -14,7 +14,7 @@ import java.util.ArrayList;
/**
* Base class for InputStream / Channel implementations.
*/
class Decoder {
public class Decoder {
private final ReadableByteChannel source;
private final DecoderJNI.Wrapper decoder;
ByteBuffer buffer;
@ -26,7 +26,7 @@ class Decoder {
* @param source underlying source
* @param inputBufferSize read buffer size
*/
public Decoder(ReadableByteChannel source, int inputBufferSize, ByteBuffer customDictionary)
public Decoder(ReadableByteChannel source, int inputBufferSize)
throws IOException {
if (inputBufferSize <= 0) {
throw new IllegalArgumentException("buffer size must be positive");
@ -35,7 +35,7 @@ class Decoder {
throw new NullPointerException("source can not be null");
}
this.source = source;
this.decoder = new DecoderJNI.Wrapper(inputBufferSize, customDictionary);
this.decoder = new DecoderJNI.Wrapper(inputBufferSize);
}
private void fail(String message) throws IOException {
@ -119,7 +119,7 @@ class Decoder {
* Decodes the given data buffer.
*/
public static byte[] decompress(byte[] data) throws IOException {
DecoderJNI.Wrapper decoder = new DecoderJNI.Wrapper(data.length, null);
DecoderJNI.Wrapper decoder = new DecoderJNI.Wrapper(data.length);
ArrayList<byte[]> output = new ArrayList<byte[]>();
int totalOutputSize = 0;
try {

9
java/org/brotli/wrapper/dec/DecoderJNI.java
View File

@ -13,7 +13,7 @@ import java.nio.ByteBuffer;
* JNI wrapper for brotli decoder.
*/
class DecoderJNI {
private static native ByteBuffer nativeCreate(long[] context, ByteBuffer customDictionary);
private static native ByteBuffer nativeCreate(long[] context);
private static native void nativePush(long[] context, int length);
private static native ByteBuffer nativePull(long[] context);
private static native void nativeDestroy(long[] context);
@ -31,12 +31,9 @@ class DecoderJNI {
private final ByteBuffer inputBuffer;
private Status lastStatus = Status.NEEDS_MORE_INPUT;
Wrapper(int inputBufferSize, ByteBuffer customDictionary) throws IOException {
if (customDictionary != null && !customDictionary.isDirect()) {
throw new IllegalArgumentException("LZ77 dictionary must be direct ByteBuffer");
}
Wrapper(int inputBufferSize) throws IOException {
this.context[1] = inputBufferSize;
this.inputBuffer = nativeCreate(this.context, customDictionary);
this.inputBuffer = nativeCreate(this.context);
if (this.context[0] == 0) {
throw new IOException("failed to initialize native brotli decoder");
}

33
java/org/brotli/wrapper/dec/decoder_jni.cc
View File

@ -15,8 +15,6 @@ namespace {
typedef struct DecoderHandle {
BrotliDecoderState* state;
jobject custom_dictionary_ref;
uint8_t* input_start;
size_t input_offset;
size_t input_length;
@ -44,7 +42,7 @@ extern "C" {
*/
JNIEXPORT jobject JNICALL
Java_org_brotli_wrapper_dec_DecoderJNI_nativeCreate(
JNIEnv* env, jobject /*jobj*/, jlongArray ctx, jobject custom_dictionary) {
JNIEnv* env, jobject /*jobj*/, jlongArray ctx) {
bool ok = true;
DecoderHandle* handle = nullptr;
jlong context[2];
@ -55,7 +53,6 @@ Java_org_brotli_wrapper_dec_DecoderJNI_nativeCreate(
ok = !!handle;
if (ok) {
handle->custom_dictionary_ref = nullptr;
handle->input_offset = 0;
handle->input_length = 0;
handle->input_start = nullptr;
@ -73,36 +70,11 @@ Java_org_brotli_wrapper_dec_DecoderJNI_nativeCreate(
ok = !!handle->state;
}
if (ok && !!custom_dictionary) {
handle->custom_dictionary_ref = env->NewGlobalRef(custom_dictionary);
if (!!handle->custom_dictionary_ref) {
uint8_t* custom_dictionary_address = static_cast<uint8_t*>(
env->GetDirectBufferAddress(handle->custom_dictionary_ref));
if (!!custom_dictionary_address) {
jlong capacity =
env->GetDirectBufferCapacity(handle->custom_dictionary_ref);
ok = (capacity > 0) && (capacity < (1 << 24));
if (ok) {
size_t custom_dictionary_size = static_cast<size_t>(capacity);
BrotliDecoderSetCustomDictionary(
handle->state, custom_dictionary_size, custom_dictionary_address);
}
} else {
ok = false;
}
} else {
ok = false;
}
}
if (ok) {
/* TODO: future versions (e.g. when 128-bit architecture comes)
might require thread-safe cookie<->handle mapping. */
context[0] = reinterpret_cast<jlong>(handle);
} else if (!!handle) {
if (!!handle->custom_dictionary_ref) {
env->DeleteGlobalRef(handle->custom_dictionary_ref);
}
if (!!handle->input_start) delete[] handle->input_start;
delete handle;
}
@ -216,9 +188,6 @@ Java_org_brotli_wrapper_dec_DecoderJNI_nativeDestroy(
env->GetLongArrayRegion(ctx, 0, 2, context);
DecoderHandle* handle = getHandle(reinterpret_cast<void*>(context[0]));
BrotliDecoderDestroyInstance(handle->state);
if (!!handle->custom_dictionary_ref) {
env->DeleteGlobalRef(handle->custom_dictionary_ref);
}
delete[] handle->input_start;
delete handle;
}

19
java/org/brotli/wrapper/enc/BUILD
View File

@ -77,22 +77,3 @@ java_test(
"@junit_junit//jar",
],
)
java_test(
name = "UseCustomDictionaryTest",
size = "large",
srcs = ["UseCustomDictionaryTest.java"],
data = [
":test_bundle",
"//:jni", # Bazel JNI workaround
],
jvm_flags = ["-DTEST_BUNDLE=$(location :test_bundle)"],
shard_count = 15,
deps = [
":enc",
"//java/org/brotli/integration:bundle_helper",
"//java/org/brotli/wrapper/common",
"//java/org/brotli/wrapper/dec",
"@junit_junit//jar",
],
)

8
java/org/brotli/wrapper/enc/BrotliEncoderChannel.java
View File

@ -26,16 +26,10 @@ public class BrotliEncoderChannel extends Encoder implements WritableByteChannel
* @param destination underlying destination
* @param params encoding settings
* @param bufferSize intermediate buffer size
* @param customDictionary initial LZ77 dictionary
*/
public BrotliEncoderChannel(WritableByteChannel destination, Encoder.Parameters params,
int bufferSize, ByteBuffer customDictionary) throws IOException {
super(destination, params, bufferSize, customDictionary);
}
public BrotliEncoderChannel(WritableByteChannel destination, Encoder.Parameters params,
int bufferSize) throws IOException {
super(destination, params, bufferSize, null);
super(destination, params, bufferSize);
}
public BrotliEncoderChannel(WritableByteChannel destination, Encoder.Parameters params)

10
java/org/brotli/wrapper/enc/BrotliOutputStream.java
View File

@ -8,7 +8,6 @@ package org.brotli.wrapper.enc;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
/**
@ -26,17 +25,10 @@ public class BrotliOutputStream extends OutputStream {
* @param destination underlying destination
* @param params encoding settings
* @param bufferSize intermediate buffer size
* @param customDictionary initial LZ77 dictionary
*/
public BrotliOutputStream(OutputStream destination, Encoder.Parameters params, int bufferSize,
ByteBuffer customDictionary) throws IOException {
this.encoder = new Encoder(
Channels.newChannel(destination), params, bufferSize, customDictionary);
}
public BrotliOutputStream(OutputStream destination, Encoder.Parameters params, int bufferSize)
throws IOException {
this.encoder = new Encoder(Channels.newChannel(destination), params, bufferSize, null);
this.encoder = new Encoder(Channels.newChannel(destination), params, bufferSize);
}
public BrotliOutputStream(OutputStream destination, Encoder.Parameters params)

10
java/org/brotli/wrapper/enc/Encoder.java
View File

@ -65,8 +65,8 @@ public class Encoder {
* @param params encoding parameters
* @param inputBufferSize read buffer size
*/
Encoder(WritableByteChannel destination, Parameters params, int inputBufferSize,
ByteBuffer customDictionary) throws IOException {
Encoder(WritableByteChannel destination, Parameters params, int inputBufferSize)
throws IOException {
if (inputBufferSize <= 0) {
throw new IllegalArgumentException("buffer size must be positive");
}
@ -74,8 +74,7 @@ public class Encoder {
throw new NullPointerException("destination can not be null");
}
this.destination = destination;
this.encoder = new EncoderJNI.Wrapper(
inputBufferSize, params.quality, params.lgwin, customDictionary);
this.encoder = new EncoderJNI.Wrapper(inputBufferSize, params.quality, params.lgwin);
this.inputBuffer = this.encoder.getInputBuffer();
}
@ -157,8 +156,7 @@ public class Encoder {
* Encodes the given data buffer.
*/
public static byte[] compress(byte[] data, Parameters params) throws IOException {
EncoderJNI.Wrapper encoder = new EncoderJNI.Wrapper(
data.length, params.quality, params.lgwin, null);
EncoderJNI.Wrapper encoder = new EncoderJNI.Wrapper(data.length, params.quality, params.lgwin);
ArrayList<byte[]> output = new ArrayList<byte[]>();
int totalOutputSize = 0;
try {

9
java/org/brotli/wrapper/enc/EncoderJNI.java
View File

@ -13,7 +13,7 @@ import java.nio.ByteBuffer;
* JNI wrapper for brotli encoder.
*/
class EncoderJNI {
private static native ByteBuffer nativeCreate(long[] context, ByteBuffer customDictionary);
private static native ByteBuffer nativeCreate(long[] context);
private static native void nativePush(long[] context, int length);
private static native ByteBuffer nativePull(long[] context);
private static native void nativeDestroy(long[] context);
@ -28,15 +28,12 @@ class EncoderJNI {
protected final long[] context = new long[4];
private final ByteBuffer inputBuffer;
Wrapper(int inputBufferSize, int quality, int lgwin, ByteBuffer customDictionary)
Wrapper(int inputBufferSize, int quality, int lgwin)
throws IOException {
if (customDictionary != null && !customDictionary.isDirect()) {
throw new IllegalArgumentException("LZ77 dictionary must be direct ByteBuffer");
}
this.context[1] = inputBufferSize;
this.context[2] = quality;
this.context[3] = lgwin;
this.inputBuffer = nativeCreate(this.context, customDictionary);
this.inputBuffer = nativeCreate(this.context);
if (this.context[0] == 0) {
throw new IOException("failed to initialize native brotli encoder");
}

104
java/org/brotli/wrapper/enc/UseCustomDictionaryTest.java
View File

@ -1,104 +0,0 @@
package org.brotli.wrapper.enc;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.brotli.integration.BundleHelper;
import org.brotli.wrapper.common.BrotliCommon;
import org.brotli.wrapper.dec.BrotliInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.List;
import junit.framework.TestCase;
import junit.framework.TestSuite;
import org.junit.runner.RunWith;
import org.junit.runners.AllTests;
/** Tests for compression / decompression aided with LZ77 dictionary. */
@RunWith(AllTests.class)
public class UseCustomDictionaryTest {
// TODO: remove when Bazel get JNI support.
static {
System.load(new java.io.File(new java.io.File(System.getProperty("java.library.path")),
"liblibjni.so").getAbsolutePath());
}
static InputStream getBundle() throws IOException {
return new FileInputStream(System.getProperty("TEST_BUNDLE"));
}
/** Creates a test suite. */
public static TestSuite suite() throws IOException {
TestSuite suite = new TestSuite();
InputStream bundle = getBundle();
try {
List<String> entries = BundleHelper.listEntries(bundle);
for (String entry : entries) {
suite.addTest(new UseCustomDictionaryTestCase(entry));
}
} finally {
bundle.close();
}
return suite;
}
/** Test case with a unique name. */
static class UseCustomDictionaryTestCase extends TestCase {
final String entryName;
UseCustomDictionaryTestCase(String entryName) {
super("UseCustomDictionaryTest." + entryName);
this.entryName = entryName;
}
@Override
protected void runTest() throws Throwable {
UseCustomDictionaryTest.run(entryName);
}
}
private static void run(String entryName) throws Throwable {
InputStream bundle = getBundle();
byte[] original;
try {
original = BundleHelper.readEntry(bundle, entryName);
} finally {
bundle.close();
}
if (original == null) {
throw new RuntimeException("Can't read bundle entry: " + entryName);
}
ByteBuffer dictionary = BrotliCommon.makeNative(original);
ByteArrayOutputStream dst = new ByteArrayOutputStream();
OutputStream encoder = new BrotliOutputStream(dst,
new Encoder.Parameters().setQuality(11).setWindow(23), 1 << 23, dictionary);
try {
encoder.write(original);
} finally {
encoder.close();
}
byte[] compressed = dst.toByteArray();
// Just copy self from LZ77 dictionary -> ultimate compression ratio.
assertTrue(compressed.length < 80 + original.length / 65536);
InputStream decoder = new BrotliInputStream(new ByteArrayInputStream(compressed),
1 << 23, dictionary);
try {
long originalCrc = BundleHelper.fingerprintStream(new ByteArrayInputStream(original));
long crc = BundleHelper.fingerprintStream(decoder);
assertEquals(originalCrc, crc);
} finally {
decoder.close();
}
}
}

33
java/org/brotli/wrapper/enc/encoder_jni.cc
View File

@ -15,8 +15,6 @@ namespace {
typedef struct EncoderHandle {
BrotliEncoderState* state;
jobject custom_dictionary_ref;
uint8_t* input_start;
size_t input_offset;
size_t input_last;
@ -44,7 +42,7 @@ extern "C" {
*/
JNIEXPORT jobject JNICALL
Java_org_brotli_wrapper_enc_EncoderJNI_nativeCreate(
JNIEnv* env, jobject /*jobj*/, jlongArray ctx, jobject custom_dictionary) {
JNIEnv* env, jobject /*jobj*/, jlongArray ctx) {
bool ok = true;
EncoderHandle* handle = nullptr;
jlong context[4];
@ -55,7 +53,6 @@ Java_org_brotli_wrapper_enc_EncoderJNI_nativeCreate(
ok = !!handle;
if (ok) {
handle->custom_dictionary_ref = nullptr;
handle->input_offset = 0;
handle->input_last = 0;
handle->input_start = nullptr;
@ -84,36 +81,11 @@ Java_org_brotli_wrapper_enc_EncoderJNI_nativeCreate(
}
}
if (ok && !!custom_dictionary) {
handle->custom_dictionary_ref = env->NewGlobalRef(custom_dictionary);
if (!!handle->custom_dictionary_ref) {
uint8_t* custom_dictionary_address = static_cast<uint8_t*>(
env->GetDirectBufferAddress(handle->custom_dictionary_ref));
if (!!custom_dictionary_address) {
jlong capacity =
env->GetDirectBufferCapacity(handle->custom_dictionary_ref);
ok = (capacity > 0) && (capacity < (1 << 24));
if (ok) {
size_t custom_dictionary_size = static_cast<size_t>(capacity);
BrotliEncoderSetCustomDictionary(
handle->state, custom_dictionary_size, custom_dictionary_address);
}
} else {
ok = false;
}
} else {
ok = false;
}
}
if (ok) {
/* TODO: future versions (e.g. when 128-bit architecture comes)
might require thread-safe cookie<->handle mapping. */
context[0] = reinterpret_cast<jlong>(handle);
} else if (!!handle) {
if (!!handle->custom_dictionary_ref) {
env->DeleteGlobalRef(handle->custom_dictionary_ref);
}
if (!!handle->input_start) delete[] handle->input_start;
delete handle;
}
@ -212,9 +184,6 @@ Java_org_brotli_wrapper_enc_EncoderJNI_nativeDestroy(
env->GetLongArrayRegion(ctx, 0, 2, context);
EncoderHandle* handle = getHandle(reinterpret_cast<void*>(context[0]));
BrotliEncoderDestroyInstance(handle->state);
if (!!handle->custom_dictionary_ref) {
env->DeleteGlobalRef(handle->custom_dictionary_ref);
}
delete[] handle->input_start;
delete handle;
}

63
python/_brotli.cc
View File

@ -125,7 +125,7 @@ PyDoc_STRVAR(brotli_Compressor_doc,
"An object to compress a byte string.\n"
"\n"
"Signature:\n"
" Compressor(mode=MODE_GENERIC, quality=11, lgwin=22, lgblock=0, dictionary='')\n"
" Compressor(mode=MODE_GENERIC, quality=11, lgwin=22, lgblock=0)\n"
"\n"
"Args:\n"
" mode (int, optional): The compression mode can be MODE_GENERIC (default),\n"
@ -138,8 +138,6 @@ PyDoc_STRVAR(brotli_Compressor_doc,
" lgblock (int, optional): Base 2 logarithm of the maximum input block size.\n"
" Range is 16 to 24. If set to 0, the value will be set based on the\n"
" quality. Defaults to 0.\n"
" dictionary (bytes, optional): Custom dictionary. Only last sliding window\n"
" size bytes will be used.\n"
"\n"
"Raises:\n"
" brotli.error: If arguments are invalid.\n");
@ -174,20 +172,16 @@ static int brotli_Compressor_init(brotli_Compressor *self, PyObject *args, PyObj
int quality = -1;
int lgwin = -1;
int lgblock = -1;
uint8_t* custom_dictionary = NULL;
size_t custom_dictionary_length = 0;
int ok;
static const char *kwlist[] = {
"mode", "quality", "lgwin", "lgblock", "dictionary", NULL};
static const char *kwlist[] = {"mode", "quality", "lgwin", "lgblock", NULL};
ok = PyArg_ParseTupleAndKeywords(args, keywds, "|O&O&O&O&s#:Compressor",
ok = PyArg_ParseTupleAndKeywords(args, keywds, "|O&O&O&O&:Compressor",
const_cast<char **>(kwlist),
&mode_convertor, &mode,
&quality_convertor, &quality,
&lgwin_convertor, &lgwin,
&lgblock_convertor, &lgblock,
&custom_dictionary, &custom_dictionary_length);
&lgblock_convertor, &lgblock);
if (!ok)
return -1;
if (!self->enc)
@ -202,15 +196,6 @@ static int brotli_Compressor_init(brotli_Compressor *self, PyObject *args, PyObj
if (lgblock != -1)
BrotliEncoderSetParameter(self->enc, BROTLI_PARAM_LGBLOCK, (uint32_t)lgblock);
if (custom_dictionary_length != 0) {
/* Unlike decoder, encoder processes dictionary immediately, that is why
it makes sense to release python GIL. */
Py_BEGIN_ALLOW_THREADS
BrotliEncoderSetCustomDictionary(self->enc, custom_dictionary_length,
custom_dictionary);
Py_END_ALLOW_THREADS
}
return 0;
}
@ -432,11 +417,7 @@ PyDoc_STRVAR(brotli_Decompressor_doc,
"An object to decompress a byte string.\n"
"\n"
"Signature:\n"
" Decompressor(dictionary='')\n"
"\n"
"Args:\n"
" dictionary (bytes, optional): Custom dictionary. Only last sliding window\n"
" size bytes will be used.\n"
" Decompressor()\n"
"\n"
"Raises:\n"
" brotli.error: If arguments are invalid.\n");
@ -467,26 +448,17 @@ static PyObject* brotli_Decompressor_new(PyTypeObject *type, PyObject *args, PyO
}
static int brotli_Decompressor_init(brotli_Decompressor *self, PyObject *args, PyObject *keywds) {
uint8_t* custom_dictionary = NULL;
size_t custom_dictionary_length = 0;
int ok;
static const char *kwlist[] = {
"dictionary", NULL};
static const char *kwlist[] = {NULL};
ok = PyArg_ParseTupleAndKeywords(args, keywds, "|s#:Decompressor",
const_cast<char **>(kwlist),
&custom_dictionary, &custom_dictionary_length);
ok = PyArg_ParseTupleAndKeywords(args, keywds, "|:Decompressor",
const_cast<char **>(kwlist));
if (!ok)
return -1;
if (!self->dec)
return -1;
if (custom_dictionary_length != 0) {
BrotliDecoderSetCustomDictionary(self->dec, custom_dictionary_length,
custom_dictionary);
}
return 0;
}
@ -644,8 +616,6 @@ PyDoc_STRVAR(brotli_decompress__doc__,
"\n"
"Args:\n"
" string (bytes): The compressed input data.\n"
" dictionary (bytes, optional): Custom dictionary. MUST be the same data\n"
" as passed to compress method.\n"
"\n"
"Returns:\n"
" The decompressed byte string.\n"
@ -655,19 +625,15 @@ PyDoc_STRVAR(brotli_decompress__doc__,
static PyObject* brotli_decompress(PyObject *self, PyObject *args, PyObject *keywds) {
PyObject *ret = NULL;
const uint8_t *input, *custom_dictionary;
size_t length, custom_dictionary_length;
const uint8_t *input;
size_t length;
int ok;
static const char *kwlist[] = {"string", "dictionary", NULL};
static const char *kwlist[] = {"string", NULL};
custom_dictionary = NULL;
custom_dictionary_length = 0;
ok = PyArg_ParseTupleAndKeywords(args, keywds, "s#|s#:decompress",
ok = PyArg_ParseTupleAndKeywords(args, keywds, "s#|:decompress",
const_cast<char **>(kwlist),
&input, &length,
&custom_dictionary, &custom_dictionary_length);
&input, &length);
if (!ok)
return NULL;
@ -677,9 +643,6 @@ static PyObject* brotli_decompress(PyObject *self, PyObject *args, PyObject *key
Py_BEGIN_ALLOW_THREADS
BrotliDecoderState* state = BrotliDecoderCreateInstance(0, 0, 0);
if (custom_dictionary_length != 0) {
BrotliDecoderSetCustomDictionary(state, custom_dictionary_length, custom_dictionary);
}
BrotliDecoderResult result = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT;
while (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {

20
python/bro.py
View File

@ -114,13 +114,6 @@ def main(args=None):
help='Base 2 logarithm of the maximum input block size. '
'Range is 16 to 24. If set to 0, the value will be set based '
'on the quality. Defaults to 0.')
params.add_argument(
'--custom-dictionary',
metavar='FILE',
type=str,
dest='dictfile',
help='Custom dictionary file.',
default=None)
# set default values using global DEFAULT_PARAMS dictionary
parser.set_defaults(**DEFAULT_PARAMS)
@ -145,25 +138,16 @@ def main(args=None):
else:
outfile = get_binary_stdio('stdout')
if options.dictfile:
if not os.path.isfile(options.dictfile):
parser.error('file "%s" not found' % options.dictfile)
with open(options.dictfile, 'rb') as dictfile:
custom_dictionary = dictfile.read()
else:
custom_dictionary = ''
try:
if options.decompress:
data = brotli.decompress(data, dictionary=custom_dictionary)
data = brotli.decompress(data)
else:
data = brotli.compress(
data,
mode=options.mode,
quality=options.quality,
lgwin=options.lgwin,
lgblock=options.lgblock,
dictionary=custom_dictionary)
lgblock=options.lgblock)
except brotli.error as e:
parser.exit(1,
'bro: error: %s: %s' % (e, options.infile or 'sys.stdin'))

7
python/brotli.py
View File

@ -23,8 +23,7 @@ Compressor = _brotli.Compressor
Decompressor = _brotli.Decompressor
# Compress a byte string.
def compress(string, mode=MODE_GENERIC, quality=11, lgwin=22, lgblock=0,
dictionary=''):
def compress(string, mode=MODE_GENERIC, quality=11, lgwin=22, lgblock=0):
"""Compress a byte string.
Args:
@ -39,8 +38,6 @@ def compress(string, mode=MODE_GENERIC, quality=11, lgwin=22, lgblock=0,
lgblock (int, optional): Base 2 logarithm of the maximum input block size.
Range is 16 to 24. If set to 0, the value will be set based on the
quality. Defaults to 0.
dictionary (bytes, optional): Custom dictionary. Only last sliding window
size bytes will be used.
Returns:
The compressed byte string.
@ -49,7 +46,7 @@ def compress(string, mode=MODE_GENERIC, quality=11, lgwin=22, lgblock=0,
brotli.error: If arguments are invalid, or compressor fails.
"""
compressor = Compressor(mode=mode, quality=quality, lgwin=lgwin,
lgblock=lgblock, dictionary=dictionary)
lgblock=lgblock)
return compressor.process(string) + compressor.finish()
# Decompress a compressed byte string.

16
python/tests/bro_test.py
View File

@ -62,8 +62,6 @@ class TestBroCompress(_test_utils.TestCase):
temp_compressed = _test_utils.get_temp_compressed_name(test_data)
original = test_data
args = [PYTHON, BRO, '-f', '-d']
if 'dictionary' in kwargs:
args.extend(['--custom-dictionary', str(kwargs['dictionary'])])
args.extend(['-i', temp_compressed, '-o', temp_uncompressed])
subprocess.check_call(args, env=TEST_ENV)
self.assertFilesMatch(temp_uncompressed, original)
@ -75,8 +73,6 @@ class TestBroCompress(_test_utils.TestCase):
args.extend(['-q', str(kwargs['quality'])])
if 'lgwin' in kwargs:
args.extend(['--lgwin', str(kwargs['lgwin'])])
if 'dictionary' in kwargs:
args.extend(['--custom-dictionary', str(kwargs['dictionary'])])
args.extend(['-i', test_data, '-o', temp_compressed])
subprocess.check_call(args, env=TEST_ENV)
@ -87,8 +83,6 @@ class TestBroCompress(_test_utils.TestCase):
args.extend(['-q', str(kwargs['quality'])])
if 'lgwin' in kwargs:
args.extend(['--lgwin', str(kwargs['lgwin'])])
if 'dictionary' in kwargs:
args.extend(['--custom-dictionary', str(kwargs['dictionary'])])
with open(temp_compressed, 'wb') as out_file:
with open(test_data, 'rb') as in_file:
subprocess.check_call(
@ -102,16 +96,6 @@ class TestBroCompress(_test_utils.TestCase):
self._compress_pipe(test_data, **kwargs)
self._check_decompression(test_data)
def _test_compress_file_custom_dictionary(self, test_data, **kwargs):
kwargs['dictionary'] = test_data
self._compress_file(test_data, **kwargs)
self._check_decompression(test_data, **kwargs)
def _test_compress_pipe_custom_dictionary(self, test_data, **kwargs):
kwargs['dictionary'] = test_data
self._compress_pipe(test_data, **kwargs)
self._check_decompression(test_data, **kwargs)
_test_utils.generate_test_methods(
TestBroCompress, variants=TestBroCompress.VARIANTS)

11
python/tests/compress_test.py
View File

@ -14,10 +14,6 @@ class TestCompress(_test_utils.TestCase):
VARIANTS = {'quality': (1, 6, 9, 11), 'lgwin': (10, 15, 20, 24)}
def _check_decompression(self, test_data, **kwargs):
# Only dictionary is supported as a kwarg to brotli.decompress.
if 'dictionary' in kwargs:
kwargs = {'dictionary': kwargs['dictionary']}
else:
kwargs = {}
# Write decompression to temp file and verify it matches the original.
temp_uncompressed = _test_utils.get_temp_uncompressed_name(test_data)
@ -38,13 +34,6 @@ class TestCompress(_test_utils.TestCase):
self._compress(test_data, **kwargs)
self._check_decompression(test_data, **kwargs)
def _test_compress_custom_dictionary(self, test_data, **kwargs):
with open(test_data, 'rb') as in_file:
dictionary = in_file.read()
kwargs['dictionary'] = dictionary
self._compress(test_data, **kwargs)
self._check_decompression(test_data, **kwargs)
_test_utils.generate_test_methods(TestCompress, variants=TestCompress.VARIANTS)