diff --git a/c/common/constants.h b/c/common/constants.h index 7b3d6a5..416ec55 100644 --- a/c/common/constants.h +++ b/c/common/constants.h @@ -38,6 +38,8 @@ BROTLI_MAX_NDIRECT + \ (BROTLI_MAX_DISTANCE_BITS << \ (BROTLI_MAX_NPOSTFIX + 1))) +/* Distance that is guaranteed to be representable in any stream. */ +#define BROTLI_MAX_DISTANCE 0x3FFFFFC /* 7.1. Context modes and context ID lookup for literals */ /* "context IDs for literals are in the range of 0..63" */ diff --git a/c/dec/decode.c b/c/dec/decode.c index ce97cba..fa5fe48 100644 --- a/c/dec/decode.c +++ b/c/dec/decode.c @@ -1226,7 +1226,9 @@ static BrotliDecoderErrorCode BROTLI_NOINLINE WriteRingBuffer( BROTLI_LOG_UINT(to_write); BROTLI_LOG_UINT(num_written); s->partial_pos_out += num_written; - if (total_out) *total_out = s->partial_pos_out - (size_t)s->custom_dict_size; + if (total_out) { + *total_out = s->partial_pos_out; + } if (num_written < to_write) { if (s->ringbuffer_size == (1 << s->window_bits) || force) { return BROTLI_DECODER_NEEDS_MORE_OUTPUT; @@ -1278,13 +1280,7 @@ static BROTLI_BOOL BROTLI_NOINLINE BrotliEnsureRingBuffer( s->ringbuffer[s->new_ringbuffer_size - 2] = 0; s->ringbuffer[s->new_ringbuffer_size - 1] = 0; - if (!old_ringbuffer) { - if (s->custom_dict) { - memcpy(s->ringbuffer, s->custom_dict, (size_t)s->custom_dict_size); - s->partial_pos_out = (size_t)s->custom_dict_size; - s->pos = s->custom_dict_size; - } - } else { + if (!!old_ringbuffer) { memcpy(s->ringbuffer, old_ringbuffer, (size_t)s->pos); BROTLI_FREE(s, old_ringbuffer); } @@ -1373,8 +1369,7 @@ static void BROTLI_NOINLINE BrotliCalculateRingBufferSize( } if (!s->ringbuffer) { - /* Custom dictionary counts as a "virtual" output. */ - output_size = s->custom_dict_size; + output_size = 0; } else { output_size = s->pos; } @@ -1752,14 +1747,14 @@ CommandPostDecodeLiterals: /* Apply copy of LZ77 back-reference, or static dictionary reference if the distance is larger than the max LZ77 distance */ if (s->distance_code > s->max_distance) { + int address = s->distance_code - s->max_distance - 1; if (i >= BROTLI_MIN_DICTIONARY_WORD_LENGTH && i <= BROTLI_MAX_DICTIONARY_WORD_LENGTH) { int offset = (int)s->dictionary->offsets_by_length[i]; - int word_id = s->distance_code - s->max_distance - 1; uint32_t shift = s->dictionary->size_bits_by_length[i]; int mask = (int)BitMask(shift); - int word_idx = word_id & mask; - int transform_idx = word_id >> shift; + int word_idx = address & mask; + int transform_idx = address >> shift; /* Compensate double distance-ring-buffer roll. */ s->dist_rb_idx += s->distance_context; offset += word_idx * i; @@ -2022,11 +2017,6 @@ BrotliDecoderResult BrotliDecoderDecompressStream( } /* Maximum distance, see section 9.1. of the spec. */ s->max_backward_distance = (1 << s->window_bits) - BROTLI_WINDOW_GAP; - /* Limit custom dictionary size. */ - if (s->custom_dict_size >= s->max_backward_distance) { - s->custom_dict += s->custom_dict_size - s->max_backward_distance; - s->custom_dict_size = s->max_backward_distance; - } /* Allocate memory for both block_type_trees and block_len_trees. */ s->block_type_trees = (HuffmanCode*)BROTLI_ALLOC(s, @@ -2323,15 +2313,6 @@ BrotliDecoderResult BrotliDecoderDecompressStream( return SaveErrorCode(s, result); } -void BrotliDecoderSetCustomDictionary( - BrotliDecoderState* s, size_t size, const uint8_t* dict) { - if (size > (1u << 24)) { - return; - } - s->custom_dict = dict; - s->custom_dict_size = (int)size; -} - BROTLI_BOOL BrotliDecoderHasMoreOutput(const BrotliDecoderState* s) { /* After unrecoverable error remaining output is considered nonsensical. */ if ((int)s->error_code < 0) { diff --git a/c/dec/state.c b/c/dec/state.c index 0db73fb..554313d 100644 --- a/c/dec/state.c +++ b/c/dec/state.c @@ -83,9 +83,6 @@ void BrotliDecoderStateInitWithCustomAllocators(BrotliDecoderState* s, s->distance_hgroup.codes = NULL; s->distance_hgroup.htrees = NULL; - s->custom_dict = NULL; - s->custom_dict_size = 0; - s->is_last_metablock = 0; s->is_uncompressed = 0; s->is_metadata = 0; diff --git a/c/dec/state.h b/c/dec/state.h index 00c373b..1d2773b 100644 --- a/c/dec/state.h +++ b/c/dec/state.h @@ -197,10 +197,6 @@ struct BrotliDecoderStateStruct { uint32_t mtf_upper_bound; uint32_t mtf[64 + 1]; - /* For custom dictionaries */ - const uint8_t* custom_dict; - int custom_dict_size; - /* less used attributes are in the end of this struct */ /* States inside function calls */ BrotliRunningMetablockHeaderState substate_metablock_header; diff --git a/c/enc/backward_references.c b/c/enc/backward_references.c index 39a74c2..3ac7f2f 100644 --- a/c/enc/backward_references.c +++ b/c/enc/backward_references.c @@ -48,6 +48,8 @@ static BROTLI_INLINE size_t ComputeDistanceCode(size_t distance, #define EXPAND_CAT(a, b) CAT(a, b) #define CAT(a, b) a ## b #define FN(X) EXPAND_CAT(X, HASHER()) +#define EXPORT_FN(X) EXPAND_CAT(X, EXPAND_CAT(PREFIX(), HASHER())) +#define PREFIX() N #define HASHER() H2 /* NOLINTNEXTLINE(build/include) */ @@ -94,6 +96,8 @@ static BROTLI_INLINE size_t ComputeDistanceCode(size_t distance, #include "./backward_references_inc.h" #undef HASHER +#undef PREFIX +#undef EXPORT_FN #undef FN #undef CAT #undef EXPAND_CAT @@ -113,11 +117,11 @@ void BrotliCreateBackwardReferences(const BrotliDictionary* dictionary, switch (params->hasher.type) { #define CASE_(N) \ case N: \ - CreateBackwardReferencesH ## N(dictionary, \ + CreateBackwardReferencesNH ## N(dictionary, \ kStaticDictionaryHash, num_bytes, position, ringbuffer, \ ringbuffer_mask, params, hasher, dist_cache, \ last_insert_len, commands, num_commands, num_literals); \ - break; + return; FOR_GENERIC_HASHERS(CASE_) #undef CASE_ default: diff --git a/c/enc/backward_references_hq.c b/c/enc/backward_references_hq.c index d766487..4545e80 100755 --- a/c/enc/backward_references_hq.c +++ b/c/enc/backward_references_hq.c @@ -629,6 +629,7 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m, const size_t store_end = num_bytes >= StoreLookaheadH10() ? position + num_bytes - StoreLookaheadH10() + 1 : position; size_t i; + size_t gap = 0; nodes[0].length = 0; nodes[0].u.cost = 0; InitZopfliCostModel(m, &model, num_bytes); @@ -640,7 +641,8 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m, const size_t pos = position + i; const size_t max_distance = BROTLI_MIN(size_t, pos, max_backward_limit); size_t num_matches = FindAllMatchesH10(hasher, dictionary, ringbuffer, - ringbuffer_mask, pos, num_bytes - i, max_distance, params, matches); + ringbuffer_mask, pos, num_bytes - i, max_distance, gap, params, + matches); size_t skip; if (num_matches > 0 && BackwardMatchLength(&matches[num_matches - 1]) > max_zopfli_len) { @@ -712,6 +714,7 @@ void BrotliCreateHqZopfliBackwardReferences( ZopfliCostModel model; ZopfliNode* nodes; BackwardMatch* matches = BROTLI_ALLOC(m, BackwardMatch, matches_size); + size_t gap = 0; if (BROTLI_IS_OOM(m)) return; for (i = 0; i + HashTypeLengthH10() - 1 < num_bytes; ++i) { const size_t pos = position + i; @@ -725,7 +728,7 @@ void BrotliCreateHqZopfliBackwardReferences( cur_match_pos + MAX_NUM_MATCHES_H10); if (BROTLI_IS_OOM(m)) return; num_found_matches = FindAllMatchesH10(hasher, dictionary, ringbuffer, - ringbuffer_mask, pos, max_length, max_distance, params, + ringbuffer_mask, pos, max_length, max_distance, gap, params, &matches[cur_match_pos]); cur_match_end = cur_match_pos + num_found_matches; for (j = cur_match_pos; j + 1 < cur_match_end; ++j) { diff --git a/c/enc/backward_references_inc.h b/c/enc/backward_references_inc.h index e7b1665..887038f 100644 --- a/c/enc/backward_references_inc.h +++ b/c/enc/backward_references_inc.h @@ -5,11 +5,11 @@ See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* template parameters: FN */ +/* template parameters: EXPORT_FN, FN */ -static BROTLI_NOINLINE void FN(CreateBackwardReferences)( - const BrotliDictionary* dictionary, const uint16_t* dictionary_hash, - size_t num_bytes, size_t position, +static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)( + const BrotliDictionary* dictionary, + const uint16_t* dictionary_hash, size_t num_bytes, size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask, const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache, size_t* last_insert_len, Command* commands, size_t* num_commands, @@ -27,6 +27,7 @@ static BROTLI_NOINLINE void FN(CreateBackwardReferences)( const size_t random_heuristics_window_size = LiteralSpreeLengthForSparseSearch(params); size_t apply_random_heuristics = position + random_heuristics_window_size; + const size_t gap = 0; /* Minimum score to accept a backward reference. */ const score_t kMinScore = BROTLI_SCORE_BASE + 100; @@ -43,7 +44,7 @@ static BROTLI_NOINLINE void FN(CreateBackwardReferences)( sr.score = kMinScore; FN(FindLongestMatch)(hasher, dictionary, dictionary_hash, ringbuffer, ringbuffer_mask, dist_cache, position, - max_length, max_distance, &sr); + max_length, max_distance, gap, &sr); if (sr.score > kMinScore) { /* Found a match. Let's look for something even better ahead. */ int delayed_backward_references_in_row = 0; @@ -59,7 +60,7 @@ static BROTLI_NOINLINE void FN(CreateBackwardReferences)( max_distance = BROTLI_MIN(size_t, position + 1, max_backward_limit); FN(FindLongestMatch)(hasher, dictionary, dictionary_hash, ringbuffer, ringbuffer_mask, dist_cache, position + 1, - max_length, max_distance, &sr2); + max_length, max_distance, gap, &sr2); if (sr2.score >= sr.score + cost_diff_lazy) { /* Ok, let's just write one byte for now and start a match from the next byte. */ @@ -80,8 +81,8 @@ static BROTLI_NOINLINE void FN(CreateBackwardReferences)( /* The first 16 codes are special short-codes, and the minimum offset is 1. */ size_t distance_code = - ComputeDistanceCode(sr.distance, max_distance, dist_cache); - if (sr.distance <= max_distance && distance_code > 0) { + ComputeDistanceCode(sr.distance, max_distance + gap, dist_cache); + if ((sr.distance <= (max_distance + gap)) && distance_code > 0) { dist_cache[3] = dist_cache[2]; dist_cache[2] = dist_cache[1]; dist_cache[1] = dist_cache[0]; diff --git a/c/enc/compress_fragment.c b/c/enc/compress_fragment.c index 96b590f..b4ca810 100644 --- a/c/enc/compress_fragment.c +++ b/c/enc/compress_fragment.c @@ -42,7 +42,7 @@ extern "C" { static const uint32_t kHashMul32 = 0x1e35a7bd; static BROTLI_INLINE uint32_t Hash(const uint8_t* p, size_t shift) { - const uint64_t h = (BROTLI_UNALIGNED_LOAD64(p) << 24) * kHashMul32; + const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(p) << 24) * kHashMul32; return (uint32_t)(h >> shift); } @@ -603,7 +603,7 @@ trawl: compression we first update "table" with the hashes of some positions within the last copy. */ { - uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 3); + uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 3); uint32_t prev_hash = HashBytesAtOffset(input_bytes, 0, shift); uint32_t cur_hash = HashBytesAtOffset(input_bytes, 3, shift); table[prev_hash] = (int)(ip - base_ip - 3); @@ -640,7 +640,7 @@ trawl: compression we first update "table" with the hashes of some positions within the last copy. */ { - uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 3); + uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 3); uint32_t prev_hash = HashBytesAtOffset(input_bytes, 0, shift); uint32_t cur_hash = HashBytesAtOffset(input_bytes, 3, shift); table[prev_hash] = (int)(ip - base_ip - 3); diff --git a/c/enc/compress_fragment_two_pass.c b/c/enc/compress_fragment_two_pass.c index cc549ed..e6611a0 100644 --- a/c/enc/compress_fragment_two_pass.c +++ b/c/enc/compress_fragment_two_pass.c @@ -41,7 +41,7 @@ extern "C" { static const uint32_t kHashMul32 = 0x1e35a7bd; static BROTLI_INLINE uint32_t Hash(const uint8_t* p, size_t shift) { - const uint64_t h = (BROTLI_UNALIGNED_LOAD64(p) << 16) * kHashMul32; + const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(p) << 16) * kHashMul32; return (uint32_t)(h >> shift); } @@ -346,7 +346,7 @@ trawl: /* We could immediately start working at ip now, but to improve compression we first update "table" with the hashes of some positions within the last copy. */ - uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 5); + uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 5); uint32_t prev_hash = HashBytesAtOffset(input_bytes, 0, shift); uint32_t cur_hash; table[prev_hash] = (int)(ip - base_ip - 5); @@ -354,7 +354,7 @@ trawl: table[prev_hash] = (int)(ip - base_ip - 4); prev_hash = HashBytesAtOffset(input_bytes, 2, shift); table[prev_hash] = (int)(ip - base_ip - 3); - input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 2); + input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 2); cur_hash = HashBytesAtOffset(input_bytes, 2, shift); prev_hash = HashBytesAtOffset(input_bytes, 0, shift); table[prev_hash] = (int)(ip - base_ip - 2); @@ -386,7 +386,7 @@ trawl: /* We could immediately start working at ip now, but to improve compression we first update "table" with the hashes of some positions within the last copy. */ - uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 5); + uint64_t input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 5); uint32_t prev_hash = HashBytesAtOffset(input_bytes, 0, shift); uint32_t cur_hash; table[prev_hash] = (int)(ip - base_ip - 5); @@ -394,7 +394,7 @@ trawl: table[prev_hash] = (int)(ip - base_ip - 4); prev_hash = HashBytesAtOffset(input_bytes, 2, shift); table[prev_hash] = (int)(ip - base_ip - 3); - input_bytes = BROTLI_UNALIGNED_LOAD64(ip - 2); + input_bytes = BROTLI_UNALIGNED_LOAD64LE(ip - 2); cur_hash = HashBytesAtOffset(input_bytes, 2, shift); prev_hash = HashBytesAtOffset(input_bytes, 0, shift); table[prev_hash] = (int)(ip - base_ip - 2); diff --git a/c/enc/encode.c b/c/enc/encode.c index d111573..7d58aa2 100644 --- a/c/enc/encode.c +++ b/c/enc/encode.c @@ -832,36 +832,6 @@ static void CopyInputToRingBuffer(BrotliEncoderState* s, } } -void BrotliEncoderSetCustomDictionary(BrotliEncoderState* s, size_t size, - const uint8_t* dict) { - size_t max_dict_size = BROTLI_MAX_BACKWARD_LIMIT(s->params.lgwin); - size_t dict_size = size; - MemoryManager* m = &s->memory_manager_; - - if (!EnsureInitialized(s)) return; - - if (dict_size == 0 || - s->params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY || - s->params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY) { - return; - } - if (size > max_dict_size) { - dict += size - max_dict_size; - dict_size = max_dict_size; - } - CopyInputToRingBuffer(s, dict_size, dict); - s->last_flush_pos_ = dict_size; - s->last_processed_pos_ = dict_size; - if (dict_size > 0) { - s->prev_byte_ = dict[dict_size - 1]; - } - if (dict_size > 1) { - s->prev_byte2_ = dict[dict_size - 2]; - } - HasherPrependCustomDictionary(m, &s->hasher_, &s->params, dict_size, dict); - if (BROTLI_IS_OOM(m)) return; -} - /* Marks all input as processed. Returns true if position wrapping occurs. */ static BROTLI_BOOL UpdateLastProcessedPos(BrotliEncoderState* s) { diff --git a/c/enc/find_match_length.h b/c/enc/find_match_length.h index b3e3d80..4184531 100644 --- a/c/enc/find_match_length.h +++ b/c/enc/find_match_length.h @@ -17,7 +17,7 @@ extern "C" { #endif /* Separate implementation for little-endian 64-bit targets, for speed. */ -#if defined(__GNUC__) && defined(_LP64) && defined(IS_LITTLE_ENDIAN) +#if defined(__GNUC__) && defined(_LP64) && defined(BROTLI_LITTLE_ENDIAN) static BROTLI_INLINE size_t FindMatchLengthWithLimit(const uint8_t* s1, const uint8_t* s2, @@ -25,13 +25,13 @@ static BROTLI_INLINE size_t FindMatchLengthWithLimit(const uint8_t* s1, size_t matched = 0; size_t limit2 = (limit >> 3) + 1; /* + 1 is for pre-decrement in while */ while (BROTLI_PREDICT_TRUE(--limit2)) { - if (BROTLI_PREDICT_FALSE(BROTLI_UNALIGNED_LOAD64(s2) == - BROTLI_UNALIGNED_LOAD64(s1 + matched))) { + if (BROTLI_PREDICT_FALSE(BROTLI_UNALIGNED_LOAD64LE(s2) == + BROTLI_UNALIGNED_LOAD64LE(s1 + matched))) { s2 += 8; matched += 8; } else { - uint64_t x = - BROTLI_UNALIGNED_LOAD64(s2) ^ BROTLI_UNALIGNED_LOAD64(s1 + matched); + uint64_t x = BROTLI_UNALIGNED_LOAD64LE(s2) ^ + BROTLI_UNALIGNED_LOAD64LE(s1 + matched); size_t matching_bits = (size_t)__builtin_ctzll(x); matched += matching_bits >> 3; return matched; diff --git a/c/enc/hash.h b/c/enc/hash.h index f97d969..c94edd3 100644 --- a/c/enc/hash.h +++ b/c/enc/hash.h @@ -173,6 +173,9 @@ static BROTLI_INLINE BROTLI_BOOL TestStaticDictionaryItem( backward = max_backward + dist + 1 + (transform_id << dictionary->size_bits_by_length[len]); } + if (backward >= BROTLI_MAX_DISTANCE) { + return BROTLI_FALSE; + } score = BackwardReferenceScore(matchlen, backward); if (score < out->score) { return BROTLI_FALSE; @@ -417,30 +420,6 @@ static BROTLI_INLINE void HasherSetup(MemoryManager* m, HasherHandle* handle, } } -/* Custom LZ77 window. */ -static BROTLI_INLINE void HasherPrependCustomDictionary( - MemoryManager* m, HasherHandle* handle, BrotliEncoderParams* params, - const size_t size, const uint8_t* dict) { - size_t overlap; - size_t i; - HasherHandle self; - HasherSetup(m, handle, params, dict, 0, size, BROTLI_FALSE); - if (BROTLI_IS_OOM(m)) return; - self = *handle; - switch (GetHasherCommon(self)->params.type) { -#define PREPEND_(N) \ - case N: \ - overlap = (StoreLookaheadH ## N()) - 1; \ - for (i = 0; i + overlap < size; i++) { \ - StoreH ## N(self, dict, ~(size_t)0, i); \ - } \ - break; - FOR_ALL_HASHERS(PREPEND_) -#undef PREPEND_ - default: break; - } -} - static BROTLI_INLINE void InitOrStitchToPreviousBlock( MemoryManager* m, HasherHandle* handle, const uint8_t* data, size_t mask, BrotliEncoderParams* params, size_t position, size_t input_size, diff --git a/c/enc/hash_forgetful_chain_inc.h b/c/enc/hash_forgetful_chain_inc.h index 88b0841..8f9ee73 100755 --- a/c/enc/hash_forgetful_chain_inc.h +++ b/c/enc/hash_forgetful_chain_inc.h @@ -158,7 +158,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle, const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix, const size_t max_length, const size_t max_backward, - HasherSearchResult* BROTLI_RESTRICT out) { + const size_t gap, HasherSearchResult* BROTLI_RESTRICT out) { HashForgetfulChain* self = FN(Self)(handle); const size_t cur_ix_masked = cur_ix & ring_buffer_mask; /* Don't accept a short copy from far away. */ @@ -241,7 +241,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle, } if (out->score == min_score) { SearchInStaticDictionary(dictionary, dictionary_hash, - handle, &data[cur_ix_masked], max_length, max_backward, out, + handle, &data[cur_ix_masked], max_length, max_backward + gap, out, BROTLI_FALSE); } } diff --git a/c/enc/hash_longest_match64_inc.h b/c/enc/hash_longest_match64_inc.h index 1581770..6b0697b 100755 --- a/c/enc/hash_longest_match64_inc.h +++ b/c/enc/hash_longest_match64_inc.h @@ -23,7 +23,7 @@ static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 8; } static BROTLI_INLINE uint32_t FN(HashBytes)(const uint8_t *data, const uint64_t mask, const int shift) { - const uint64_t h = (BROTLI_UNALIGNED_LOAD64(data) & mask) * kHashMul64Long; + const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(data) & mask) * kHashMul64Long; /* The higher bits contain more mixture from the multiplication, so we take our results from there. */ return (uint32_t)(h >> shift); @@ -161,7 +161,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle, const BrotliDictionary* dictionary, const uint16_t* dictionary_hash, const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix, - const size_t max_length, const size_t max_backward, + const size_t max_length, const size_t max_backward, const size_t gap, HasherSearchResult* BROTLI_RESTRICT out) { HasherCommon* common = GetHasherCommon(handle); HashLongestMatch* self = FN(Self)(handle); @@ -258,7 +258,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle, } if (min_score == out->score) { SearchInStaticDictionary(dictionary, dictionary_hash, - handle, &data[cur_ix_masked], max_length, max_backward, out, + handle, &data[cur_ix_masked], max_length, max_backward + gap, out, BROTLI_FALSE); } } diff --git a/c/enc/hash_longest_match_inc.h b/c/enc/hash_longest_match_inc.h index fe26206..dc5335f 100644 --- a/c/enc/hash_longest_match_inc.h +++ b/c/enc/hash_longest_match_inc.h @@ -154,7 +154,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle, const BrotliDictionary* dictionary, const uint16_t* dictionary_hash, const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix, - const size_t max_length, const size_t max_backward, + const size_t max_length, const size_t max_backward, const size_t gap, HasherSearchResult* BROTLI_RESTRICT out) { HasherCommon* common = GetHasherCommon(handle); HashLongestMatch* self = FN(Self)(handle); @@ -250,7 +250,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle, } if (min_score == out->score) { SearchInStaticDictionary(dictionary, dictionary_hash, - handle, &data[cur_ix_masked], max_length, max_backward, out, + handle, &data[cur_ix_masked], max_length, max_backward + gap, out, BROTLI_FALSE); } } diff --git a/c/enc/hash_longest_match_quickly_inc.h b/c/enc/hash_longest_match_quickly_inc.h index b8ddc31..2c78351 100644 --- a/c/enc/hash_longest_match_quickly_inc.h +++ b/c/enc/hash_longest_match_quickly_inc.h @@ -22,7 +22,7 @@ static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return 8; } the address in. The HashLongestMatch and HashLongestMatchQuickly classes have separate, different implementations of hashing. */ static uint32_t FN(HashBytes)(const uint8_t* data) { - const uint64_t h = ((BROTLI_UNALIGNED_LOAD64(data) << (64 - 8 * HASH_LEN)) * + const uint64_t h = ((BROTLI_UNALIGNED_LOAD64LE(data) << (64 - 8 * HASH_LEN)) * kHashMul64); /* The higher bits contain more mixture from the multiplication, so we take our results from there. */ @@ -129,7 +129,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)( const uint16_t* dictionary_hash, const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix, const size_t max_length, const size_t max_backward, - HasherSearchResult* BROTLI_RESTRICT out) { + const size_t gap, HasherSearchResult* BROTLI_RESTRICT out) { HashLongestMatchQuickly* self = FN(Self)(handle); const size_t best_len_in = out->len; const size_t cur_ix_masked = cur_ix & ring_buffer_mask; @@ -222,7 +222,7 @@ static BROTLI_INLINE void FN(FindLongestMatch)( } if (USE_DICTIONARY && min_score == out->score) { SearchInStaticDictionary(dictionary, dictionary_hash, - handle, &data[cur_ix_masked], max_length, max_backward, out, + handle, &data[cur_ix_masked], max_length, max_backward + gap, out, BROTLI_TRUE); } self->buckets_[key + ((cur_ix >> 3) % BUCKET_SWEEP)] = (uint32_t)cur_ix; diff --git a/c/enc/hash_to_binary_tree_inc.h b/c/enc/hash_to_binary_tree_inc.h index 0b2554c..1530c1b 100755 --- a/c/enc/hash_to_binary_tree_inc.h +++ b/c/enc/hash_to_binary_tree_inc.h @@ -19,8 +19,8 @@ #define BUCKET_SIZE (1 << BUCKET_BITS) -static size_t FN(HashTypeLength)(void) { return 4; } -static size_t FN(StoreLookahead)(void) { return MAX_TREE_COMP_LENGTH; } +static BROTLI_INLINE size_t FN(HashTypeLength)(void) { return 4; } +static BROTLI_INLINE size_t FN(StoreLookahead)(void) { return MAX_TREE_COMP_LENGTH; } static uint32_t FN(HashBytes)(const uint8_t *data) { uint32_t h = BROTLI_UNALIGNED_LOAD32(data) * kHashMul32; @@ -199,7 +199,7 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)( static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle, const BrotliDictionary* dictionary, const uint8_t* data, const size_t ring_buffer_mask, const size_t cur_ix, - const size_t max_length, const size_t max_backward, + const size_t max_length, const size_t max_backward, const size_t gap, const BrotliEncoderParams* params, BackwardMatch* matches) { BackwardMatch* const orig_matches = matches; const size_t cur_ix_masked = cur_ix & ring_buffer_mask; @@ -248,8 +248,10 @@ static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle, for (l = minlen; l <= maxlen; ++l) { uint32_t dict_id = dict_matches[l]; if (dict_id < kInvalidMatch) { - InitDictionaryBackwardMatch(matches++, - max_backward + (dict_id >> 5) + 1, l, dict_id & 31); + size_t distance = max_backward + gap + (dict_id >> 5) + 1; + if (distance < BROTLI_MAX_DISTANCE) { + InitDictionaryBackwardMatch(matches++, distance, l, dict_id & 31); + } } } } diff --git a/c/enc/port.h b/c/enc/port.h index 0d5f24c..0e5f0e7 100644 --- a/c/enc/port.h +++ b/c/enc/port.h @@ -26,36 +26,37 @@ #define __LITTLE_ENDIAN LITTLE_ENDIAN #endif -/* define the macro IS_LITTLE_ENDIAN +/* define the macro BROTLI_LITTLE_ENDIAN using the above endian definitions from endian.h if endian.h was included */ #ifdef __BYTE_ORDER #if __BYTE_ORDER == __LITTLE_ENDIAN -#define IS_LITTLE_ENDIAN +#define BROTLI_LITTLE_ENDIAN #endif #else #if defined(__LITTLE_ENDIAN__) -#define IS_LITTLE_ENDIAN +#define BROTLI_LITTLE_ENDIAN #endif #endif /* __BYTE_ORDER */ #if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) -#define IS_LITTLE_ENDIAN +#define BROTLI_LITTLE_ENDIAN #endif /* Enable little-endian optimization for x64 architecture on Windows. */ #if (defined(_WIN32) || defined(_WIN64)) && defined(_M_X64) -#define IS_LITTLE_ENDIAN +#define BROTLI_LITTLE_ENDIAN #endif /* Portable handling of unaligned loads, stores, and copies. On some platforms, like ARM, the copy functions can be more efficient then a load and a store. */ -#if defined(ARCH_PIII) || \ - defined(ARCH_ATHLON) || defined(ARCH_K8) || defined(_ARCH_PPC) +#if defined(BROTLI_LITTLE_ENDIAN) && (\ + defined(ARCH_PIII) || defined(ARCH_ATHLON) || \ + defined(ARCH_K8) || defined(_ARCH_PPC)) /* x86 and x86-64 can perform unaligned loads/stores directly; modern PowerPC hardware can also do unaligned integer loads and stores; @@ -63,14 +64,12 @@ */ #define BROTLI_UNALIGNED_LOAD32(_p) (*(const uint32_t *)(_p)) -#define BROTLI_UNALIGNED_LOAD64(_p) (*(const uint64_t *)(_p)) +#define BROTLI_UNALIGNED_LOAD64LE(_p) (*(const uint64_t *)(_p)) -#define BROTLI_UNALIGNED_STORE32(_p, _val) \ - (*(uint32_t *)(_p) = (_val)) -#define BROTLI_UNALIGNED_STORE64(_p, _val) \ +#define BROTLI_UNALIGNED_STORE64LE(_p, _val) \ (*(uint64_t *)(_p) = (_val)) -#elif defined(__arm__) && \ +#elif defined(BROTLI_LITTLE_ENDIAN) && defined(__arm__) && \ !defined(__ARM_ARCH_5__) && \ !defined(__ARM_ARCH_5T__) && \ !defined(__ARM_ARCH_5TE__) && \ @@ -88,16 +87,14 @@ slowly (trip through kernel mode). */ #define BROTLI_UNALIGNED_LOAD32(_p) (*(const uint32_t *)(_p)) -#define BROTLI_UNALIGNED_STORE32(_p, _val) \ - (*(uint32_t *)(_p) = (_val)) -static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64(const void *p) { +static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void *p) { uint64_t t; memcpy(&t, p, sizeof t); return t; } -static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64(void *p, uint64_t v) { +static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void *p, uint64_t v) { memcpy(p, &v, sizeof v); } @@ -112,20 +109,47 @@ static BROTLI_INLINE uint32_t BROTLI_UNALIGNED_LOAD32(const void *p) { return t; } -static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64(const void *p) { +#if defined(BROTLI_LITTLE_ENDIAN) + +static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void *p) { uint64_t t; memcpy(&t, p, sizeof t); return t; } -static BROTLI_INLINE void BROTLI_UNALIGNED_STORE32(void *p, uint32_t v) { +static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void *p, uint64_t v) { memcpy(p, &v, sizeof v); } -static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64(void *p, uint64_t v) { - memcpy(p, &v, sizeof v); +#else /* BROTLI_LITTLE_ENDIAN */ + +static BROTLI_INLINE uint64_t BROTLI_UNALIGNED_LOAD64LE(const void *p) { + const uint8_t* in = (const uint8_t*)p; + uint64_t value = (uint64_t)(in[0]); + value |= (uint64_t)(in[1]) << 8; + value |= (uint64_t)(in[2]) << 16; + value |= (uint64_t)(in[3]) << 24; + value |= (uint64_t)(in[4]) << 32; + value |= (uint64_t)(in[5]) << 40; + value |= (uint64_t)(in[6]) << 48; + value |= (uint64_t)(in[7]) << 56; + return value; } +static BROTLI_INLINE void BROTLI_UNALIGNED_STORE64LE(void *p, uint64_t v) { + uint8_t* out = (uint8_t*)p; + out[0] = (uint8_t)v; + out[1] = (uint8_t)(v >> 8); + out[2] = (uint8_t)(v >> 16); + out[3] = (uint8_t)(v >> 24); + out[4] = (uint8_t)(v >> 32); + out[5] = (uint8_t)(v >> 40); + out[6] = (uint8_t)(v >> 48); + out[7] = (uint8_t)(v >> 56); +} + +#endif /* BROTLI_LITTLE_ENDIAN */ + #endif #define TEMPLATE_(T) \ diff --git a/c/enc/write_bits.h b/c/enc/write_bits.h index 3999c91..83fdddc 100644 --- a/c/enc/write_bits.h +++ b/c/enc/write_bits.h @@ -40,7 +40,7 @@ static BROTLI_INLINE void BrotliWriteBits(size_t n_bits, uint64_t bits, size_t * BROTLI_RESTRICT pos, uint8_t * BROTLI_RESTRICT array) { -#ifdef IS_LITTLE_ENDIAN +#ifdef BROTLI_LITTLE_ENDIAN /* This branch of the code can write up to 56 bits at a time, 7 bits are lost by being perhaps already in *p and at least 1 bit is needed to initialize the bit-stream ahead (i.e. if 7 @@ -54,7 +54,7 @@ static BROTLI_INLINE void BrotliWriteBits(size_t n_bits, assert((bits >> n_bits) == 0); assert(n_bits <= 56); v |= bits << (*pos & 7); - BROTLI_UNALIGNED_STORE64(p, v); /* Set some bits. */ + BROTLI_UNALIGNED_STORE64LE(p, v); /* Set some bits. */ *pos += n_bits; #else /* implicit & 0xff is assumed for uint8_t arithmetics */ diff --git a/c/include/brotli/decode.h b/c/include/brotli/decode.h index 356edd1..d1d21c4 100755 --- a/c/include/brotli/decode.h +++ b/c/include/brotli/decode.h @@ -84,8 +84,9 @@ typedef enum { BROTLI_ERROR_CODE(_ERROR_FORMAT_, PADDING_1, -14) SEPARATOR \ BROTLI_ERROR_CODE(_ERROR_FORMAT_, PADDING_2, -15) SEPARATOR \ \ - /* -16..-18 codes are reserved */ \ + /* -16..-17 codes are reserved */ \ \ + BROTLI_ERROR_CODE(_ERROR_, COMPOUND_DICTIONARY, -18) SEPARATOR \ BROTLI_ERROR_CODE(_ERROR_, DICTIONARY_NOT_SET, -19) SEPARATOR \ BROTLI_ERROR_CODE(_ERROR_, INVALID_ARGUMENTS, -20) SEPARATOR \ \ @@ -241,31 +242,6 @@ BROTLI_DEC_API BrotliDecoderResult BrotliDecoderDecompressStream( BrotliDecoderState* state, size_t* available_in, const uint8_t** next_in, size_t* available_out, uint8_t** next_out, size_t* total_out); -/** - * Prepends LZ77 dictionary. - * - * Fills the fresh ::BrotliDecoderState with additional data corpus for LZ77 - * backward references. - * - * @note Not to be confused with the static dictionary (see RFC7932 section 8). - * @warning The dictionary must exist in memory until decoding is done and - * is owned by the caller. - * - * Workflow: - * -# Allocate and initialize state with ::BrotliDecoderCreateInstance - * -# Invoke ::BrotliDecoderSetCustomDictionary - * -# Use ::BrotliDecoderDecompressStream - * -# Clean up and free state with ::BrotliDecoderDestroyInstance - * - * @param state decoder instance - * @param size length of @p dict; should be less or equal to 2^24 (16MiB), - * otherwise the dictionary will be ignored - * @param dict "dictionary"; @b MUST be the same as used during compression - */ -BROTLI_DEC_API void BrotliDecoderSetCustomDictionary( - BrotliDecoderState* state, size_t size, - const uint8_t dict[BROTLI_ARRAY_PARAM(size)]); - /** * Checks if decoder has more output. * @@ -327,7 +303,8 @@ BROTLI_DEC_API BROTLI_BOOL BrotliDecoderIsUsed(const BrotliDecoderState* state); * the input and produced all of the output * @returns ::BROTLI_FALSE otherwise */ -BROTLI_DEC_API BROTLI_BOOL BrotliDecoderIsFinished(const BrotliDecoderState* state); +BROTLI_DEC_API BROTLI_BOOL BrotliDecoderIsFinished( + const BrotliDecoderState* state); /** * Acquires a detailed error code. diff --git a/c/include/brotli/encode.h b/c/include/brotli/encode.h index 98a5ff8..4bb76e1 100755 --- a/c/include/brotli/encode.h +++ b/c/include/brotli/encode.h @@ -223,29 +223,6 @@ BROTLI_ENC_API BrotliEncoderState* BrotliEncoderCreateInstance( */ BROTLI_ENC_API void BrotliEncoderDestroyInstance(BrotliEncoderState* state); -/** - * Prepends imaginary LZ77 dictionary. - * - * Fills the fresh ::BrotliEncoderState with additional data corpus for LZ77 - * backward references. - * - * @note Not to be confused with the static dictionary (see RFC7932 section 8). - * - * Workflow: - * -# Allocate and initialize state with ::BrotliEncoderCreateInstance - * -# Set ::BROTLI_PARAM_LGWIN parameter - * -# Invoke ::BrotliEncoderSetCustomDictionary - * -# Use ::BrotliEncoderCompressStream - * -# Clean up and free state with ::BrotliEncoderDestroyInstance - * - * @param state encoder instance - * @param size length of @p dict; at most "window size" bytes are used - * @param dict "dictionary"; @b MUST use same dictionary during decompression - */ -BROTLI_ENC_API void BrotliEncoderSetCustomDictionary( - BrotliEncoderState* state, size_t size, - const uint8_t dict[BROTLI_ARRAY_PARAM(size)]); - /** * Calculates the output size bound for the given @p input_size. * diff --git a/c/tools/brotli.c b/c/tools/brotli.c index 8b6f9e2..fe20e0d 100755 --- a/c/tools/brotli.c +++ b/c/tools/brotli.c @@ -91,7 +91,6 @@ typedef struct { BROTLI_BOOL test_integrity; BROTLI_BOOL decompress; const char* output_path; - const char* dictionary_path; const char* suffix; int not_input_indices[MAX_OPTIONS]; size_t longest_path_len; @@ -100,8 +99,6 @@ typedef struct { /* Inner state */ int argc; char** argv; - uint8_t* dictionary; - size_t dictionary_size; char* modified_path; /* Storage for path with appended / cut suffix */ int iterator; int ignore; @@ -285,9 +282,6 @@ static Command ParseParams(Context* params) { if (params->lgwin != 0 && params->lgwin < BROTLI_MIN_WINDOW_BITS) { return COMMAND_INVALID; } - } else if (c == 'D') { - if (params->dictionary_path) return COMMAND_INVALID; - params->dictionary_path = argv[i]; } else if (c == 'S') { if (suffix_set) return COMMAND_INVALID; suffix_set = BROTLI_TRUE; @@ -342,10 +336,7 @@ static Command ParseParams(Context* params) { if (!value || value[1] == 0) return COMMAND_INVALID; key_len = (size_t)(value - arg); value++; - if (strncmp("dictionary", arg, key_len) == 0) { - if (params->dictionary_path) return COMMAND_INVALID; - params->dictionary_path = value; - } else if (strncmp("lgwin", arg, key_len) == 0) { + if (strncmp("lgwin", arg, key_len) == 0) { if (lgwin_set) return COMMAND_INVALID; lgwin_set = ParseInt(value, 0, BROTLI_MAX_WINDOW_BITS, ¶ms->lgwin); @@ -427,7 +418,7 @@ Options:\n\ fprintf(stdout, "\ window size = 2**NUM - 16\n\ 0 lets compressor decide over the optimal value\n\ - -D FILE, --dictionary=FILE use FILE as LZ77 dictionary\n"); +"); fprintf(stdout, "\ -S SUF, --suffix=SUF output file suffix (default:'%s')\n", DEFAULT_SUFFIX); @@ -482,23 +473,6 @@ static BROTLI_BOOL OpenOutputFile(const char* output_path, FILE** f, return BROTLI_TRUE; } -static int64_t FileSize(const char* path) { - FILE* f = fopen(path, "rb"); - int64_t retval; - if (f == NULL) { - return -1; - } - if (fseek(f, 0L, SEEK_END) != 0) { - fclose(f); - return -1; - } - retval = ftell(f); - if (fclose(f) != 0) { - return -1; - } - return retval; -} - /* Copy file times and permissions. TODO(eustas): this is a "best effort" implementation; honest cross-platform fully featured implementation is way too hacky; add more hacks by request. */ @@ -532,58 +506,6 @@ static void CopyStat(const char* input_path, const char* output_path) { } } -/* Result ownership is passed to caller. - |*dictionary_size| is set to resulting buffer size. */ -static BROTLI_BOOL ReadDictionary(Context* context) { - static const int kMaxDictionarySize = (1 << 24) - 16; - FILE* f; - int64_t file_size_64; - uint8_t* buffer; - size_t bytes_read; - - if (context->dictionary_path == NULL) return BROTLI_TRUE; - f = fopen(context->dictionary_path, "rb"); - if (f == NULL) { - fprintf(stderr, "failed to open dictionary file [%s]: %s\n", - PrintablePath(context->dictionary_path), strerror(errno)); - return BROTLI_FALSE; - } - - file_size_64 = FileSize(context->dictionary_path); - if (file_size_64 == -1) { - fprintf(stderr, "could not get size of dictionary file [%s]", - PrintablePath(context->dictionary_path)); - fclose(f); - return BROTLI_FALSE; - } - - if (file_size_64 > kMaxDictionarySize) { - fprintf(stderr, "dictionary [%s] is larger than maximum allowed: %d\n", - PrintablePath(context->dictionary_path), kMaxDictionarySize); - fclose(f); - return BROTLI_FALSE; - } - context->dictionary_size = (size_t)file_size_64; - - buffer = (uint8_t*)malloc(context->dictionary_size); - if (!buffer) { - fprintf(stderr, "could not read dictionary: out of memory\n"); - fclose(f); - return BROTLI_FALSE; - } - bytes_read = fread(buffer, sizeof(uint8_t), context->dictionary_size, f); - if (bytes_read != context->dictionary_size) { - free(buffer); - fprintf(stderr, "failed to read dictionary [%s]: %s\n", - PrintablePath(context->dictionary_path), strerror(errno)); - fclose(f); - return BROTLI_FALSE; - } - fclose(f); - context->dictionary = buffer; - return BROTLI_TRUE; -} - static BROTLI_BOOL NextFile(Context* context) { const char* arg; size_t arg_len; @@ -764,10 +686,6 @@ static BROTLI_BOOL DecompressFiles(Context* context) { fprintf(stderr, "out of memory\n"); return BROTLI_FALSE; } - if (context->dictionary) { - BrotliDecoderSetCustomDictionary(s, - context->dictionary_size, context->dictionary); - } is_ok = OpenFiles(context); if (is_ok) is_ok = DecompressFile(context, s); BrotliDecoderDestroyInstance(s); @@ -833,10 +751,6 @@ static BROTLI_BOOL CompressFiles(Context* context) { BROTLI_PARAM_QUALITY, (uint32_t)context->quality); BrotliEncoderSetParameter(s, BROTLI_PARAM_LGWIN, (uint32_t)context->lgwin); - if (context->dictionary) { - BrotliEncoderSetCustomDictionary(s, - context->dictionary_size, context->dictionary); - } is_ok = OpenFiles(context); if (is_ok) is_ok = CompressFile(context, s); BrotliEncoderDestroyInstance(s); @@ -862,7 +776,6 @@ int main(int argc, char** argv) { context.write_to_stdout = BROTLI_FALSE; context.decompress = BROTLI_FALSE; context.output_path = NULL; - context.dictionary_path = NULL; context.suffix = DEFAULT_SUFFIX; for (i = 0; i < MAX_OPTIONS; ++i) context.not_input_indices[i] = 0; context.longest_path_len = 1; @@ -870,8 +783,6 @@ int main(int argc, char** argv) { context.argc = argc; context.argv = argv; - context.dictionary = NULL; - context.dictionary_size = 0; context.modified_path = NULL; context.iterator = 0; context.ignore = 0; @@ -886,7 +797,6 @@ int main(int argc, char** argv) { if (command == COMMAND_COMPRESS || command == COMMAND_DECOMPRESS || command == COMMAND_TEST_INTEGRITY) { - if (!ReadDictionary(&context)) is_ok = BROTLI_FALSE; if (is_ok) { size_t modified_path_len = context.longest_path_len + strlen(context.suffix) + 1; @@ -931,7 +841,6 @@ int main(int argc, char** argv) { if (context.iterator_error) is_ok = BROTLI_FALSE; - free(context.dictionary); free(context.modified_path); free(context.buffer); diff --git a/c/tools/brotli.md b/c/tools/brotli.md index fc6d9bf..c029869 100755 --- a/c/tools/brotli.md +++ b/c/tools/brotli.md @@ -84,9 +84,6 @@ OPTIONS `(2**NUM - 16)`; 0 lets compressor decide over the optimal value; bigger windows size improve density; decoder might require up to window size memory to operate -* `-D FILE`, `--dictionary=FILE`: - use FILE as LZ77 dictionary; same dictionary MUST be used both for - compression and decompression * `-S SUF`, `--suffix=SUF`: output file suffix (default: `.br`) * `-V`, `--version`: diff --git a/docs/brotli.1 b/docs/brotli.1 index b755e67..c55b906 100755 --- a/docs/brotli.1 +++ b/docs/brotli.1 @@ -1,4 +1,4 @@ -.TH "BROTLI" "1" "May 2017" "brotli 1.0.0" "User commands" +.TH "BROTLI" "1" "August 2017" "brotli 1.0.0" "User commands" .SH "NAME" \fBbrotli\fR \- brotli, unbrotli \- compress or decompress files .SH SYNOPSIS @@ -108,10 +108,6 @@ Conflicting or duplicate \fIoptions\fR are not allowed\. windows size improve density; decoder might require up to window size memory to operate .IP \(bu 2 -\fB\-D FILE\fP, \fB\-\-dictionary=FILE\fP: - use FILE as LZ77 dictionary; same dictionary MUST be used both for - compression and decompression -.IP \(bu 2 \fB\-S SUF\fP, \fB\-\-suffix=SUF\fP: output file suffix (default: \fB\|\.br\fP) .IP \(bu 2 diff --git a/docs/decode.h.3 b/docs/decode.h.3 index 447c67b..965d07f 100755 --- a/docs/decode.h.3 +++ b/docs/decode.h.3 @@ -1,4 +1,4 @@ -.TH "decode.h" 3 "Tue Jun 13 2017" "Brotli" \" -*- nroff -*- +.TH "decode.h" 3 "Wed Aug 2 2017" "Brotli" \" -*- nroff -*- .ad l .nh .SH NAME @@ -72,10 +72,6 @@ decode.h \- API for Brotli decompression\&. .br .RI "\fIChecks if instance has already consumed input\&. \fP" .ti -1c -.RI "void \fBBrotliDecoderSetCustomDictionary\fP (\fBBrotliDecoderState\fP *state, size_t size, const uint8_t dict[size])" -.br -.RI "\fIPrepends LZ77 dictionary\&. \fP" -.ti -1c .RI "\fBBROTLI_BOOL\fP \fBBrotliDecoderSetParameter\fP (\fBBrotliDecoderState\fP *state, \fBBrotliDecoderParameter\fP param, uint32_t value)" .br .RI "\fISets the specified parameter to the given decoder instance\&. \fP" @@ -346,42 +342,6 @@ Checks if instance has already consumed input\&. Instance that returns \fBBROTLI .RE .PP -.SS "void BrotliDecoderSetCustomDictionary (\fBBrotliDecoderState\fP * state, size_t size, const uint8_t dict[size])" - -.PP -Prepends LZ77 dictionary\&. Fills the fresh \fBBrotliDecoderState\fP with additional data corpus for LZ77 backward references\&. -.PP -\fBNote:\fP -.RS 4 -Not to be confused with the static dictionary (see RFC7932 section 8)\&. -.RE -.PP -\fBWarning:\fP -.RS 4 -The dictionary must exist in memory until decoding is done and is owned by the caller\&. -.RE -.PP -Workflow: -.IP "1." 4 -Allocate and initialize state with \fBBrotliDecoderCreateInstance\fP -.IP "2." 4 -Invoke \fBBrotliDecoderSetCustomDictionary\fP -.IP "3." 4 -Use \fBBrotliDecoderDecompressStream\fP -.IP "4." 4 -Clean up and free state with \fBBrotliDecoderDestroyInstance\fP -.PP -.PP -\fBParameters:\fP -.RS 4 -\fIstate\fP decoder instance -.br -\fIsize\fP length of \fCdict\fP; should be less or equal to 2^24 (16MiB), otherwise the dictionary will be ignored -.br -\fIdict\fP 'dictionary'; \fBMUST\fP be the same as used during compression -.RE -.PP - .SS "\fBBROTLI_BOOL\fP BrotliDecoderSetParameter (\fBBrotliDecoderState\fP * state, \fBBrotliDecoderParameter\fP param, uint32_t value)" .PP diff --git a/docs/encode.h.3 b/docs/encode.h.3 index 51570b7..01b4f4f 100755 --- a/docs/encode.h.3 +++ b/docs/encode.h.3 @@ -1,4 +1,4 @@ -.TH "encode.h" 3 "Tue Feb 28 2017" "Brotli" \" -*- nroff -*- +.TH "encode.h" 3 "Wed Aug 2 2017" "Brotli" \" -*- nroff -*- .ad l .nh .SH NAME @@ -100,10 +100,6 @@ encode.h \- API for Brotli compression\&. .br .RI "\fICalculates the output size bound for the given \fCinput_size\fP\&. \fP" .ti -1c -.RI "void \fBBrotliEncoderSetCustomDictionary\fP (\fBBrotliEncoderState\fP *state, size_t size, const uint8_t dict[size])" -.br -.RI "\fIPrepends imaginary LZ77 dictionary\&. \fP" -.ti -1c .RI "\fBBROTLI_BOOL\fP \fBBrotliEncoderSetParameter\fP (\fBBrotliEncoderState\fP *state, \fBBrotliEncoderParameter\fP param, uint32_t value)" .br .RI "\fISets the specified parameter to the given encoder instance\&. \fP" @@ -482,39 +478,6 @@ Result is not applicable to \fBBrotliEncoderCompressStream\fP output, because ev .RE .PP -.SS "void BrotliEncoderSetCustomDictionary (\fBBrotliEncoderState\fP * state, size_t size, const uint8_t dict[size])" - -.PP -Prepends imaginary LZ77 dictionary\&. Fills the fresh \fBBrotliEncoderState\fP with additional data corpus for LZ77 backward references\&. -.PP -\fBNote:\fP -.RS 4 -Not to be confused with the static dictionary (see RFC7932 section 8)\&. -.RE -.PP -Workflow: -.IP "1." 4 -Allocate and initialize state with \fBBrotliEncoderCreateInstance\fP -.IP "2." 4 -Set \fBBROTLI_PARAM_LGWIN\fP parameter -.IP "3." 4 -Invoke \fBBrotliEncoderSetCustomDictionary\fP -.IP "4." 4 -Use \fBBrotliEncoderCompressStream\fP -.IP "5." 4 -Clean up and free state with \fBBrotliEncoderDestroyInstance\fP -.PP -.PP -\fBParameters:\fP -.RS 4 -\fIstate\fP encoder instance -.br -\fIsize\fP length of \fCdict\fP; at most 'window size' bytes are used -.br -\fIdict\fP 'dictionary'; \fBMUST\fP use same dictionary during decompression -.RE -.PP - .SS "\fBBROTLI_BOOL\fP BrotliEncoderSetParameter (\fBBrotliEncoderState\fP * state, \fBBrotliEncoderParameter\fP param, uint32_t value)" .PP diff --git a/docs/types.h.3 b/docs/types.h.3 index 7dd8dc4..e72ae6e 100755 --- a/docs/types.h.3 +++ b/docs/types.h.3 @@ -1,4 +1,4 @@ -.TH "types.h" 3 "Tue Feb 28 2017" "Brotli" \" -*- nroff -*- +.TH "types.h" 3 "Wed Aug 2 2017" "Brotli" \" -*- nroff -*- .ad l .nh .SH NAME diff --git a/java/org/brotli/dec/BUILD b/java/org/brotli/dec/BUILD index cd5d8d1..8a2558c 100755 --- a/java/org/brotli/dec/BUILD +++ b/java/org/brotli/dec/BUILD @@ -42,12 +42,6 @@ java_test( runtime_deps = [":test_lib"], ) -java_test( - name = "EnumTest", - test_class = "org.brotli.dec.EnumTest", - runtime_deps = [":test_lib"], -) - java_test( name = "SynthTest", test_class = "org.brotli.dec.SynthTest", diff --git a/java/org/brotli/dec/BitReader.java b/java/org/brotli/dec/BitReader.java index a3bb53d..929c7f0 100755 --- a/java/org/brotli/dec/BitReader.java +++ b/java/org/brotli/dec/BitReader.java @@ -6,52 +6,34 @@ package org.brotli.dec; -import java.io.IOException; -import java.io.InputStream; - /** * Bit reading helpers. */ final class BitReader { - /** - * Input byte buffer, consist of a ring-buffer and a "slack" region where bytes from the start of - * the ring-buffer are copied. - */ - private static final int CAPACITY = 1024; - private static final int SLACK = 16; - private static final int INT_BUFFER_SIZE = CAPACITY + SLACK; - private static final int BYTE_READ_SIZE = CAPACITY << 2; - private static final int BYTE_BUFFER_SIZE = INT_BUFFER_SIZE << 2; + // Possible values: {5, 6}. 5 corresponds to 32-bit build, 6 to 64-bit. This value is used for + // conditional compilation -> produced artifacts might be binary INCOMPATIBLE (JLS 13.2). + private static final int LOG_BITNESS = 6; + private static final int BITNESS = 1 << LOG_BITNESS; - private final byte[] byteBuffer = new byte[BYTE_BUFFER_SIZE]; - private final int[] intBuffer = new int[INT_BUFFER_SIZE]; - private final IntReader intReader = new IntReader(); + private static final int BYTENESS = BITNESS / 8; + private static final int CAPACITY = 4096; + // After encountering the end of the input stream, this amount of zero bytes will be appended. + private static final int SLACK = 64; + private static final int BUFFER_SIZE = CAPACITY + SLACK; + // Don't bother to replenish the buffer while this number of bytes is available. + private static final int SAFEGUARD = 36; + private static final int WATERLINE = CAPACITY - SAFEGUARD; - private InputStream input; + // "Half" refers to "half of native integer type", i.e. on 64-bit machines it is 32-bit type, + // on 32-bit machines it is 16-bit. + private static final int HALF_BITNESS = BITNESS / 2; + private static final int HALF_SIZE = BYTENESS / 2; + private static final int HALVES_CAPACITY = CAPACITY / HALF_SIZE; + private static final int HALF_BUFFER_SIZE = BUFFER_SIZE / HALF_SIZE; + private static final int HALF_WATERLINE = WATERLINE / HALF_SIZE; - /** - * Input stream is finished. - */ - private boolean endOfStreamReached; - - /** - * Pre-fetched bits. - */ - long accumulator; - - /** - * Current bit-reading position in accumulator. - */ - int bitOffset; - - /** - * Offset of next item in intBuffer. - */ - private int intOffset; - - /* Number of bytes in unfinished "int" item. */ - private int tailBytes = 0; + private static final int LOG_HALF_SIZE = LOG_BITNESS - 4; /** * Fills up the input buffer. @@ -62,141 +44,148 @@ final class BitReader { * buffer. */ // TODO: Split to check and read; move read outside of decoding loop. - static void readMoreInput(BitReader br) { - if (br.intOffset <= CAPACITY - 9) { + static void readMoreInput(State s) { + if (s.halfOffset <= HALF_WATERLINE) { return; } - if (br.endOfStreamReached) { - if (intAvailable(br) >= -2) { + if (s.endOfStreamReached != 0) { + if (halfAvailable(s) >= -2) { return; } throw new BrotliRuntimeException("No more input"); } - int readOffset = br.intOffset << 2; - int bytesRead = BYTE_READ_SIZE - readOffset; - System.arraycopy(br.byteBuffer, readOffset, br.byteBuffer, 0, bytesRead); - br.intOffset = 0; - try { - while (bytesRead < BYTE_READ_SIZE) { - int len = br.input.read(br.byteBuffer, bytesRead, BYTE_READ_SIZE - bytesRead); - // EOF is -1 in Java, but 0 in C#. - if (len <= 0) { - br.endOfStreamReached = true; - br.tailBytes = bytesRead; - bytesRead += 3; - break; - } - bytesRead += len; + int readOffset = s.halfOffset << LOG_HALF_SIZE; + int bytesInBuffer = CAPACITY - readOffset; + // Move unused bytes to the head of the buffer. + Utils.copyBytesWithin(s.byteBuffer, 0, readOffset, CAPACITY); + s.halfOffset = 0; + while (bytesInBuffer < CAPACITY) { + int spaceLeft = CAPACITY - bytesInBuffer; + int len = Utils.readInput(s.input, s.byteBuffer, bytesInBuffer, spaceLeft); + // EOF is -1 in Java, but 0 in C#. + if (len <= 0) { + s.endOfStreamReached = 1; + s.tailBytes = bytesInBuffer; + bytesInBuffer += HALF_SIZE - 1; + break; } - } catch (IOException e) { - throw new BrotliRuntimeException("Failed to read input", e); + bytesInBuffer += len; } - IntReader.convert(br.intReader, bytesRead >> 2); + bytesToNibbles(s, bytesInBuffer); } - static void checkHealth(BitReader br, boolean endOfStream) { - if (!br.endOfStreamReached) { + static void checkHealth(State s, int endOfStream) { + if (s.endOfStreamReached == 0) { return; } - int byteOffset = (br.intOffset << 2) + ((br.bitOffset + 7) >> 3) - 8; - if (byteOffset > br.tailBytes) { + int byteOffset = (s.halfOffset << LOG_HALF_SIZE) + ((s.bitOffset + 7) >> 3) - BYTENESS; + if (byteOffset > s.tailBytes) { throw new BrotliRuntimeException("Read after end"); } - if (endOfStream && (byteOffset != br.tailBytes)) { + if ((endOfStream != 0) && (byteOffset != s.tailBytes)) { throw new BrotliRuntimeException("Unused bytes after end"); } } - /** - * Advances the Read buffer by 5 bytes to make room for reading next 24 bits. - */ - static void fillBitWindow(BitReader br) { - if (br.bitOffset >= 32) { - br.accumulator = ((long) br.intBuffer[br.intOffset++] << 32) | (br.accumulator >>> 32); - br.bitOffset -= 32; + static void fillBitWindow(State s) { + if (s.bitOffset >= HALF_BITNESS) { + if (BITNESS == 64) { + s.accumulator64 = ((long) s.intBuffer[s.halfOffset++] << HALF_BITNESS) + | (s.accumulator64 >>> HALF_BITNESS); + } else { + s.accumulator32 = ((int) s.shortBuffer[s.halfOffset++] << HALF_BITNESS) + | (s.accumulator32 >>> HALF_BITNESS); + } + s.bitOffset -= HALF_BITNESS; } } - /** - * Reads the specified number of bits from Read Buffer. - */ - static int readBits(BitReader br, int n) { - fillBitWindow(br); - int val = (int) (br.accumulator >>> br.bitOffset) & ((1 << n) - 1); - br.bitOffset += n; + static int peekBits(State s) { + if (BITNESS == 64) { + return (int) (s.accumulator64 >>> s.bitOffset); + } else { + return s.accumulator32 >>> s.bitOffset; + } + } + + static int readFewBits(State s, int n) { + fillBitWindow(s); + int val = peekBits(s) & ((1 << n) - 1); + s.bitOffset += n; return val; } - /** - * Initialize bit reader. - * - *
Initialisation turns bit reader to a ready state. Also a number of bytes is prefetched to - * accumulator. Because of that this method may block until enough data could be read from input. - * - * @param br BitReader POJO - * @param input data source - */ - static void init(BitReader br, InputStream input) { - if (br.input != null) { - throw new IllegalStateException("Bit reader already has associated input stream"); - } - IntReader.init(br.intReader, br.byteBuffer, br.intBuffer); - br.input = input; - br.accumulator = 0; - br.bitOffset = 64; - br.intOffset = CAPACITY; - br.endOfStreamReached = false; - prepare(br); - } - - private static void prepare(BitReader br) { - readMoreInput(br); - checkHealth(br, false); - fillBitWindow(br); - fillBitWindow(br); - } - - static void reload(BitReader br) { - if (br.bitOffset == 64) { - prepare(br); + static int readBits(State s, int n) { + if (HALF_BITNESS >= 24) { + return readFewBits(s, n); + } else { + if (n <= 16) { + return readFewBits(s, n); + } else if (s.bitOffset + n <= BITNESS) { + return readFewBits(s, n); + } else { + int lo = readFewBits(s, 16); + int hi = readFewBits(s, n - 16); + return lo | (hi << 16); + } } } - static void close(BitReader br) throws IOException { - InputStream is = br.input; - br.input = null; - if (is != null) { - is.close(); + static void initBitReader(State s) { + s.byteBuffer = new byte[BUFFER_SIZE]; + if (BITNESS == 64) { + s.accumulator64 = 0; + s.intBuffer = new int[HALF_BUFFER_SIZE]; + } else { + s.accumulator32 = 0; + s.shortBuffer = new short[HALF_BUFFER_SIZE]; + } + s.bitOffset = BITNESS; + s.halfOffset = HALVES_CAPACITY; + s.endOfStreamReached = 0; + prepare(s); + } + + static void prepare(State s) { + readMoreInput(s); + checkHealth(s, 0); + fillBitWindow(s); + fillBitWindow(s); + } + + static void reload(State s) { + if (s.bitOffset == BITNESS) { + prepare(s); } } - static void jumpToByteBoundary(BitReader br) { - int padding = (64 - br.bitOffset) & 7; + static void jumpToByteBoundary(State s) { + int padding = (BITNESS - s.bitOffset) & 7; if (padding != 0) { - int paddingBits = BitReader.readBits(br, padding); + int paddingBits = readBits(s, padding); if (paddingBits != 0) { throw new BrotliRuntimeException("Corrupted padding bits"); } } } - static int intAvailable(BitReader br) { - int limit = CAPACITY; - if (br.endOfStreamReached) { - limit = (br.tailBytes + 3) >> 2; + static int halfAvailable(State s) { + int limit = HALVES_CAPACITY; + if (s.endOfStreamReached != 0) { + limit = (s.tailBytes + (HALF_SIZE - 1)) >> LOG_HALF_SIZE; } - return limit - br.intOffset; + return limit - s.halfOffset; } - static void copyBytes(BitReader br, byte[] data, int offset, int length) { - if ((br.bitOffset & 7) != 0) { + static void copyBytes(State s, byte[] data, int offset, int length) { + if ((s.bitOffset & 7) != 0) { throw new BrotliRuntimeException("Unaligned copyBytes"); } // Drain accumulator. - while ((br.bitOffset != 64) && (length != 0)) { - data[offset++] = (byte) (br.accumulator >>> br.bitOffset); - br.bitOffset += 8; + while ((s.bitOffset != BITNESS) && (length != 0)) { + data[offset++] = (byte) peekBits(s); + s.bitOffset += 8; length--; } if (length == 0) { @@ -204,43 +193,63 @@ final class BitReader { } // Get data from shadow buffer with "sizeof(int)" granularity. - int copyInts = Math.min(intAvailable(br), length >> 2); - if (copyInts > 0) { - int readOffset = br.intOffset << 2; - System.arraycopy(br.byteBuffer, readOffset, data, offset, copyInts << 2); - offset += copyInts << 2; - length -= copyInts << 2; - br.intOffset += copyInts; + int copyNibbles = Math.min(halfAvailable(s), length >> LOG_HALF_SIZE); + if (copyNibbles > 0) { + int readOffset = s.halfOffset << LOG_HALF_SIZE; + int delta = copyNibbles << LOG_HALF_SIZE; + System.arraycopy(s.byteBuffer, readOffset, data, offset, delta); + offset += delta; + length -= delta; + s.halfOffset += copyNibbles; } if (length == 0) { return; } // Read tail bytes. - if (intAvailable(br) > 0) { + if (halfAvailable(s) > 0) { // length = 1..3 - fillBitWindow(br); + fillBitWindow(s); while (length != 0) { - data[offset++] = (byte) (br.accumulator >>> br.bitOffset); - br.bitOffset += 8; + data[offset++] = (byte) peekBits(s); + s.bitOffset += 8; length--; } - checkHealth(br, false); + checkHealth(s, 0); return; } // Now it is possible to copy bytes directly. - try { - while (length > 0) { - int len = br.input.read(data, offset, length); - if (len == -1) { - throw new BrotliRuntimeException("Unexpected end of input"); - } - offset += len; - length -= len; + while (length > 0) { + int len = Utils.readInput(s.input, data, offset, length); + if (len == -1) { + throw new BrotliRuntimeException("Unexpected end of input"); + } + offset += len; + length -= len; + } + } + + /** + * Translates bytes to halves (int/short). + */ + static void bytesToNibbles(State s, int byteLen) { + byte[] byteBuffer = s.byteBuffer; + int halfLen = byteLen >> LOG_HALF_SIZE; + if (BITNESS == 64) { + int[] intBuffer = s.intBuffer; + for (int i = 0; i < halfLen; ++i) { + intBuffer[i] = ((byteBuffer[i * 4] & 0xFF)) + | ((byteBuffer[(i * 4) + 1] & 0xFF) << 8) + | ((byteBuffer[(i * 4) + 2] & 0xFF) << 16) + | ((byteBuffer[(i * 4) + 3] & 0xFF) << 24); + } + } else { + short[] shortBuffer = s.shortBuffer; + for (int i = 0; i < halfLen; ++i) { + shortBuffer[i] = (short) ((byteBuffer[i * 2] & 0xFF) + | ((byteBuffer[(i * 2) + 1] & 0xFF) << 8)); } - } catch (IOException e) { - throw new BrotliRuntimeException("Failed to read input", e); } } } diff --git a/java/org/brotli/dec/BitReaderTest.java b/java/org/brotli/dec/BitReaderTest.java index bc76ce1..fa57640 100755 --- a/java/org/brotli/dec/BitReaderTest.java +++ b/java/org/brotli/dec/BitReaderTest.java @@ -21,11 +21,11 @@ public class BitReaderTest { @Test public void testReadAfterEos() { - BitReader reader = new BitReader(); - BitReader.init(reader, new ByteArrayInputStream(new byte[1])); + State reader = new State(); + Decode.initState(reader, new ByteArrayInputStream(new byte[1])); BitReader.readBits(reader, 9); try { - BitReader.checkHealth(reader, false); + BitReader.checkHealth(reader, 0); } catch (BrotliRuntimeException ex) { // This exception is expected. return; diff --git a/java/org/brotli/dec/BrotliInputStream.java b/java/org/brotli/dec/BrotliInputStream.java index a886b5d..a2bca95 100755 --- a/java/org/brotli/dec/BrotliInputStream.java +++ b/java/org/brotli/dec/BrotliInputStream.java @@ -50,7 +50,7 @@ public class BrotliInputStream extends InputStream { * @throws IOException in case of corrupted data or source stream problems */ public BrotliInputStream(InputStream source) throws IOException { - this(source, DEFAULT_INTERNAL_BUFFER_SIZE, null); + this(source, DEFAULT_INTERNAL_BUFFER_SIZE); } /** @@ -67,25 +67,6 @@ public class BrotliInputStream extends InputStream { * @throws IOException in case of corrupted data or source stream problems */ public BrotliInputStream(InputStream source, int byteReadBufferSize) throws IOException { - this(source, byteReadBufferSize, null); - } - - /** - * Creates a {@link InputStream} wrapper that decompresses brotli data. - * - *
For byte-by-byte reading ({@link #read()}) internal buffer of specified size is - * allocated and used. - * - *
Will block the thread until first kilobyte of data of source is available. - * - * @param source compressed data source - * @param byteReadBufferSize size of internal buffer used in case of - * byte-by-byte reading - * @param customDictionary custom dictionary data; {@code null} if not used - * @throws IOException in case of corrupted data or source stream problems - */ - public BrotliInputStream(InputStream source, int byteReadBufferSize, - byte[] customDictionary) throws IOException { if (byteReadBufferSize <= 0) { throw new IllegalArgumentException("Bad buffer size:" + byteReadBufferSize); } else if (source == null) { @@ -95,13 +76,10 @@ public class BrotliInputStream extends InputStream { this.remainingBufferBytes = 0; this.bufferOffset = 0; try { - State.setInput(state, source); + Decode.initState(state, source); } catch (BrotliRuntimeException ex) { throw new IOException("Brotli decoder initialization failed", ex); } - if (customDictionary != null) { - Decode.setCustomDictionary(state, customDictionary); - } } /** @@ -109,7 +87,7 @@ public class BrotliInputStream extends InputStream { */ @Override public void close() throws IOException { - State.close(state); + Decode.close(state); } /** diff --git a/java/org/brotli/dec/Context.java b/java/org/brotli/dec/Context.java index a604f7c..d9f3f91 100755 --- a/java/org/brotli/dec/Context.java +++ b/java/org/brotli/dec/Context.java @@ -11,152 +11,48 @@ package org.brotli.dec; */ final class Context { - static final int[] LOOKUP = { - // CONTEXT_UTF8, last byte. - // ASCII range. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 4, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 8, 12, 16, 12, 12, 20, 12, 16, 24, 28, 12, 12, 32, 12, 36, 12, - 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 32, 32, 24, 40, 28, 12, - 12, 48, 52, 52, 52, 48, 52, 52, 52, 48, 52, 52, 52, 52, 52, 48, - 52, 52, 52, 52, 52, 48, 52, 52, 52, 52, 52, 24, 12, 28, 12, 12, - 12, 56, 60, 60, 60, 56, 60, 60, 60, 56, 60, 60, 60, 60, 60, 56, - 60, 60, 60, 60, 60, 56, 60, 60, 60, 60, 60, 24, 12, 28, 12, 0, + static final int[] LOOKUP = new int[2048]; - // UTF8 continuation byte range. - 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, - 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, - 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, - 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + private static final String UTF_MAP = " !! ! \"#$##%#$&'##(#)#+++++++++" + + "+((&*'##,---,---,-----,-----,-----'###.///.///./////./////./////'# "; + private static final String UTF_RLE = "A/* ': & : $ \u0081 @"; - // UTF8 lead byte range. - 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, - 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, - 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, - 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, + private static void unpackLookupTable(int[] lookup, String map, String rle) { + // LSB6, MSB6, SIGNED + for (int i = 0; i < 256; ++i) { + lookup[i] = i & 0x3F; + lookup[512 + i] = i >> 2; + lookup[1792 + i] = 2 + (i >> 6); + } + // UTF8 + for (int i = 0; i < 128; ++i) { + lookup[1024 + i] = 4 * (map.charAt(i) - 32); + } + for (int i = 0; i < 64; ++i) { + lookup[1152 + i] = i & 1; + lookup[1216 + i] = 2 + (i & 1); + } + int offset = 1280; + for (int k = 0; k < 19; ++k) { + int value = k & 3; + int rep = rle.charAt(k) - 32; + for (int i = 0; i < rep; ++i) { + lookup[offset++] = value; + } + } + // SIGNED + for (int i = 0; i < 16; ++i) { + lookup[1792 + i] = 1; + lookup[2032 + i] = 6; + } + lookup[1792] = 0; + lookup[2047] = 7; + for (int i = 0; i < 256; ++i) { + lookup[1536 + i] = lookup[1792 + i] << 3; + } + } - // CONTEXT_UTF8 second last byte. - // ASCII range. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, - 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, - 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 1, 0, - - // UTF8 continuation byte range. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - // UTF8 lead byte range. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - - // CONTEXT_SIGNED, second last byte. - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, - - // CONTEXT_SIGNED, last byte, same as the above values shifted by 3 bits. - 0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, - 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, - 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, - 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 56, - - // CONTEXT_LSB6, last byte. - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, - 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, - 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, - 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, - 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, - - // CONTEXT_MSB6, last byte. - 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, - 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, - 8, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 11, 11, 11, 11, - 12, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15, - 16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18, 19, 19, 19, 19, - 20, 20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 23, 23, 23, 23, - 24, 24, 24, 24, 25, 25, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27, - 28, 28, 28, 28, 29, 29, 29, 29, 30, 30, 30, 30, 31, 31, 31, 31, - 32, 32, 32, 32, 33, 33, 33, 33, 34, 34, 34, 34, 35, 35, 35, 35, - 36, 36, 36, 36, 37, 37, 37, 37, 38, 38, 38, 38, 39, 39, 39, 39, - 40, 40, 40, 40, 41, 41, 41, 41, 42, 42, 42, 42, 43, 43, 43, 43, - 44, 44, 44, 44, 45, 45, 45, 45, 46, 46, 46, 46, 47, 47, 47, 47, - 48, 48, 48, 48, 49, 49, 49, 49, 50, 50, 50, 50, 51, 51, 51, 51, - 52, 52, 52, 52, 53, 53, 53, 53, 54, 54, 54, 54, 55, 55, 55, 55, - 56, 56, 56, 56, 57, 57, 57, 57, 58, 58, 58, 58, 59, 59, 59, 59, - 60, 60, 60, 60, 61, 61, 61, 61, 62, 62, 62, 62, 63, 63, 63, 63, - - // CONTEXT_{M,L}SB6, second last byte, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - }; - - static final int[] LOOKUP_OFFSETS = { - // CONTEXT_LSB6 - 1024, 1536, - // CONTEXT_MSB6 - 1280, 1536, - // CONTEXT_UTF8 - 0, 256, - // CONTEXT_SIGNED - 768, 512 - }; + static { + unpackLookupTable(LOOKUP, UTF_MAP, UTF_RLE); + } } diff --git a/java/org/brotli/dec/Decode.java b/java/org/brotli/dec/Decode.java index c988760..4e1e35e 100755 --- a/java/org/brotli/dec/Decode.java +++ b/java/org/brotli/dec/Decode.java @@ -6,25 +6,31 @@ package org.brotli.dec; -import static org.brotli.dec.RunningState.BLOCK_START; -import static org.brotli.dec.RunningState.CLOSED; -import static org.brotli.dec.RunningState.COMPRESSED_BLOCK_START; -import static org.brotli.dec.RunningState.COPY_LOOP; -import static org.brotli.dec.RunningState.COPY_UNCOMPRESSED; -import static org.brotli.dec.RunningState.COPY_WRAP_BUFFER; -import static org.brotli.dec.RunningState.FINISHED; -import static org.brotli.dec.RunningState.INSERT_LOOP; -import static org.brotli.dec.RunningState.MAIN_LOOP; -import static org.brotli.dec.RunningState.READ_METADATA; -import static org.brotli.dec.RunningState.TRANSFORM; -import static org.brotli.dec.RunningState.UNINITIALIZED; -import static org.brotli.dec.RunningState.WRITE; +import java.io.IOException; +import java.io.InputStream; /** * API for Brotli decompression. */ final class Decode { + //---------------------------------------------------------------------------- + // RunningState + //---------------------------------------------------------------------------- + private static final int UNINITIALIZED = 0; + private static final int BLOCK_START = 1; + private static final int COMPRESSED_BLOCK_START = 2; + private static final int MAIN_LOOP = 3; + private static final int READ_METADATA = 4; + private static final int COPY_UNCOMPRESSED = 5; + private static final int INSERT_LOOP = 6; + private static final int COPY_LOOP = 7; + private static final int COPY_WRAP_BUFFER = 8; + private static final int TRANSFORM = 9; + private static final int FINISHED = 10; + private static final int CLOSED = 11; + private static final int WRITE = 12; + private static final int DEFAULT_CODE_LENGTH = 8; private static final int CODE_LENGTH_REPEAT_CODE = 16; private static final int NUM_LITERAL_CODES = 256; @@ -36,6 +42,12 @@ final class Decode { private static final int HUFFMAN_TABLE_BITS = 8; private static final int HUFFMAN_TABLE_MASK = 0xFF; + /** + * Maximum possible Huffman table size for an alphabet size of 704, max code length 15 and root + * table bits 8. + */ + static final int HUFFMAN_TABLE_SIZE = 1080; + private static final int CODE_LENGTH_CODES = 18; private static final int[] CODE_LENGTH_CODE_ORDER = { 1, 2, 3, 4, 0, 5, 17, 6, 16, 7, 8, 9, 10, 11, 12, 13, 14, 15, @@ -58,85 +70,189 @@ final class Decode { 0x020000, 0x020004, 0x020003, 0x030002, 0x020000, 0x020004, 0x020003, 0x040005 }; + static final int[] DICTIONARY_OFFSETS_BY_LENGTH = { + 0, 0, 0, 0, 0, 4096, 9216, 21504, 35840, 44032, 53248, 63488, 74752, 87040, 93696, 100864, + 104704, 106752, 108928, 113536, 115968, 118528, 119872, 121280, 122016 + }; + + static final int[] DICTIONARY_SIZE_BITS_BY_LENGTH = { + 0, 0, 0, 0, 10, 10, 11, 11, 10, 10, 10, 10, 10, 9, 9, 8, 7, 7, 8, 7, 7, 6, 6, 5, 5 + }; + + static final int MIN_WORD_LENGTH = 4; + + static final int MAX_WORD_LENGTH = 24; + + static final int MAX_TRANSFORMED_WORD_LENGTH = 5 + MAX_WORD_LENGTH + 8; + + //---------------------------------------------------------------------------- + // Prefix code LUT. + //---------------------------------------------------------------------------- + static final int[] BLOCK_LENGTH_OFFSET = { + 1, 5, 9, 13, 17, 25, 33, 41, 49, 65, 81, 97, 113, 145, 177, 209, 241, 305, 369, 497, + 753, 1265, 2289, 4337, 8433, 16625 + }; + + static final int[] BLOCK_LENGTH_N_BITS = { + 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 7, 8, 9, 10, 11, 12, 13, 24 + }; + + static final int[] INSERT_LENGTH_OFFSET = { + 0, 1, 2, 3, 4, 5, 6, 8, 10, 14, 18, 26, 34, 50, 66, 98, 130, 194, 322, 578, 1090, 2114, 6210, + 22594 + }; + + static final int[] INSERT_LENGTH_N_BITS = { + 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 12, 14, 24 + }; + + static final int[] COPY_LENGTH_OFFSET = { + 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 18, 22, 30, 38, 54, 70, 102, 134, 198, 326, 582, 1094, + 2118 + }; + + static final int[] COPY_LENGTH_N_BITS = { + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 24 + }; + + static final int[] INSERT_RANGE_LUT = { + 0, 0, 8, 8, 0, 16, 8, 16, 16 + }; + + static final int[] COPY_RANGE_LUT = { + 0, 8, 0, 8, 16, 0, 16, 8, 16 + }; + + private static int decodeWindowBits(State s) { + if (BitReader.readFewBits(s, 1) == 0) { + return 16; + } + int n = BitReader.readFewBits(s, 3); + if (n != 0) { + return 17 + n; + } + n = BitReader.readFewBits(s, 3); + if (n != 0) { + return 8 + n; + } + return 17; + } + + /** + * Associate input with decoder state. + * + * @param s uninitialized state without associated input + * @param input compressed data source + */ + static void initState(State s, InputStream input) { + if (s.runningState != UNINITIALIZED) { + throw new IllegalStateException("State MUST be uninitialized"); + } + s.blockTrees = new int[6 * HUFFMAN_TABLE_SIZE]; + s.input = input; + BitReader.initBitReader(s); + int windowBits = decodeWindowBits(s); + if (windowBits == 9) { /* Reserved case for future expansion. */ + throw new BrotliRuntimeException("Invalid 'windowBits' code"); + } + s.maxRingBufferSize = 1 << windowBits; + s.maxBackwardDistance = s.maxRingBufferSize - 16; + s.runningState = BLOCK_START; + } + + static void close(State s) throws IOException { + if (s.runningState == UNINITIALIZED) { + throw new IllegalStateException("State MUST be initialized"); + } + if (s.runningState == CLOSED) { + return; + } + s.runningState = CLOSED; + if (s.input != null) { + Utils.closeInput(s.input); + s.input = null; + } + } + /** * Decodes a number in the range [0..255], by reading 1 - 11 bits. */ - private static int decodeVarLenUnsignedByte(BitReader br) { - if (BitReader.readBits(br, 1) != 0) { - int n = BitReader.readBits(br, 3); + private static int decodeVarLenUnsignedByte(State s) { + if (BitReader.readFewBits(s, 1) != 0) { + int n = BitReader.readFewBits(s, 3); if (n == 0) { return 1; } else { - return BitReader.readBits(br, n) + (1 << n); + return BitReader.readFewBits(s, n) + (1 << n); } } return 0; } - private static void decodeMetaBlockLength(BitReader br, State state) { - state.inputEnd = BitReader.readBits(br, 1) == 1; - state.metaBlockLength = 0; - state.isUncompressed = false; - state.isMetadata = false; - if (state.inputEnd && BitReader.readBits(br, 1) != 0) { + private static void decodeMetaBlockLength(State s) { + s.inputEnd = BitReader.readFewBits(s, 1); + s.metaBlockLength = 0; + s.isUncompressed = 0; + s.isMetadata = 0; + if ((s.inputEnd != 0) && BitReader.readFewBits(s, 1) != 0) { return; } - int sizeNibbles = BitReader.readBits(br, 2) + 4; + int sizeNibbles = BitReader.readFewBits(s, 2) + 4; if (sizeNibbles == 7) { - state.isMetadata = true; - if (BitReader.readBits(br, 1) != 0) { + s.isMetadata = 1; + if (BitReader.readFewBits(s, 1) != 0) { throw new BrotliRuntimeException("Corrupted reserved bit"); } - int sizeBytes = BitReader.readBits(br, 2); + int sizeBytes = BitReader.readFewBits(s, 2); if (sizeBytes == 0) { return; } for (int i = 0; i < sizeBytes; i++) { - int bits = BitReader.readBits(br, 8); + int bits = BitReader.readFewBits(s, 8); if (bits == 0 && i + 1 == sizeBytes && sizeBytes > 1) { throw new BrotliRuntimeException("Exuberant nibble"); } - state.metaBlockLength |= bits << (i * 8); + s.metaBlockLength |= bits << (i * 8); } } else { for (int i = 0; i < sizeNibbles; i++) { - int bits = BitReader.readBits(br, 4); + int bits = BitReader.readFewBits(s, 4); if (bits == 0 && i + 1 == sizeNibbles && sizeNibbles > 4) { throw new BrotliRuntimeException("Exuberant nibble"); } - state.metaBlockLength |= bits << (i * 4); + s.metaBlockLength |= bits << (i * 4); } } - state.metaBlockLength++; - if (!state.inputEnd) { - state.isUncompressed = BitReader.readBits(br, 1) == 1; + s.metaBlockLength++; + if (s.inputEnd == 0) { + s.isUncompressed = BitReader.readFewBits(s, 1); } } /** * Decodes the next Huffman code from bit-stream. */ - private static int readSymbol(int[] table, int offset, BitReader br) { - int val = (int) (br.accumulator >>> br.bitOffset); + private static int readSymbol(int[] table, int offset, State s) { + int val = BitReader.peekBits(s); offset += val & HUFFMAN_TABLE_MASK; int bits = table[offset] >> 16; int sym = table[offset] & 0xFFFF; if (bits <= HUFFMAN_TABLE_BITS) { - br.bitOffset += bits; + s.bitOffset += bits; return sym; } offset += sym; int mask = (1 << bits) - 1; offset += (val & mask) >>> HUFFMAN_TABLE_BITS; - br.bitOffset += ((table[offset] >> 16) + HUFFMAN_TABLE_BITS); + s.bitOffset += ((table[offset] >> 16) + HUFFMAN_TABLE_BITS); return table[offset] & 0xFFFF; } - private static int readBlockLength(int[] table, int offset, BitReader br) { - BitReader.fillBitWindow(br); - int code = readSymbol(table, offset, br); - int n = Prefix.BLOCK_LENGTH_N_BITS[code]; - return Prefix.BLOCK_LENGTH_OFFSET[code] + BitReader.readBits(br, n); + private static int readBlockLength(int[] table, int offset, State s) { + BitReader.fillBitWindow(s); + int code = readSymbol(table, offset, s); + int n = BLOCK_LENGTH_N_BITS[code]; + return BLOCK_LENGTH_OFFSET[code] + BitReader.readBits(s, n); } private static int translateShortCodes(int code, int[] ringBuffer, int index) { @@ -171,7 +287,7 @@ final class Decode { } private static void readHuffmanCodeLengths( - int[] codeLengthCodeLengths, int numSymbols, int[] codeLengths, BitReader br) { + int[] codeLengthCodeLengths, int numSymbols, int[] codeLengths, State s) { int symbol = 0; int prevCodeLen = DEFAULT_CODE_LENGTH; int repeat = 0; @@ -182,10 +298,10 @@ final class Decode { Huffman.buildHuffmanTable(table, 0, 5, codeLengthCodeLengths, CODE_LENGTH_CODES); while (symbol < numSymbols && space > 0) { - BitReader.readMoreInput(br); - BitReader.fillBitWindow(br); - int p = (int) ((br.accumulator >>> br.bitOffset)) & 31; - br.bitOffset += table[p] >> 16; + BitReader.readMoreInput(s); + BitReader.fillBitWindow(s); + int p = BitReader.peekBits(s) & 31; + s.bitOffset += table[p] >> 16; int codeLen = table[p] & 0xFFFF; if (codeLen < CODE_LENGTH_REPEAT_CODE) { repeat = 0; @@ -209,7 +325,7 @@ final class Decode { repeat -= 2; repeat <<= extraBits; } - repeat += BitReader.readBits(br, extraBits) + 3; + repeat += BitReader.readFewBits(s, extraBits) + 3; int repeatDelta = repeat - oldRepeat; if (symbol + repeatDelta > numSymbols) { throw new BrotliRuntimeException("symbol + repeatDelta > numSymbols"); // COV_NF_LINE @@ -226,22 +342,33 @@ final class Decode { throw new BrotliRuntimeException("Unused space"); // COV_NF_LINE } // TODO: Pass max_symbol to Huffman table builder instead? - Utils.fillWithZeroes(codeLengths, symbol, numSymbols - symbol); + Utils.fillIntsWithZeroes(codeLengths, symbol, numSymbols); + } + + static int checkDupes(int[] symbols, int length) { + for (int i = 0; i < length - 1; ++i) { + for (int j = i + 1; j < length; ++j) { + if (symbols[i] == symbols[j]) { + return 0; + } + } + } + return 1; } // TODO: Use specialized versions for smaller tables. - static void readHuffmanCode(int alphabetSize, int[] table, int offset, BitReader br) { - boolean ok = true; + static void readHuffmanCode(int alphabetSize, int[] table, int offset, State s) { + int ok = 1; int simpleCodeOrSkip; - BitReader.readMoreInput(br); + BitReader.readMoreInput(s); // TODO: Avoid allocation. int[] codeLengths = new int[alphabetSize]; - simpleCodeOrSkip = BitReader.readBits(br, 2); + simpleCodeOrSkip = BitReader.readFewBits(s, 2); if (simpleCodeOrSkip == 1) { // Read symbols, codes & code lengths directly. int maxBitsCounter = alphabetSize - 1; int maxBits = 0; int[] symbols = new int[4]; - int numSymbols = BitReader.readBits(br, 2) + 1; + int numSymbols = BitReader.readFewBits(s, 2) + 1; while (maxBitsCounter != 0) { maxBitsCounter >>= 1; maxBits++; @@ -249,42 +376,36 @@ final class Decode { // TODO: uncomment when codeLengths is reused. // Utils.fillWithZeroes(codeLengths, 0, alphabetSize); for (int i = 0; i < numSymbols; i++) { - symbols[i] = BitReader.readBits(br, maxBits) % alphabetSize; + symbols[i] = BitReader.readFewBits(s, maxBits) % alphabetSize; codeLengths[symbols[i]] = 2; } codeLengths[symbols[0]] = 1; switch (numSymbols) { - case 1: - break; case 2: - ok = symbols[0] != symbols[1]; codeLengths[symbols[1]] = 1; break; - case 3: - ok = symbols[0] != symbols[1] && symbols[0] != symbols[2] && symbols[1] != symbols[2]; - break; case 4: - default: - ok = symbols[0] != symbols[1] && symbols[0] != symbols[2] && symbols[0] != symbols[3] - && symbols[1] != symbols[2] && symbols[1] != symbols[3] && symbols[2] != symbols[3]; - if (BitReader.readBits(br, 1) == 1) { + if (BitReader.readFewBits(s, 1) == 1) { codeLengths[symbols[2]] = 3; codeLengths[symbols[3]] = 3; } else { codeLengths[symbols[0]] = 2; } break; + default: + break; } + ok = checkDupes(symbols, numSymbols); } else { // Decode Huffman-coded code lengths. int[] codeLengthCodeLengths = new int[CODE_LENGTH_CODES]; int space = 32; int numCodes = 0; for (int i = simpleCodeOrSkip; i < CODE_LENGTH_CODES && space > 0; i++) { int codeLenIdx = CODE_LENGTH_CODE_ORDER[i]; - BitReader.fillBitWindow(br); - int p = (int) (br.accumulator >>> br.bitOffset) & 15; + BitReader.fillBitWindow(s); + int p = BitReader.peekBits(s) & 15; // TODO: Demultiplex FIXED_TABLE. - br.bitOffset += FIXED_TABLE[p] >> 16; + s.bitOffset += FIXED_TABLE[p] >> 16; int v = FIXED_TABLE[p] & 0xFFFF; codeLengthCodeLengths[codeLenIdx] = v; if (v != 0) { @@ -292,40 +413,42 @@ final class Decode { numCodes++; } } - ok = (numCodes == 1 || space == 0); - readHuffmanCodeLengths(codeLengthCodeLengths, alphabetSize, codeLengths, br); + if (space != 0 && numCodes != 1) { + ok = 0; + } + readHuffmanCodeLengths(codeLengthCodeLengths, alphabetSize, codeLengths, s); } - if (!ok) { + if (ok == 0) { throw new BrotliRuntimeException("Can't readHuffmanCode"); // COV_NF_LINE } Huffman.buildHuffmanTable(table, offset, HUFFMAN_TABLE_BITS, codeLengths, alphabetSize); } - private static int decodeContextMap(int contextMapSize, byte[] contextMap, BitReader br) { - BitReader.readMoreInput(br); - int numTrees = decodeVarLenUnsignedByte(br) + 1; + private static int decodeContextMap(int contextMapSize, byte[] contextMap, State s) { + BitReader.readMoreInput(s); + int numTrees = decodeVarLenUnsignedByte(s) + 1; if (numTrees == 1) { - Utils.fillWithZeroes(contextMap, 0, contextMapSize); + Utils.fillBytesWithZeroes(contextMap, 0, contextMapSize); return numTrees; } - boolean useRleForZeros = BitReader.readBits(br, 1) == 1; + int useRleForZeros = BitReader.readFewBits(s, 1); int maxRunLengthPrefix = 0; - if (useRleForZeros) { - maxRunLengthPrefix = BitReader.readBits(br, 4) + 1; + if (useRleForZeros != 0) { + maxRunLengthPrefix = BitReader.readFewBits(s, 4) + 1; } - int[] table = new int[Huffman.HUFFMAN_MAX_TABLE_SIZE]; - readHuffmanCode(numTrees + maxRunLengthPrefix, table, 0, br); + int[] table = new int[HUFFMAN_TABLE_SIZE]; + readHuffmanCode(numTrees + maxRunLengthPrefix, table, 0, s); for (int i = 0; i < contextMapSize; ) { - BitReader.readMoreInput(br); - BitReader.fillBitWindow(br); - int code = readSymbol(table, 0, br); + BitReader.readMoreInput(s); + BitReader.fillBitWindow(s); + int code = readSymbol(table, 0, s); if (code == 0) { contextMap[i] = 0; i++; } else if (code <= maxRunLengthPrefix) { - int reps = (1 << code) + BitReader.readBits(br, code); + int reps = (1 << code) + BitReader.readFewBits(s, code); while (reps != 0) { if (i >= contextMapSize) { throw new BrotliRuntimeException("Corrupted context map"); // COV_NF_LINE @@ -339,21 +462,18 @@ final class Decode { i++; } } - if (BitReader.readBits(br, 1) == 1) { + if (BitReader.readFewBits(s, 1) == 1) { inverseMoveToFrontTransform(contextMap, contextMapSize); } return numTrees; } - private static void decodeBlockTypeAndLength(State state, int treeType) { - final BitReader br = state.br; - final int[] ringBuffers = state.blockTypeRb; - final int offset = treeType * 2; - BitReader.fillBitWindow(br); - int blockType = readSymbol( - state.blockTypeTrees, treeType * Huffman.HUFFMAN_MAX_TABLE_SIZE, br); - state.blockLength[treeType] = readBlockLength(state.blockLenTrees, - treeType * Huffman.HUFFMAN_MAX_TABLE_SIZE, br); + private static int decodeBlockTypeAndLength(State s, int treeType, int numBlockTypes) { + final int[] ringBuffers = s.rings; + final int offset = 4 + treeType * 2; + BitReader.fillBitWindow(s); + int blockType = readSymbol(s.blockTrees, treeType * HUFFMAN_TABLE_SIZE, s); + int result = readBlockLength(s.blockTrees, (treeType + 3) * HUFFMAN_TABLE_SIZE, s); if (blockType == 1) { blockType = ringBuffers[offset + 1] + 1; @@ -362,455 +482,445 @@ final class Decode { } else { blockType -= 2; } - if (blockType >= state.numBlockTypes[treeType]) { - blockType -= state.numBlockTypes[treeType]; + if (blockType >= numBlockTypes) { + blockType -= numBlockTypes; } ringBuffers[offset] = ringBuffers[offset + 1]; ringBuffers[offset + 1] = blockType; + return result; } - private static void decodeLiteralBlockSwitch(State state) { - decodeBlockTypeAndLength(state, 0); - int literalBlockType = state.blockTypeRb[1]; - state.contextMapSlice = literalBlockType << LITERAL_CONTEXT_BITS; - state.literalTreeIndex = state.contextMap[state.contextMapSlice] & 0xFF; - state.literalTree = state.hGroup0.trees[state.literalTreeIndex]; - int contextMode = state.contextModes[literalBlockType]; - state.contextLookupOffset1 = Context.LOOKUP_OFFSETS[contextMode]; - state.contextLookupOffset2 = Context.LOOKUP_OFFSETS[contextMode + 1]; + private static void decodeLiteralBlockSwitch(State s) { + s.literalBlockLength = decodeBlockTypeAndLength(s, 0, s.numLiteralBlockTypes); + int literalBlockType = s.rings[5]; + s.contextMapSlice = literalBlockType << LITERAL_CONTEXT_BITS; + s.literalTreeIndex = s.contextMap[s.contextMapSlice] & 0xFF; + s.literalTree = s.hGroup0[s.literalTreeIndex]; + int contextMode = s.contextModes[literalBlockType]; + s.contextLookupOffset1 = contextMode << 9; + s.contextLookupOffset2 = s.contextLookupOffset1 + 256; } - private static void decodeCommandBlockSwitch(State state) { - decodeBlockTypeAndLength(state, 1); - state.treeCommandOffset = state.hGroup1.trees[state.blockTypeRb[3]]; + private static void decodeCommandBlockSwitch(State s) { + s.commandBlockLength = decodeBlockTypeAndLength(s, 1, s.numCommandBlockTypes); + s.treeCommandOffset = s.hGroup1[s.rings[7]]; } - private static void decodeDistanceBlockSwitch(State state) { - decodeBlockTypeAndLength(state, 2); - state.distContextMapSlice = state.blockTypeRb[5] << DISTANCE_CONTEXT_BITS; + private static void decodeDistanceBlockSwitch(State s) { + s.distanceBlockLength = decodeBlockTypeAndLength(s, 2, s.numDistanceBlockTypes); + s.distContextMapSlice = s.rings[9] << DISTANCE_CONTEXT_BITS; } - private static void maybeReallocateRingBuffer(State state) { - int newSize = state.maxRingBufferSize; - if ((long) newSize > state.expectedTotalSize) { + private static void maybeReallocateRingBuffer(State s) { + int newSize = s.maxRingBufferSize; + if (newSize > s.expectedTotalSize) { /* TODO: Handle 2GB+ cases more gracefully. */ - int minimalNewSize = (int) state.expectedTotalSize + state.customDictionary.length; + int minimalNewSize = s.expectedTotalSize; while ((newSize >> 1) > minimalNewSize) { newSize >>= 1; } - if (!state.inputEnd && newSize < 16384 && state.maxRingBufferSize >= 16384) { + if ((s.inputEnd == 0) && newSize < 16384 && s.maxRingBufferSize >= 16384) { newSize = 16384; } } - if (newSize <= state.ringBufferSize) { + if (newSize <= s.ringBufferSize) { return; } - int ringBufferSizeWithSlack = newSize + Dictionary.MAX_TRANSFORMED_WORD_LENGTH; + int ringBufferSizeWithSlack = newSize + MAX_TRANSFORMED_WORD_LENGTH; byte[] newBuffer = new byte[ringBufferSizeWithSlack]; - if (state.ringBuffer != null) { - System.arraycopy(state.ringBuffer, 0, newBuffer, 0, state.ringBufferSize); - } else { - /* Prepend custom dictionary, if any. */ - if (state.customDictionary.length != 0) { - int length = state.customDictionary.length; - int offset = 0; - if (length > state.maxBackwardDistance) { - offset = length - state.maxBackwardDistance; - length = state.maxBackwardDistance; - } - System.arraycopy(state.customDictionary, offset, newBuffer, 0, length); - state.pos = length; - state.bytesToIgnore = length; - } + if (s.ringBuffer.length != 0) { + System.arraycopy(s.ringBuffer, 0, newBuffer, 0, s.ringBufferSize); } - state.ringBuffer = newBuffer; - state.ringBufferSize = newSize; + s.ringBuffer = newBuffer; + s.ringBufferSize = newSize; } - /** - * Reads next metablock header. - * - * @param state decoding state - */ - private static void readMetablockInfo(State state) { - final BitReader br = state.br; - - if (state.inputEnd) { - state.nextRunningState = FINISHED; - state.bytesToWrite = state.pos; - state.bytesWritten = 0; - state.runningState = WRITE; + private static void readNextMetablockHeader(State s) { + if (s.inputEnd != 0) { + s.nextRunningState = FINISHED; + s.bytesToWrite = s.pos; + s.bytesWritten = 0; + s.runningState = WRITE; return; } // TODO: Reset? Do we need this? - state.hGroup0.codes = null; - state.hGroup0.trees = null; - state.hGroup1.codes = null; - state.hGroup1.trees = null; - state.hGroup2.codes = null; - state.hGroup2.trees = null; + s.hGroup0 = new int[0]; + s.hGroup1 = new int[0]; + s.hGroup2 = new int[0]; - BitReader.readMoreInput(br); - decodeMetaBlockLength(br, state); - if (state.metaBlockLength == 0 && !state.isMetadata) { + BitReader.readMoreInput(s); + decodeMetaBlockLength(s); + if ((s.metaBlockLength == 0) && (s.isMetadata == 0)) { return; } - if (state.isUncompressed || state.isMetadata) { - BitReader.jumpToByteBoundary(br); - state.runningState = state.isMetadata ? READ_METADATA : COPY_UNCOMPRESSED; + if ((s.isUncompressed != 0) || (s.isMetadata != 0)) { + BitReader.jumpToByteBoundary(s); + s.runningState = (s.isMetadata != 0) ? READ_METADATA : COPY_UNCOMPRESSED; } else { - state.runningState = COMPRESSED_BLOCK_START; + s.runningState = COMPRESSED_BLOCK_START; } - if (state.isMetadata) { + if (s.isMetadata != 0) { return; } - state.expectedTotalSize += state.metaBlockLength; - if (state.ringBufferSize < state.maxRingBufferSize) { - maybeReallocateRingBuffer(state); + s.expectedTotalSize += s.metaBlockLength; + if (s.expectedTotalSize > 1 << 30) { + s.expectedTotalSize = 1 << 30; + } + if (s.ringBufferSize < s.maxRingBufferSize) { + maybeReallocateRingBuffer(s); } } - private static void readMetablockHuffmanCodesAndContextMaps(State state) { - final BitReader br = state.br; - - for (int i = 0; i < 3; i++) { - state.numBlockTypes[i] = decodeVarLenUnsignedByte(br) + 1; - state.blockLength[i] = 1 << 28; - if (state.numBlockTypes[i] > 1) { - readHuffmanCode(state.numBlockTypes[i] + 2, state.blockTypeTrees, - i * Huffman.HUFFMAN_MAX_TABLE_SIZE, br); - readHuffmanCode(NUM_BLOCK_LENGTH_CODES, state.blockLenTrees, - i * Huffman.HUFFMAN_MAX_TABLE_SIZE, br); - state.blockLength[i] = readBlockLength(state.blockLenTrees, - i * Huffman.HUFFMAN_MAX_TABLE_SIZE, br); - } + private static int readMetablockPartition(State s, int treeType, int numBlockTypes) { + if (numBlockTypes <= 1) { + return 1 << 28; } + readHuffmanCode(numBlockTypes + 2, s.blockTrees, treeType * HUFFMAN_TABLE_SIZE, s); + readHuffmanCode(NUM_BLOCK_LENGTH_CODES, s.blockTrees, (treeType + 3) * HUFFMAN_TABLE_SIZE, s); + return readBlockLength(s.blockTrees, (treeType + 3) * HUFFMAN_TABLE_SIZE, s); + } - BitReader.readMoreInput(br); - state.distancePostfixBits = BitReader.readBits(br, 2); - state.numDirectDistanceCodes = - NUM_DISTANCE_SHORT_CODES + (BitReader.readBits(br, 4) << state.distancePostfixBits); - state.distancePostfixMask = (1 << state.distancePostfixBits) - 1; - int numDistanceCodes = state.numDirectDistanceCodes + (48 << state.distancePostfixBits); + private static void readMetablockHuffmanCodesAndContextMaps(State s) { + s.numLiteralBlockTypes = decodeVarLenUnsignedByte(s) + 1; + s.literalBlockLength = readMetablockPartition(s, 0, s.numLiteralBlockTypes); + s.numCommandBlockTypes = decodeVarLenUnsignedByte(s) + 1; + s.commandBlockLength = readMetablockPartition(s, 1, s.numCommandBlockTypes); + s.numDistanceBlockTypes = decodeVarLenUnsignedByte(s) + 1; + s.distanceBlockLength = readMetablockPartition(s, 2, s.numDistanceBlockTypes); + + BitReader.readMoreInput(s); + s.distancePostfixBits = BitReader.readFewBits(s, 2); + s.numDirectDistanceCodes = + NUM_DISTANCE_SHORT_CODES + (BitReader.readFewBits(s, 4) << s.distancePostfixBits); + s.distancePostfixMask = (1 << s.distancePostfixBits) - 1; + int numDistanceCodes = s.numDirectDistanceCodes + (48 << s.distancePostfixBits); // TODO: Reuse? - state.contextModes = new byte[state.numBlockTypes[0]]; - for (int i = 0; i < state.numBlockTypes[0];) { + s.contextModes = new byte[s.numLiteralBlockTypes]; + for (int i = 0; i < s.numLiteralBlockTypes;) { /* Ensure that less than 256 bits read between readMoreInput. */ - int limit = Math.min(i + 96, state.numBlockTypes[0]); + int limit = Math.min(i + 96, s.numLiteralBlockTypes); for (; i < limit; ++i) { - state.contextModes[i] = (byte) (BitReader.readBits(br, 2) << 1); + s.contextModes[i] = (byte) (BitReader.readFewBits(s, 2)); } - BitReader.readMoreInput(br); + BitReader.readMoreInput(s); } // TODO: Reuse? - state.contextMap = new byte[state.numBlockTypes[0] << LITERAL_CONTEXT_BITS]; - int numLiteralTrees = decodeContextMap(state.numBlockTypes[0] << LITERAL_CONTEXT_BITS, - state.contextMap, br); - state.trivialLiteralContext = true; - for (int j = 0; j < state.numBlockTypes[0] << LITERAL_CONTEXT_BITS; j++) { - if (state.contextMap[j] != j >> LITERAL_CONTEXT_BITS) { - state.trivialLiteralContext = false; + s.contextMap = new byte[s.numLiteralBlockTypes << LITERAL_CONTEXT_BITS]; + int numLiteralTrees = decodeContextMap(s.numLiteralBlockTypes << LITERAL_CONTEXT_BITS, + s.contextMap, s); + s.trivialLiteralContext = 1; + for (int j = 0; j < s.numLiteralBlockTypes << LITERAL_CONTEXT_BITS; j++) { + if (s.contextMap[j] != j >> LITERAL_CONTEXT_BITS) { + s.trivialLiteralContext = 0; break; } } // TODO: Reuse? - state.distContextMap = new byte[state.numBlockTypes[2] << DISTANCE_CONTEXT_BITS]; - int numDistTrees = decodeContextMap(state.numBlockTypes[2] << DISTANCE_CONTEXT_BITS, - state.distContextMap, br); + s.distContextMap = new byte[s.numDistanceBlockTypes << DISTANCE_CONTEXT_BITS]; + int numDistTrees = decodeContextMap(s.numDistanceBlockTypes << DISTANCE_CONTEXT_BITS, + s.distContextMap, s); - HuffmanTreeGroup.init(state.hGroup0, NUM_LITERAL_CODES, numLiteralTrees); - HuffmanTreeGroup.init(state.hGroup1, NUM_INSERT_AND_COPY_CODES, state.numBlockTypes[1]); - HuffmanTreeGroup.init(state.hGroup2, numDistanceCodes, numDistTrees); + s.hGroup0 = decodeHuffmanTreeGroup(NUM_LITERAL_CODES, numLiteralTrees, s); + s.hGroup1 = + decodeHuffmanTreeGroup(NUM_INSERT_AND_COPY_CODES, s.numCommandBlockTypes, s); + s.hGroup2 = decodeHuffmanTreeGroup(numDistanceCodes, numDistTrees, s); - HuffmanTreeGroup.decode(state.hGroup0, br); - HuffmanTreeGroup.decode(state.hGroup1, br); - HuffmanTreeGroup.decode(state.hGroup2, br); + s.contextMapSlice = 0; + s.distContextMapSlice = 0; + s.contextLookupOffset1 = (int) (s.contextModes[0]) << 9; + s.contextLookupOffset2 = s.contextLookupOffset1 + 256; + s.literalTreeIndex = 0; + s.literalTree = s.hGroup0[0]; + s.treeCommandOffset = s.hGroup1[0]; - state.contextMapSlice = 0; - state.distContextMapSlice = 0; - state.contextLookupOffset1 = Context.LOOKUP_OFFSETS[state.contextModes[0]]; - state.contextLookupOffset2 = Context.LOOKUP_OFFSETS[state.contextModes[0] + 1]; - state.literalTreeIndex = 0; - state.literalTree = state.hGroup0.trees[0]; - state.treeCommandOffset = state.hGroup1.trees[0]; // TODO: == 0? - - state.blockTypeRb[0] = state.blockTypeRb[2] = state.blockTypeRb[4] = 1; - state.blockTypeRb[1] = state.blockTypeRb[3] = state.blockTypeRb[5] = 0; + s.rings[4] = 1; + s.rings[5] = 0; + s.rings[6] = 1; + s.rings[7] = 0; + s.rings[8] = 1; + s.rings[9] = 0; } - private static void copyUncompressedData(State state) { - final BitReader br = state.br; - final byte[] ringBuffer = state.ringBuffer; + private static void copyUncompressedData(State s) { + final byte[] ringBuffer = s.ringBuffer; // Could happen if block ends at ring buffer end. - if (state.metaBlockLength <= 0) { - BitReader.reload(br); - state.runningState = BLOCK_START; + if (s.metaBlockLength <= 0) { + BitReader.reload(s); + s.runningState = BLOCK_START; return; } - int chunkLength = Math.min(state.ringBufferSize - state.pos, state.metaBlockLength); - BitReader.copyBytes(br, ringBuffer, state.pos, chunkLength); - state.metaBlockLength -= chunkLength; - state.pos += chunkLength; - if (state.pos == state.ringBufferSize) { - state.nextRunningState = COPY_UNCOMPRESSED; - state.bytesToWrite = state.ringBufferSize; - state.bytesWritten = 0; - state.runningState = WRITE; + int chunkLength = Math.min(s.ringBufferSize - s.pos, s.metaBlockLength); + BitReader.copyBytes(s, ringBuffer, s.pos, chunkLength); + s.metaBlockLength -= chunkLength; + s.pos += chunkLength; + if (s.pos == s.ringBufferSize) { + s.nextRunningState = COPY_UNCOMPRESSED; + s.bytesToWrite = s.ringBufferSize; + s.bytesWritten = 0; + s.runningState = WRITE; return; } - BitReader.reload(br); - state.runningState = BLOCK_START; + BitReader.reload(s); + s.runningState = BLOCK_START; } - private static boolean writeRingBuffer(State state) { - /* Ignore custom dictionary bytes. */ - if (state.bytesToIgnore != 0) { - state.bytesWritten += state.bytesToIgnore; - state.bytesToIgnore = 0; + private static int writeRingBuffer(State s) { + if (s.bytesToIgnore != 0) { + s.bytesWritten += s.bytesToIgnore; + s.bytesToIgnore = 0; } - int toWrite = Math.min(state.outputLength - state.outputUsed, - state.bytesToWrite - state.bytesWritten); + int toWrite = Math.min(s.outputLength - s.outputUsed, + s.bytesToWrite - s.bytesWritten); if (toWrite != 0) { - System.arraycopy(state.ringBuffer, state.bytesWritten, state.output, - state.outputOffset + state.outputUsed, toWrite); - state.outputUsed += toWrite; - state.bytesWritten += toWrite; + System.arraycopy(s.ringBuffer, s.bytesWritten, s.output, + s.outputOffset + s.outputUsed, toWrite); + s.outputUsed += toWrite; + s.bytesWritten += toWrite; } - return state.outputUsed < state.outputLength; + if (s.outputUsed < s.outputLength) { + return 1; + } else { + return 0; + } } - static void setCustomDictionary(State state, byte[] data) { - state.customDictionary = (data == null) ? new byte[0] : data; + private static int[] decodeHuffmanTreeGroup(int alphabetSize, int n, State s) { + int[] group = new int[n + (n * HUFFMAN_TABLE_SIZE)]; + int next = n; + for (int i = 0; i < n; i++) { + group[i] = next; + Decode.readHuffmanCode(alphabetSize, group, next, s); + next += HUFFMAN_TABLE_SIZE; + } + return group; } /** * Actual decompress implementation. */ - static void decompress(State state) { - if (state.runningState == UNINITIALIZED) { + static void decompress(State s) { + if (s.runningState == UNINITIALIZED) { throw new IllegalStateException("Can't decompress until initialized"); } - if (state.runningState == CLOSED) { + if (s.runningState == CLOSED) { throw new IllegalStateException("Can't decompress after close"); } - final BitReader br = state.br; - int ringBufferMask = state.ringBufferSize - 1; - byte[] ringBuffer = state.ringBuffer; + int ringBufferMask = s.ringBufferSize - 1; + byte[] ringBuffer = s.ringBuffer; - while (state.runningState != FINISHED) { + while (s.runningState != FINISHED) { // TODO: extract cases to methods for the better readability. - switch (state.runningState) { + switch (s.runningState) { case BLOCK_START: - if (state.metaBlockLength < 0) { + if (s.metaBlockLength < 0) { throw new BrotliRuntimeException("Invalid metablock length"); } - readMetablockInfo(state); + readNextMetablockHeader(s); /* Ring-buffer would be reallocated here. */ - ringBufferMask = state.ringBufferSize - 1; - ringBuffer = state.ringBuffer; + ringBufferMask = s.ringBufferSize - 1; + ringBuffer = s.ringBuffer; continue; case COMPRESSED_BLOCK_START: - readMetablockHuffmanCodesAndContextMaps(state); - state.runningState = MAIN_LOOP; + readMetablockHuffmanCodesAndContextMaps(s); + s.runningState = MAIN_LOOP; // Fall through case MAIN_LOOP: - if (state.metaBlockLength <= 0) { - state.runningState = BLOCK_START; + if (s.metaBlockLength <= 0) { + s.runningState = BLOCK_START; continue; } - BitReader.readMoreInput(br); - if (state.blockLength[1] == 0) { - decodeCommandBlockSwitch(state); + BitReader.readMoreInput(s); + if (s.commandBlockLength == 0) { + decodeCommandBlockSwitch(s); } - state.blockLength[1]--; - BitReader.fillBitWindow(br); - int cmdCode = readSymbol(state.hGroup1.codes, state.treeCommandOffset, br); + s.commandBlockLength--; + BitReader.fillBitWindow(s); + int cmdCode = readSymbol(s.hGroup1, s.treeCommandOffset, s); int rangeIdx = cmdCode >>> 6; - state.distanceCode = 0; + s.distanceCode = 0; if (rangeIdx >= 2) { rangeIdx -= 2; - state.distanceCode = -1; + s.distanceCode = -1; } - int insertCode = Prefix.INSERT_RANGE_LUT[rangeIdx] + ((cmdCode >>> 3) & 7); - int copyCode = Prefix.COPY_RANGE_LUT[rangeIdx] + (cmdCode & 7); - state.insertLength = Prefix.INSERT_LENGTH_OFFSET[insertCode] + BitReader - .readBits(br, Prefix.INSERT_LENGTH_N_BITS[insertCode]); - state.copyLength = Prefix.COPY_LENGTH_OFFSET[copyCode] + BitReader - .readBits(br, Prefix.COPY_LENGTH_N_BITS[copyCode]); + int insertCode = INSERT_RANGE_LUT[rangeIdx] + ((cmdCode >>> 3) & 7); + int copyCode = COPY_RANGE_LUT[rangeIdx] + (cmdCode & 7); + s.insertLength = INSERT_LENGTH_OFFSET[insertCode] + BitReader + .readBits(s, INSERT_LENGTH_N_BITS[insertCode]); + s.copyLength = COPY_LENGTH_OFFSET[copyCode] + BitReader + .readBits(s, COPY_LENGTH_N_BITS[copyCode]); - state.j = 0; - state.runningState = INSERT_LOOP; + s.j = 0; + s.runningState = INSERT_LOOP; // Fall through case INSERT_LOOP: - if (state.trivialLiteralContext) { - while (state.j < state.insertLength) { - BitReader.readMoreInput(br); - if (state.blockLength[0] == 0) { - decodeLiteralBlockSwitch(state); + if (s.trivialLiteralContext != 0) { + while (s.j < s.insertLength) { + BitReader.readMoreInput(s); + if (s.literalBlockLength == 0) { + decodeLiteralBlockSwitch(s); } - state.blockLength[0]--; - BitReader.fillBitWindow(br); - ringBuffer[state.pos] = - (byte) readSymbol(state.hGroup0.codes, state.literalTree, br); - state.j++; - if (state.pos++ == ringBufferMask) { - state.nextRunningState = INSERT_LOOP; - state.bytesToWrite = state.ringBufferSize; - state.bytesWritten = 0; - state.runningState = WRITE; + s.literalBlockLength--; + BitReader.fillBitWindow(s); + ringBuffer[s.pos] = + (byte) readSymbol(s.hGroup0, s.literalTree, s); + s.j++; + if (s.pos++ == ringBufferMask) { + s.nextRunningState = INSERT_LOOP; + s.bytesToWrite = s.ringBufferSize; + s.bytesWritten = 0; + s.runningState = WRITE; break; } } } else { - int prevByte1 = ringBuffer[(state.pos - 1) & ringBufferMask] & 0xFF; - int prevByte2 = ringBuffer[(state.pos - 2) & ringBufferMask] & 0xFF; - while (state.j < state.insertLength) { - BitReader.readMoreInput(br); - if (state.blockLength[0] == 0) { - decodeLiteralBlockSwitch(state); + int prevByte1 = ringBuffer[(s.pos - 1) & ringBufferMask] & 0xFF; + int prevByte2 = ringBuffer[(s.pos - 2) & ringBufferMask] & 0xFF; + while (s.j < s.insertLength) { + BitReader.readMoreInput(s); + if (s.literalBlockLength == 0) { + decodeLiteralBlockSwitch(s); } - int literalTreeIndex = state.contextMap[state.contextMapSlice - + (Context.LOOKUP[state.contextLookupOffset1 + prevByte1] - | Context.LOOKUP[state.contextLookupOffset2 + prevByte2])] & 0xFF; - state.blockLength[0]--; + int literalTreeIndex = s.contextMap[s.contextMapSlice + + (Context.LOOKUP[s.contextLookupOffset1 + prevByte1] + | Context.LOOKUP[s.contextLookupOffset2 + prevByte2])] & 0xFF; + s.literalBlockLength--; prevByte2 = prevByte1; - BitReader.fillBitWindow(br); + BitReader.fillBitWindow(s); prevByte1 = readSymbol( - state.hGroup0.codes, state.hGroup0.trees[literalTreeIndex], br); - ringBuffer[state.pos] = (byte) prevByte1; - state.j++; - if (state.pos++ == ringBufferMask) { - state.nextRunningState = INSERT_LOOP; - state.bytesToWrite = state.ringBufferSize; - state.bytesWritten = 0; - state.runningState = WRITE; + s.hGroup0, s.hGroup0[literalTreeIndex], s); + ringBuffer[s.pos] = (byte) prevByte1; + s.j++; + if (s.pos++ == ringBufferMask) { + s.nextRunningState = INSERT_LOOP; + s.bytesToWrite = s.ringBufferSize; + s.bytesWritten = 0; + s.runningState = WRITE; break; } } } - if (state.runningState != INSERT_LOOP) { + if (s.runningState != INSERT_LOOP) { continue; } - state.metaBlockLength -= state.insertLength; - if (state.metaBlockLength <= 0) { - state.runningState = MAIN_LOOP; + s.metaBlockLength -= s.insertLength; + if (s.metaBlockLength <= 0) { + s.runningState = MAIN_LOOP; continue; } - if (state.distanceCode < 0) { - BitReader.readMoreInput(br); - if (state.blockLength[2] == 0) { - decodeDistanceBlockSwitch(state); + if (s.distanceCode < 0) { + BitReader.readMoreInput(s); + if (s.distanceBlockLength == 0) { + decodeDistanceBlockSwitch(s); } - state.blockLength[2]--; - BitReader.fillBitWindow(br); - state.distanceCode = readSymbol(state.hGroup2.codes, state.hGroup2.trees[ - state.distContextMap[state.distContextMapSlice - + (state.copyLength > 4 ? 3 : state.copyLength - 2)] & 0xFF], br); - if (state.distanceCode >= state.numDirectDistanceCodes) { - state.distanceCode -= state.numDirectDistanceCodes; - int postfix = state.distanceCode & state.distancePostfixMask; - state.distanceCode >>>= state.distancePostfixBits; - int n = (state.distanceCode >>> 1) + 1; - int offset = ((2 + (state.distanceCode & 1)) << n) - 4; - state.distanceCode = state.numDirectDistanceCodes + postfix - + ((offset + BitReader.readBits(br, n)) << state.distancePostfixBits); + s.distanceBlockLength--; + BitReader.fillBitWindow(s); + s.distanceCode = readSymbol(s.hGroup2, s.hGroup2[ + s.distContextMap[s.distContextMapSlice + + (s.copyLength > 4 ? 3 : s.copyLength - 2)] & 0xFF], s); + if (s.distanceCode >= s.numDirectDistanceCodes) { + s.distanceCode -= s.numDirectDistanceCodes; + int postfix = s.distanceCode & s.distancePostfixMask; + s.distanceCode >>>= s.distancePostfixBits; + int n = (s.distanceCode >>> 1) + 1; + int offset = ((2 + (s.distanceCode & 1)) << n) - 4; + s.distanceCode = s.numDirectDistanceCodes + postfix + + ((offset + BitReader.readBits(s, n)) << s.distancePostfixBits); } } // Convert the distance code to the actual distance by possibly looking up past distances // from the ringBuffer. - state.distance = translateShortCodes(state.distanceCode, state.distRb, state.distRbIdx); - if (state.distance < 0) { + s.distance = translateShortCodes(s.distanceCode, s.rings, s.distRbIdx); + if (s.distance < 0) { throw new BrotliRuntimeException("Negative distance"); // COV_NF_LINE } - if (state.maxDistance != state.maxBackwardDistance - && state.pos < state.maxBackwardDistance) { - state.maxDistance = state.pos; + if (s.maxDistance != s.maxBackwardDistance + && s.pos < s.maxBackwardDistance) { + s.maxDistance = s.pos; } else { - state.maxDistance = state.maxBackwardDistance; + s.maxDistance = s.maxBackwardDistance; } - state.copyDst = state.pos; - if (state.distance > state.maxDistance) { - state.runningState = TRANSFORM; + s.copyDst = s.pos; + if (s.distance > s.maxDistance) { + s.runningState = TRANSFORM; continue; } - if (state.distanceCode > 0) { - state.distRb[state.distRbIdx & 3] = state.distance; - state.distRbIdx++; + if (s.distanceCode > 0) { + s.rings[s.distRbIdx & 3] = s.distance; + s.distRbIdx++; } - if (state.copyLength > state.metaBlockLength) { + if (s.copyLength > s.metaBlockLength) { throw new BrotliRuntimeException("Invalid backward reference"); // COV_NF_LINE } - state.j = 0; - state.runningState = COPY_LOOP; + s.j = 0; + s.runningState = COPY_LOOP; // fall through case COPY_LOOP: - int src = (state.pos - state.distance) & ringBufferMask; - int dst = state.pos; - int copyLength = state.copyLength - state.j; + int src = (s.pos - s.distance) & ringBufferMask; + int dst = s.pos; + int copyLength = s.copyLength - s.j; if ((src + copyLength < ringBufferMask) && (dst + copyLength < ringBufferMask)) { for (int k = 0; k < copyLength; ++k) { ringBuffer[dst++] = ringBuffer[src++]; } - state.j += copyLength; - state.metaBlockLength -= copyLength; - state.pos += copyLength; + s.j += copyLength; + s.metaBlockLength -= copyLength; + s.pos += copyLength; } else { - for (; state.j < state.copyLength;) { - ringBuffer[state.pos] = - ringBuffer[(state.pos - state.distance) & ringBufferMask]; - state.metaBlockLength--; - state.j++; - if (state.pos++ == ringBufferMask) { - state.nextRunningState = COPY_LOOP; - state.bytesToWrite = state.ringBufferSize; - state.bytesWritten = 0; - state.runningState = WRITE; + for (; s.j < s.copyLength;) { + ringBuffer[s.pos] = + ringBuffer[(s.pos - s.distance) & ringBufferMask]; + s.metaBlockLength--; + s.j++; + if (s.pos++ == ringBufferMask) { + s.nextRunningState = COPY_LOOP; + s.bytesToWrite = s.ringBufferSize; + s.bytesWritten = 0; + s.runningState = WRITE; break; } } } - if (state.runningState == COPY_LOOP) { - state.runningState = MAIN_LOOP; + if (s.runningState == COPY_LOOP) { + s.runningState = MAIN_LOOP; } continue; case TRANSFORM: - if (state.copyLength >= Dictionary.MIN_WORD_LENGTH - && state.copyLength <= Dictionary.MAX_WORD_LENGTH) { - int offset = Dictionary.OFFSETS_BY_LENGTH[state.copyLength]; - int wordId = state.distance - state.maxDistance - 1; - int shift = Dictionary.SIZE_BITS_BY_LENGTH[state.copyLength]; + if (s.copyLength >= MIN_WORD_LENGTH + && s.copyLength <= MAX_WORD_LENGTH) { + int offset = DICTIONARY_OFFSETS_BY_LENGTH[s.copyLength]; + int wordId = s.distance - s.maxDistance - 1; + int shift = DICTIONARY_SIZE_BITS_BY_LENGTH[s.copyLength]; int mask = (1 << shift) - 1; int wordIdx = wordId & mask; int transformIdx = wordId >>> shift; - offset += wordIdx * state.copyLength; - if (transformIdx < Transform.TRANSFORMS.length) { - int len = Transform.transformDictionaryWord(ringBuffer, state.copyDst, - Dictionary.getData(), offset, state.copyLength, - Transform.TRANSFORMS[transformIdx]); - state.copyDst += len; - state.pos += len; - state.metaBlockLength -= len; - if (state.copyDst >= state.ringBufferSize) { - state.nextRunningState = COPY_WRAP_BUFFER; - state.bytesToWrite = state.ringBufferSize; - state.bytesWritten = 0; - state.runningState = WRITE; + offset += wordIdx * s.copyLength; + if (transformIdx < Transform.NUM_TRANSFORMS) { + int len = Transform.transformDictionaryWord(ringBuffer, s.copyDst, + Dictionary.getData(), offset, s.copyLength, transformIdx); + s.copyDst += len; + s.pos += len; + s.metaBlockLength -= len; + if (s.copyDst >= s.ringBufferSize) { + s.nextRunningState = COPY_WRAP_BUFFER; + s.bytesToWrite = s.ringBufferSize; + s.bytesWritten = 0; + s.runningState = WRITE; continue; } } else { @@ -819,52 +929,51 @@ final class Decode { } else { throw new BrotliRuntimeException("Invalid backward reference"); // COV_NF_LINE } - state.runningState = MAIN_LOOP; + s.runningState = MAIN_LOOP; continue; case COPY_WRAP_BUFFER: - System.arraycopy(ringBuffer, state.ringBufferSize, ringBuffer, 0, - state.copyDst - state.ringBufferSize); - state.runningState = MAIN_LOOP; + Utils.copyBytesWithin(ringBuffer, 0, s.ringBufferSize, s.copyDst); + s.runningState = MAIN_LOOP; continue; case READ_METADATA: - while (state.metaBlockLength > 0) { - BitReader.readMoreInput(br); + while (s.metaBlockLength > 0) { + BitReader.readMoreInput(s); // Optimize - BitReader.readBits(br, 8); - state.metaBlockLength--; + BitReader.readFewBits(s, 8); + s.metaBlockLength--; } - state.runningState = BLOCK_START; + s.runningState = BLOCK_START; continue; case COPY_UNCOMPRESSED: - copyUncompressedData(state); + copyUncompressedData(s); continue; case WRITE: - if (!writeRingBuffer(state)) { + if (writeRingBuffer(s) == 0) { // Output buffer is full. return; } - if (state.pos >= state.maxBackwardDistance) { - state.maxDistance = state.maxBackwardDistance; + if (s.pos >= s.maxBackwardDistance) { + s.maxDistance = s.maxBackwardDistance; } - state.pos &= ringBufferMask; - state.runningState = state.nextRunningState; + s.pos &= ringBufferMask; + s.runningState = s.nextRunningState; continue; default: - throw new BrotliRuntimeException("Unexpected state " + state.runningState); + throw new BrotliRuntimeException("Unexpected state " + s.runningState); } } - if (state.runningState == FINISHED) { - if (state.metaBlockLength < 0) { + if (s.runningState == FINISHED) { + if (s.metaBlockLength < 0) { throw new BrotliRuntimeException("Invalid metablock length"); } - BitReader.jumpToByteBoundary(br); - BitReader.checkHealth(state.br, true); + BitReader.jumpToByteBoundary(s); + BitReader.checkHealth(s, 1); } } } diff --git a/java/org/brotli/dec/DecodeTest.java b/java/org/brotli/dec/DecodeTest.java index 690ab9f..a0c2784 100755 --- a/java/org/brotli/dec/DecodeTest.java +++ b/java/org/brotli/dec/DecodeTest.java @@ -21,6 +21,14 @@ import org.junit.runners.JUnit4; @RunWith(JUnit4.class) public class DecodeTest { + static byte[] readUniBytes(String uniBytes) { + byte[] result = new byte[uniBytes.length()]; + for (int i = 0; i < result.length; ++i) { + result[i] = (byte) uniBytes.charAt(i); + } + return result; + } + private byte[] decompress(byte[] data, boolean byByte) throws IOException { byte[] buffer = new byte[65536]; ByteArrayInputStream input = new ByteArrayInputStream(data); @@ -49,35 +57,9 @@ public class DecodeTest { return output.toByteArray(); } - private byte[] decompressWithDictionary(byte[] data, byte[] dictionary) throws IOException { - byte[] buffer = new byte[65536]; - ByteArrayInputStream input = new ByteArrayInputStream(data); - ByteArrayOutputStream output = new ByteArrayOutputStream(); - BrotliInputStream brotliInput = new BrotliInputStream( - input, BrotliInputStream.DEFAULT_INTERNAL_BUFFER_SIZE, dictionary); - while (true) { - int len = brotliInput.read(buffer, 0, buffer.length); - if (len <= 0) { - break; - } - output.write(buffer, 0, len); - } - brotliInput.close(); - return output.toByteArray(); - } - - private void checkDecodeResourceWithDictionary(String expected, String compressed, - String dictionary) throws IOException { - byte[] expectedBytes = Transform.readUniBytes(expected); - byte[] compressedBytes = Transform.readUniBytes(compressed); - byte[] dictionaryBytes = Transform.readUniBytes(dictionary); - byte[] actual = decompressWithDictionary(compressedBytes, dictionaryBytes); - assertArrayEquals(expectedBytes, actual); - } - private void checkDecodeResource(String expected, String compressed) throws IOException { - byte[] expectedBytes = Transform.readUniBytes(expected); - byte[] compressedBytes = Transform.readUniBytes(compressed); + byte[] expectedBytes = readUniBytes(expected); + byte[] compressedBytes = readUniBytes(compressed); byte[] actual = decompress(compressedBytes, false); assertArrayEquals(expectedBytes, actual); byte[] actualByByte = decompress(compressedBytes, true); @@ -168,20 +150,11 @@ public class DecodeTest { + "\u0091\u00FF\u0087\u00E9\u001E"); } - @Test - public void testFoxFox() throws IOException { - checkDecodeResourceWithDictionary( - "The quick brown fox jumps over the lazy dog", - "\u001B*\u0000\u0000 \u0000\u00C2\u0098\u00B0\u00CA\u0001", - "The quick brown fox jumps over the lazy dog"); - } - @Test public void testUtils() { new Context(); new Decode(); new Dictionary(); new Huffman(); - new Prefix(); } } diff --git a/java/org/brotli/dec/Dictionary.java b/java/org/brotli/dec/Dictionary.java index 6cae81b..c40f28b 100755 --- a/java/org/brotli/dec/Dictionary.java +++ b/java/org/brotli/dec/Dictionary.java @@ -48,19 +48,4 @@ public final class Dictionary { /* Might have been set when {@link DictionaryData} was loaded.*/ return data; } - - static final int[] OFFSETS_BY_LENGTH = { - 0, 0, 0, 0, 0, 4096, 9216, 21504, 35840, 44032, 53248, 63488, 74752, 87040, 93696, 100864, - 104704, 106752, 108928, 113536, 115968, 118528, 119872, 121280, 122016 - }; - - static final int[] SIZE_BITS_BY_LENGTH = { - 0, 0, 0, 0, 10, 10, 11, 11, 10, 10, 10, 10, 10, 9, 9, 8, 7, 7, 8, 7, 7, 6, 6, 5, 5 - }; - - static final int MIN_WORD_LENGTH = 4; - - static final int MAX_WORD_LENGTH = 24; - - static final int MAX_TRANSFORMED_WORD_LENGTH = 5 + MAX_WORD_LENGTH + 8; } diff --git a/java/org/brotli/dec/DictionaryData.java b/java/org/brotli/dec/DictionaryData.java index 04d570d..b7acebe 100755 --- a/java/org/brotli/dec/DictionaryData.java +++ b/java/org/brotli/dec/DictionaryData.java @@ -16,34 +16,38 @@ import java.nio.ByteBuffer; final class DictionaryData { private static final String DATA0 = "timedownlifeleftbackcodedatashowonlysitecityopenjustlikefreeworktextyearoverbodyloveformbookplaylivelinehelphomesidemorewordlongthemviewfindpagedaysfullheadtermeachareafromtruemarkableuponhighdatelandnewsevennextcasebothpostusedmadehandherewhatnameLinkblogsizebaseheldmakemainuser') +holdendswithNewsreadweresigntakehavegameseencallpathwellplusmenufilmpartjointhislistgoodneedwayswestjobsmindalsologorichuseslastteamarmyfoodkingwilleastwardbestfirePageknowaway.pngmovethanloadgiveselfnotemuchfeedmanyrockicononcelookhidediedHomerulehostajaxinfoclublawslesshalfsomesuchzone100%onescareTimeracebluefourweekfacehopegavehardlostwhenparkkeptpassshiproomHTMLplanTypedonesavekeepflaglinksoldfivetookratetownjumpthusdarkcardfilefearstaykillthatfallautoever.comtalkshopvotedeepmoderestturnbornbandfellroseurl(skinrolecomeactsagesmeetgold.jpgitemvaryfeltthensenddropViewcopy1.0\"stopelseliestourpack.gifpastcss?graymean>rideshotlatesaidroadvar feeljohnrickportfast'UA-deadpoorbilltypeU.S.woodmust2px;Inforankwidewantwalllead[0];paulwavesure$('#waitmassarmsgoesgainlangpaid!-- lockunitrootwalkfirmwifexml\"songtest20pxkindrowstoolfontmailsafestarmapscorerainflowbabyspansays4px;6px;artsfootrealwikiheatsteptriporg/lakeweaktoldFormcastfansbankveryrunsjulytask1px;goalgrewslowedgeid=\"sets5px;.js?40pxif (soonseatnonetubezerosentreedfactintogiftharm18pxcamehillboldzoomvoideasyringfillpeakinitcost3px;jacktagsbitsrolleditknewnearironfreddiskwentsoilputs/js/holyT22:ISBNT20:adamsees
P>Q\u0002P8P7P=P>P4P>Q\u0002P>P6P5P>P=P8Q\u0005P\u001DP0P5P5P1Q\u000BPP2Q\u000BP2P>P\u001DP>P>P1P\u001FP>P;P8P=P8P P$P\u001DP5P\u001CQ\u000BQ\u0002Q\u000BP\u001EP=P8Pthing.org/multiheardPowerstandtokensolid(thisbringshipsstafftriedcallsfullyfactsagentThis //-->adminegyptEvent15px;Emailtrue\"crossspentblogsbox\">notedleavechinasizesguestrobotheavytrue,sevengrandcrimesignsawaredancephase>\n \n \r\nname=diegopage swiss-->\n\n#fff;\">Log.com\"treatsheet) && 14px;sleepntentfiledja:c\u0003id=\"cName\"worseshots-box-delta\n<bears:48Z spendbakershops= \"\";php\">ction13px;brianhellosize=o=%2F joinmaybe, fjsimg\" \")[0]MTopBType\"newlyDanskczechtrailknowsfaq\">zh-cn10);\n-1\");type=bluestrulydavis.js';>\r\n\r\nform jesus100% menu.\r\n\t\r\nwalesrisksumentddingb-likteachgif\" vegasdanskeestishqipsuomisobredesdeentretodospuedeaC1osestC!tienehastaotrospartedondenuevohacerformamismomejormundoaquC-dC-assC3loayudafechatodastantomenosdatosotrassitiomuchoahoralugarmayorestoshorastenerantesfotosestaspaC-snuevasaludforosmedioquienmesespoderchileserC!vecesdecirjosC)estarventagrupohechoellostengoamigocosasnivelgentemismaairesjuliotemashaciafavorjuniolibrepuntobuenoautorabrilbuenatextomarzosaberlistaluegocC3moenerojuegoperC:haberestoynuncamujervalorfueralibrogustaigualvotoscasosguC-apuedosomosavisousteddebennochebuscafaltaeurosseriedichocursoclavecasasleC3nplazolargoobrasvistaapoyojuntotratavistocrearcampohemoscincocargopisosordenhacenC!readiscopedrocercapuedapapelmenorC:tilclarojorgecalleponertardenadiemarcasigueellassiglocochemotosmadreclaserestoniC1oquedapasarbancohijosviajepabloC)stevienereinodejarfondocanalnorteletracausatomarmanoslunesautosvillavendopesartipostengamarcollevapadreunidovamoszonasambosbandamariaabusomuchasubirriojavivirgradochicaallC-jovendichaestantalessalirsuelopesosfinesllamabuscoC)stalleganegroplazahumorpagarjuntadobleislasbolsabaC1ohablaluchaC\u0001readicenjugarnotasvalleallC!cargadolorabajoestC)gustomentemariofirmacostofichaplatahogarartesleyesaquelmuseobasespocosmitadcielochicomiedoganarsantoetapadebesplayaredessietecortecoreadudasdeseoviejodeseaaguas"domaincommonstatuseventsmastersystemactionbannerremovescrollupdateglobalmediumfilternumberchangeresultpublicscreenchoosenormaltravelissuessourcetargetspringmodulemobileswitchphotosborderregionitselfsocialactivecolumnrecordfollowtitle>eitherlengthfamilyfriendlayoutauthorcreatereviewsummerserverplayedplayerexpandpolicyformatdoublepointsseriespersonlivingdesignmonthsforcesuniqueweightpeopleenergynaturesearchfigurehavingcustomoffsetletterwindowsubmitrendergroupsuploadhealthmethodvideosschoolfutureshadowdebatevaluesObjectothersrightsleaguechromesimplenoticesharedendingseasonreportonlinesquarebuttonimagesenablemovinglatestwinterFranceperiodstrongrepeatLondondetailformeddemandsecurepassedtoggleplacesdevicestaticcitiesstreamyellowattackstreetflighthiddeninfo\">openedusefulvalleycausesleadersecretseconddamagesportsexceptratingsignedthingseffectfieldsstatesofficevisualeditorvolumeReportmuseummoviesparentaccessmostlymother\" id=\"marketgroundchancesurveybeforesymbolmomentspeechmotioninsidematterCenterobjectexistsmiddleEuropegrowthlegacymannerenoughcareeransweroriginportalclientselectrandomclosedtopicscomingfatheroptionsimplyraisedescapechosenchurchdefinereasoncorneroutputmemoryiframepolicemodelsNumberduringoffersstyleskilledlistedcalledsilvermargindeletebetterbrowselimitsGlobalsinglewidgetcenterbudgetnowrapcreditclaimsenginesafetychoicespirit-stylespreadmakingneededrussiapleaseextentScriptbrokenallowschargedividefactormember-basedtheoryconfigaroundworkedhelpedChurchimpactshouldalwayslogo\" bottomlist\">){var prefixorangeHeader.push(couplegardenbridgelaunchReviewtakingvisionlittledatingButtonbeautythemesforgotSearchanchoralmostloadedChangereturnstringreloadMobileincomesupplySourceordersviewed courseAbout island: The dialoghousesBEGIN MexicostartscentreheightaddingIslandassetsEmpireSchooleffortdirectnearlymanualSelect.\n\nOnejoinedmenu\">PhilipawardshandleimportOfficeregardskillsnationSportsdegreeweekly (e.g.behinddoctorloggedunitedbeyond-scaleacceptservedmarineFootercamera
P=P0P3P4P5PP3P>P4P2P>Q\u0002Q\u0002P0P
P2P0Q\u0001P2P0P
Q\u0002Q\u0003Q\u0002P=P0P4P4P=Q\u000FP\u0012P>Q\u0002Q\u0002Q\u0000P8P=P5P9P\u0012P0Q\u0001P=P8P Q\u0002Q\u0000Q\u0003P1P\u001EP=P8PP P9P4P2P5P>P=P>Q\u0001Q\u0003P4`$\u0015`%\u0007`$9`%\u0008`$\u0015`%\u0000`$8`%\u0007`$\u0015`$>`$\u0015`%\u000B`$\u0014`$0`$*`$0`$(`%\u0007`$\u000F`$\u0015`$\u0015`$?`$-`%\u0000`$\u0007`$8`$\u0015`$0`$$`%\u000B`$9`%\u000B`$\u0006`$*`$9`%\u0000`$/`$9`$/`$>`$$`$\u0015`$%`$>jagran`$\u0006`$\u001C`$\u001C`%\u000B`$\u0005`$,`$&`%\u000B`$\u0017`$\u0008`$\u001C`$>`$\u0017`$\u000F`$9`$.`$\u0007`$(`$5`$9`$/`%\u0007`$%`%\u0007`$%`%\u0000`$\u0018`$0`$\u001C`$,`$&`%\u0000`$\u0015`$\u0008`$\u001C`%\u0000`$5`%\u0007`$(`$\u0008`$(`$\u000F`$9`$0`$\t`$8`$.`%\u0007`$\u0015`$.`$5`%\u000B`$2`%\u0007`$8`$,`$.`$\u0008`$&`%\u0007`$\u0013`$0`$\u0006`$.`$,`$8`$-`$0`$,`$(`$\u001A`$2`$.`$(`$\u0006`$\u0017`$8`%\u0000`$2`%\u0000X9Y\u0004Y\tX%Y\u0004Y\tY\u0007X0X'X\"X.X1X9X/X/X'Y\u0004Y\tY\u0007X0Y\u0007X5Y\u0008X1X:Y\nX1Y\u0003X'Y\u0006Y\u0008Y\u0004X'X(Y\nY\u0006X9X1X6X0Y\u0004Y\u0003Y\u0007Y\u0006X'Y\nY\u0008Y\u0005Y\u0002X'Y\u0004X9Y\u0004Y\nX'Y\u0006X'Y\u0004Y\u0003Y\u0006X-X*Y\tY\u0002X(Y\u0004Y\u0008X-X)X'X.X1Y\u0001Y\u0002X7X9X(X/X1Y\u0003Y\u0006X%X0X'Y\u0003Y\u0005X'X'X-X/X%Y\u0004X'Y\u0001Y\nY\u0007X(X9X6Y\u0003Y\nY\u0001X(X-X+Y\u0008Y\u0005Y\u0006Y\u0008Y\u0007Y\u0008X#Y\u0006X'X,X/X'Y\u0004Y\u0007X'X3Y\u0004Y\u0005X9Y\u0006X/Y\u0004Y\nX3X9X(X1X5Y\u0004Y\tY\u0005Y\u0006X0X(Y\u0007X'X#Y\u0006Y\u0007Y\u0005X+Y\u0004Y\u0003Y\u0006X*X'Y\u0004X'X-Y\nX+Y\u0005X5X1X4X1X-X-Y\u0008Y\u0004Y\u0008Y\u0001Y\nX'X0X'Y\u0004Y\u0003Y\u0004Y\u0005X1X)X'Y\u0006X*X'Y\u0004Y\u0001X#X(Y\u0008X.X'X5X#Y\u0006X*X'Y\u0006Y\u0007X'Y\u0004Y\nX9X6Y\u0008Y\u0008Y\u0002X/X'X(Y\u0006X.Y\nX1X(Y\u0006X*Y\u0004Y\u0003Y\u0005X4X'X!Y\u0008Y\u0007Y\nX'X(Y\u0008Y\u0002X5X5Y\u0008Y\u0005X'X1Y\u0002Y\u0005X#X-X/Y\u0006X-Y\u0006X9X/Y\u0005X1X#Y\nX'X-X)Y\u0003X*X(X/Y\u0008Y\u0006Y\nX,X(Y\u0005Y\u0006Y\u0007X*X-X*X,Y\u0007X)X3Y\u0006X)Y\nX*Y\u0005Y\u0003X1X)X:X2X)Y\u0006Y\u0001X3X(Y\nX*Y\u0004Y\u0004Y\u0007Y\u0004Y\u0006X'X*Y\u0004Y\u0003Y\u0002Y\u0004X(Y\u0004Y\u0005X'X9Y\u0006Y\u0007X#Y\u0008Y\u0004X4Y\nX!Y\u0006Y\u0008X1X#Y\u0005X'Y\u0001Y\nY\u0003X(Y\u0003Y\u0004X0X'X*X1X*X(X(X#Y\u0006Y\u0007Y\u0005X3X'Y\u0006Y\u0003X(Y\nX9Y\u0001Y\u0002X/X-X3Y\u0006Y\u0004Y\u0007Y\u0005X4X9X1X#Y\u0007Y\u0004X4Y\u0007X1Y\u0002X7X1X7Y\u0004X(profileservicedefaulthimselfdetailscontentsupportstartedmessagesuccessfashion
countryaccountcreatedstoriesresultsrunningprocesswritingobjectsvisiblewelcomearticleunknownnetworkcompanydynamicbrowserprivacyproblemServicerespectdisplayrequestreservewebsitehistoryfriendsoptionsworkingversionmillionchannelwindow.addressvisitedweathercorrectproductedirectforwardyou canremovedsubjectcontrolarchivecurrentreadinglibrarylimitedmanagerfurthersummarymachineminutesprivatecontextprogramsocietynumberswrittenenabledtriggersourcesloadingelementpartnerfinallyperfectmeaningsystemskeepingculture",journalprojectsurfaces"expiresreviewsbalanceEnglishContentthroughPlease opinioncontactaverageprimaryvillageSpanishgallerydeclinemeetingmissionpopularqualitymeasuregeneralspeciessessionsectionwriterscounterinitialreportsfiguresmembersholdingdisputeearlierexpressdigitalpictureAnothermarriedtrafficleadingchangedcentralvictoryimages/reasonsstudiesfeaturelistingmust beschoolsVersionusuallyepisodeplayinggrowingobviousoverlaypresentactions\r\nwrapperalreadycertainrealitystorageanotherdesktopofferedpatternunusualDigitalcapitalWebsitefailureconnectreducedAndroiddecadesregular & animalsreleaseAutomatgettingmethodsnothingPopularcaptionletterscapturesciencelicensechangesEngland=1&History = new CentralupdatedSpecialNetworkrequirecommentwarningCollegetoolbarremainsbecauseelectedDeutschfinanceworkersquicklybetweenexactlysettingdiseaseSocietyweaponsexhibit<!--Controlclassescoveredoutlineattacksdevices(windowpurposetitle=\"Mobile killingshowingItaliandroppedheavilyeffects-1']);\nconfirmCurrentadvancesharingopeningdrawingbillionorderedGermanyrelatedincludewhetherdefinedSciencecatalogArticlebuttonslargestuniformjourneysidebarChicagoholidayGeneralpassage,"animatefeelingarrivedpassingnaturalroughly.\n\nThe but notdensityBritainChineselack oftributeIreland\" data-factorsreceivethat isLibraryhusbandin factaffairsCharlesradicalbroughtfindinglanding:lang=\"return leadersplannedpremiumpackageAmericaEdition]"Messageneed tovalue=\"complexlookingstationbelievesmaller-mobilerecordswant tokind ofFirefoxyou aresimilarstudiedmaximumheadingrapidlyclimatekingdomemergedamountsfoundedpioneerformuladynastyhow to SupportrevenueeconomyResultsbrothersoldierlargelycalling."AccountEdward segmentRobert effortsPacificlearnedup withheight:we haveAngelesnations_searchappliedacquiremassivegranted: falsetreatedbiggestbenefitdrivingStudiesminimumperhapsmorningsellingis usedreversevariant role=\"missingachievepromotestudentsomeoneextremerestorebottom:evolvedall thesitemapenglishway to AugustsymbolsCompanymattersmusicalagainstserving})();\r\npaymenttroubleconceptcompareparentsplayersregionsmonitor ''The winningexploreadaptedGalleryproduceabilityenhancecareers). The collectSearch ancientexistedfooter handlerprintedconsoleEasternexportswindowsChannelillegalneutralsuggest_headersigning.html\">settledwesterncausing-webkitclaimedJusticechaptervictimsThomas mozillapromisepartieseditionoutside:false,hundredOlympic_buttonauthorsreachedchronicdemandssecondsprotectadoptedprepareneithergreatlygreateroverallimprovecommandspecialsearch.worshipfundingthoughthighestinsteadutilityquarterCulturetestingclearlyexposedBrowserliberal} catchProjectexamplehide();FloridaanswersallowedEmperordefenseseriousfreedomSeveral-buttonFurtherout of != nulltrainedDenmarkvoid(0)/all.jspreventRequestStephen\n\nWhen observe
P6P5Q\u0002P5P4Q\u0000Q\u0003P3P8Q\u0005Q\u0001P;Q\u0003Q\u0007P0P5Q\u0001P5P9Q\u0007P0Q\u0001P2Q\u0001P5P3P4P0P P>Q\u0001Q\u0001P8Q\u000FP\u001CP>Q\u0001P:P2P5P4Q\u0000Q\u0003P3P8P5P3P>Q\u0000P>P4P0P2P>P?Q\u0000P>Q\u0001P4P0P=P=Q\u000BQ\u0005P4P>P;P6P=Q\u000BP8P P P2P>P3P>P?P>P Q\tP8Q\u0001P0P9Q\u0002P>P2P?P>Q\u0007P5P Q\tQ\u000CP4P>P;P6P=P>Q\u0001Q\u0001Q\u000BP;P:P8P1Q\u000BQ\u0001Q\u0002Q\u0000P>P4P0P=P=Q\u000BP5P P3P8P5P?Q\u0000P>P5P:Q\u0002P!P5P9Q\u0007P0Q\u0001P P4P5P;P8Q\u0002P0P:P>P3P>P>P=P;P0P9P=P3P>Q\u0000P>P4P5P2P5Q\u0000Q\u0001P8Q\u000FQ\u0001Q\u0002Q\u0000P0P=P5Q\u0004P8P;Q\u000CPQ\u0002Q\u0003P0P?Q\u0000P5P;Q\u000FP2P>P>P1Q\tP5P>P4P=P>P3P>Q\u0001P2P>P5P3P>Q\u0001Q\u0002P0Q\u0002Q\u000CP8P4Q\u0000Q\u0003P3P>P9Q\u0004P>Q\u0000Q\u0003P
P
P2P=Q\u000FQ\u0000P0P7P=Q\u000BQ\u0005P8Q\u0001P:P0Q\u0002Q\u000CP=P5P4P5P;Q\u000EQ\u000FP=P2P0Q\u0000Q\u000FP