mirror of
https://github.com/google/brotli.git
synced 2024-11-08 13:20:05 +00:00
Prepare for copybara (#939)
Co-authored-by: Eugene Kliuchnikov <eustas@chromium.org>
This commit is contained in:
parent
62662f87cd
commit
8376f72ed6
@ -13,6 +13,7 @@ extern "C" {
|
||||
|
||||
#if !defined(BROTLI_EXTERNAL_DICTIONARY_DATA)
|
||||
static const uint8_t kBrotliDictionaryData[] =
|
||||
/* GENERATED CODE START */
|
||||
{
|
||||
116,105,109,101,100,111,119,110,108,105,102,101,108,101,102,116,98,97,99,107,99,
|
||||
111,100,101,100,97,116,97,115,104,111,119,111,110,108,121,115,105,116,101,99,105
|
||||
@ -5860,6 +5861,7 @@ static const uint8_t kBrotliDictionaryData[] =
|
||||
,164,181,224,164,190,224,164,136,224,164,184,224,164,149,224,165,141,224,164,176
|
||||
,224,164,191,224,164,175,224,164,164,224,164,190
|
||||
}
|
||||
/* GENERATED CODE END */
|
||||
;
|
||||
#endif /* !BROTLI_EXTERNAL_DICTIONARY_DATA */
|
||||
|
||||
|
@ -156,24 +156,6 @@ OR:
|
||||
#define BROTLI_NOINLINE
|
||||
#endif
|
||||
|
||||
/* BROTLI_INTERNAL could be defined to override visibility, e.g. for tests. */
|
||||
#if !defined(BROTLI_INTERNAL)
|
||||
#if defined(_WIN32) || defined(__CYGWIN__)
|
||||
#define BROTLI_INTERNAL
|
||||
#elif BROTLI_GNUC_VERSION_CHECK(3, 3, 0) || \
|
||||
BROTLI_TI_VERSION_CHECK(8, 0, 0) || \
|
||||
BROTLI_INTEL_VERSION_CHECK(16, 0, 0) || \
|
||||
BROTLI_ARM_VERSION_CHECK(4, 1, 0) || \
|
||||
BROTLI_IBM_VERSION_CHECK(13, 1, 0) || \
|
||||
BROTLI_SUNPRO_VERSION_CHECK(5, 11, 0) || \
|
||||
(BROTLI_TI_VERSION_CHECK(7, 3, 0) && \
|
||||
defined(__TI_GNU_ATTRIBUTE_SUPPORT__) && defined(__TI_EABI__))
|
||||
#define BROTLI_INTERNAL __attribute__ ((visibility ("hidden")))
|
||||
#else
|
||||
#define BROTLI_INTERNAL
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/* <<< <<< <<< end of hedley macros. */
|
||||
|
||||
#if BROTLI_GNUC_HAS_ATTRIBUTE(unused, 2, 7, 0) || \
|
||||
@ -485,11 +467,11 @@ static BROTLI_INLINE void BrotliDump(const char* f, int l, const char* fn) {
|
||||
#define BROTLI_DUMP() (void)(0)
|
||||
#endif
|
||||
|
||||
/* TODO: add appropriate icc/sunpro/arm/ibm/ti checks. */
|
||||
/* TODO(eustas): add appropriate icc/sunpro/arm/ibm/ti checks. */
|
||||
#if (BROTLI_GNUC_VERSION_CHECK(3, 0, 0) || defined(__llvm__)) && \
|
||||
!defined(BROTLI_BUILD_NO_RBIT)
|
||||
#if defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8_ANY)
|
||||
/* TODO: detect ARMv6T2 and enable this code for it. */
|
||||
/* TODO(eustas): detect ARMv6T2 and enable this code for it. */
|
||||
static BROTLI_INLINE brotli_reg_t BrotliRBit(brotli_reg_t input) {
|
||||
brotli_reg_t output;
|
||||
__asm__("rbit %0, %1\n" : "=r"(output) : "r"(input));
|
||||
|
@ -416,7 +416,7 @@ static BROTLI_BOOL ParseDictionary(const uint8_t* encoded, size_t size,
|
||||
BrotliSharedDictionary already contains data, compound dictionaries
|
||||
will be appended, but an error will be returned if it already has
|
||||
custom words or transforms.
|
||||
TODO: link to RFC for shared brotli once published. */
|
||||
TODO(lode): link to RFC for shared brotli once published. */
|
||||
static BROTLI_BOOL DecodeSharedDictionary(
|
||||
const uint8_t* encoded, size_t size, BrotliSharedDictionary* dict) {
|
||||
uint32_t num_prefix = 0;
|
||||
@ -492,7 +492,7 @@ BrotliSharedDictionary* BrotliSharedDictionaryCreateInstance(
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* TODO: explicitly initialize all the fields? */
|
||||
/* TODO(eustas): explicitly initialize all the fields? */
|
||||
memset(dict, 0, sizeof(BrotliSharedDictionary));
|
||||
|
||||
dict->context_based = BROTLI_FALSE;
|
||||
|
@ -218,7 +218,7 @@ static BROTLI_NOINLINE BrotliDecoderErrorCode DecodeVarLenUint8(
|
||||
|
||||
default:
|
||||
return
|
||||
BROTLI_FAILURE(BROTLI_DECODER_ERROR_UNREACHABLE);
|
||||
BROTLI_FAILURE(BROTLI_DECODER_ERROR_UNREACHABLE); /* COV_NF_LINE */
|
||||
}
|
||||
}
|
||||
|
||||
@ -339,7 +339,7 @@ static BrotliDecoderErrorCode BROTLI_NOINLINE DecodeMetaBlockLength(
|
||||
|
||||
default:
|
||||
return
|
||||
BROTLI_FAILURE(BROTLI_DECODER_ERROR_UNREACHABLE);
|
||||
BROTLI_FAILURE(BROTLI_DECODER_ERROR_UNREACHABLE); /* COV_NF_LINE */
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -865,7 +865,7 @@ static BrotliDecoderErrorCode ReadHuffmanCode(uint32_t alphabet_size_max,
|
||||
|
||||
default:
|
||||
return
|
||||
BROTLI_FAILURE(BROTLI_DECODER_ERROR_UNREACHABLE);
|
||||
BROTLI_FAILURE(BROTLI_DECODER_ERROR_UNREACHABLE); /* COV_NF_LINE */
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1112,7 +1112,7 @@ static BrotliDecoderErrorCode DecodeContextMap(uint32_t context_map_size,
|
||||
|
||||
default:
|
||||
return
|
||||
BROTLI_FAILURE(BROTLI_DECODER_ERROR_UNREACHABLE);
|
||||
BROTLI_FAILURE(BROTLI_DECODER_ERROR_UNREACHABLE); /* COV_NF_LINE */
|
||||
}
|
||||
}
|
||||
|
||||
@ -1356,7 +1356,7 @@ static BROTLI_BOOL BROTLI_NOINLINE BrotliEnsureRingBuffer(
|
||||
static BrotliDecoderErrorCode BROTLI_NOINLINE CopyUncompressedBlockToOutput(
|
||||
size_t* available_out, uint8_t** next_out, size_t* total_out,
|
||||
BrotliDecoderState* s) {
|
||||
/* TODO: avoid allocation for single uncompressed block. */
|
||||
/* TODO(eustas): avoid allocation for single uncompressed block. */
|
||||
if (!BrotliEnsureRingBuffer(s)) {
|
||||
return BROTLI_FAILURE(BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_1);
|
||||
}
|
||||
@ -1866,7 +1866,7 @@ static BROTLI_INLINE BrotliDecoderErrorCode ProcessCommandsInternal(
|
||||
} else if (s->state == BROTLI_STATE_COMMAND_POST_WRAP_COPY) {
|
||||
goto CommandPostWrapCopy;
|
||||
} else {
|
||||
return BROTLI_FAILURE(BROTLI_DECODER_ERROR_UNREACHABLE);
|
||||
return BROTLI_FAILURE(BROTLI_DECODER_ERROR_UNREACHABLE); /* COV_NF_LINE */
|
||||
}
|
||||
|
||||
CommandBegin:
|
||||
@ -2600,7 +2600,7 @@ BrotliDecoderResult BrotliDecoderDecompressStream(
|
||||
case 1: hgroup = &s->insert_copy_hgroup; break;
|
||||
case 2: hgroup = &s->distance_hgroup; break;
|
||||
default: return SaveErrorCode(s, BROTLI_FAILURE(
|
||||
BROTLI_DECODER_ERROR_UNREACHABLE));
|
||||
BROTLI_DECODER_ERROR_UNREACHABLE)); /* COV_NF_LINE */
|
||||
}
|
||||
result = HuffmanTreeGroupDecode(hgroup, s);
|
||||
if (result != BROTLI_DECODER_SUCCESS) break;
|
||||
@ -2779,6 +2779,23 @@ uint32_t BrotliDecoderVersion() {
|
||||
return BROTLI_VERSION;
|
||||
}
|
||||
|
||||
/* Escalate internal functions visibility; for testing purposes only. */
|
||||
#if defined(BROTLI_TEST)
|
||||
BROTLI_BOOL SafeReadSymbolForTest(
|
||||
const HuffmanCode*, BrotliBitReader*, uint32_t*);
|
||||
BROTLI_BOOL SafeReadSymbolForTest(
|
||||
const HuffmanCode* table, BrotliBitReader* br, uint32_t* result) {
|
||||
return SafeReadSymbol(table, br, result);
|
||||
}
|
||||
|
||||
void InverseMoveToFrontTransformForTest(
|
||||
uint8_t*, uint32_t, BrotliDecoderState*);
|
||||
void InverseMoveToFrontTransformForTest(
|
||||
uint8_t* v, uint32_t l, BrotliDecoderState* s) {
|
||||
InverseMoveToFrontTransform(v, l, s);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
@ -170,7 +170,7 @@ void BrotliSplitBlock(MemoryManager* m,
|
||||
kCommandStrideLength, kCommandBlockSwitchCost, params,
|
||||
insert_and_copy_split);
|
||||
if (BROTLI_IS_OOM(m)) return;
|
||||
/* TODO: reuse for distances? */
|
||||
/* TODO(eustas): reuse for distances? */
|
||||
BROTLI_FREE(m, insert_and_copy_codes);
|
||||
}
|
||||
|
||||
@ -197,6 +197,20 @@ void BrotliSplitBlock(MemoryManager* m,
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(BROTLI_TEST)
|
||||
size_t CountLiteralsForTest(const Command*, const size_t);
|
||||
size_t CountLiteralsForTest(const Command* cmds, const size_t num_commands) {
|
||||
return CountLiterals(cmds, num_commands);
|
||||
}
|
||||
|
||||
void CopyLiteralsToByteArrayForTest(const Command*,
|
||||
const size_t, const uint8_t*, const size_t, const size_t, uint8_t*);
|
||||
void CopyLiteralsToByteArrayForTest(const Command* cmds,
|
||||
const size_t num_commands, const uint8_t* data, const size_t offset,
|
||||
const size_t mask, uint8_t* literals) {
|
||||
CopyLiteralsToByteArray(cmds, num_commands, data, offset, mask, literals);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
|
@ -234,7 +234,7 @@ static void FN(ClusterBlocks)(MemoryManager* m,
|
||||
uint32_t* BROTLI_RESTRICT const remap = u32 + 3 * HISTOGRAMS_PER_BATCH;
|
||||
uint32_t* BROTLI_RESTRICT const block_lengths =
|
||||
u32 + 4 * HISTOGRAMS_PER_BATCH;
|
||||
/* TODO: move to arena? */
|
||||
/* TODO(eustas): move to arena? */
|
||||
HistogramType* tmp = BROTLI_ALLOC(m, HistogramType, 2);
|
||||
|
||||
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(histogram_symbols) ||
|
||||
@ -335,7 +335,7 @@ static void FN(ClusterBlocks)(MemoryManager* m,
|
||||
FN(HistogramAdd)(tmp, data[pos++]);
|
||||
}
|
||||
/* Among equally good histograms prefer last used. */
|
||||
/* TODO: should we give a block-switch discount here? */
|
||||
/* TODO(eustas): should we give a block-switch discount here? */
|
||||
best_out = (i == 0) ? histogram_symbols[0] : histogram_symbols[i - 1];
|
||||
best_bits = FN(BrotliHistogramBitCostDistance)(
|
||||
tmp, &all_histograms[best_out], tmp + 1);
|
||||
|
@ -286,7 +286,7 @@ void BrotliStoreHuffmanTree(const uint8_t* depths, size_t num,
|
||||
/* Write the Huffman tree into the brotli-representation.
|
||||
The command alphabet is the largest, so this allocation will fit all
|
||||
alphabets. */
|
||||
/* TODO: fix me */
|
||||
/* TODO(eustas): fix me */
|
||||
uint8_t huffman_tree[BROTLI_NUM_COMMAND_SYMBOLS];
|
||||
uint8_t huffman_tree_extra_bits[BROTLI_NUM_COMMAND_SYMBOLS];
|
||||
size_t huffman_tree_size = 0;
|
||||
@ -778,7 +778,7 @@ static void BuildAndStoreBlockSplitCode(const uint8_t* types,
|
||||
++length_histo[BlockLengthPrefixCode(lengths[i])];
|
||||
}
|
||||
StoreVarLenUint8(num_types - 1, storage_ix, storage);
|
||||
if (num_types > 1) { /* TODO: else? could StoreBlockSwitch occur? */
|
||||
if (num_types > 1) { /* TODO(eustas): else? could StoreBlockSwitch occur? */
|
||||
BuildAndStoreHuffmanTree(&type_histo[0], num_types + 2, num_types + 2, tree,
|
||||
&code->type_depths[0], &code->type_bits[0],
|
||||
storage_ix, storage);
|
||||
@ -1150,12 +1150,12 @@ static void StoreDataWithHuffmanCodes(const uint8_t* input,
|
||||
}
|
||||
}
|
||||
|
||||
/* TODO: pull alloc/dealloc to caller? */
|
||||
/* TODO(eustas): pull alloc/dealloc to caller? */
|
||||
typedef struct MetablockArena {
|
||||
HistogramLiteral lit_histo;
|
||||
HistogramCommand cmd_histo;
|
||||
HistogramDistance dist_histo;
|
||||
/* TODO: merge bits and depth? */
|
||||
/* TODO(eustas): merge bits and depth? */
|
||||
uint8_t lit_depth[BROTLI_NUM_LITERAL_SYMBOLS];
|
||||
uint16_t lit_bits[BROTLI_NUM_LITERAL_SYMBOLS];
|
||||
uint8_t cmd_depth[BROTLI_NUM_COMMAND_SYMBOLS];
|
||||
@ -1323,6 +1323,13 @@ void BrotliStoreUncompressedMetaBlock(BROTLI_BOOL is_final_block,
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(BROTLI_TEST)
|
||||
void GetBlockLengthPrefixCodeForTest(uint32_t len, size_t* code,
|
||||
uint32_t* n_extra, uint32_t* extra) {
|
||||
GetBlockLengthPrefixCode(len, code, n_extra, extra);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
@ -77,6 +77,10 @@ BROTLI_INTERNAL void BrotliStoreUncompressedMetaBlock(
|
||||
size_t position, size_t mask, size_t len,
|
||||
size_t* BROTLI_RESTRICT storage_ix, uint8_t* BROTLI_RESTRICT storage);
|
||||
|
||||
#if defined(BROTLI_TEST)
|
||||
void GetBlockLengthPrefixCodeForTest(uint32_t, size_t*, uint32_t*, uint32_t*);
|
||||
#endif
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
@ -228,7 +228,7 @@ BROTLI_INTERNAL size_t FN(BrotliHistogramReindex)(MemoryManager* m,
|
||||
++next_index;
|
||||
}
|
||||
}
|
||||
/* TODO: by using idea of "cycle-sort" we can avoid allocation of
|
||||
/* TODO(eustas): by using idea of "cycle-sort" we can avoid allocation of
|
||||
tmp and reduce the number of copying by the factor of 2. */
|
||||
tmp = BROTLI_ALLOC(m, HistogramType, next_index);
|
||||
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tmp)) return 0;
|
||||
@ -259,7 +259,7 @@ BROTLI_INTERNAL void FN(BrotliClusterHistograms)(
|
||||
size_t pairs_capacity = max_input_histograms * max_input_histograms / 2;
|
||||
/* For the first pass of clustering, we allow all pairs. */
|
||||
HistogramPair* pairs = BROTLI_ALLOC(m, HistogramPair, pairs_capacity + 1);
|
||||
/* TODO: move to "persistent" arena? */
|
||||
/* TODO(eustas): move to "persistent" arena? */
|
||||
HistogramType* tmp = BROTLI_ALLOC(m, HistogramType, 1);
|
||||
size_t i;
|
||||
|
||||
|
@ -52,7 +52,7 @@ static PreparedDictionary* CreatePreparedDictionaryWithParams(MemoryManager* m,
|
||||
next_bucket = (uint32_t*)(&bucket_heads[num_buckets]);
|
||||
memset(num, 0, num_buckets * sizeof(num[0]));
|
||||
|
||||
/* TODO: apply custom "store" order. */
|
||||
/* TODO(eustas): apply custom "store" order. */
|
||||
for (i = 0; i + 7 < source_size; ++i) {
|
||||
const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(&source[i]) & hash_mask) *
|
||||
kPreparedDictionaryHashMul64Long;
|
||||
|
@ -130,7 +130,7 @@ static void BuildAndStoreCommandPrefixCode(BrotliOnePassArena* s,
|
||||
uint16_t* const bits = s->cmd_bits;
|
||||
uint8_t* BROTLI_RESTRICT const tmp_depth = s->tmp_depth;
|
||||
uint16_t* BROTLI_RESTRICT const tmp_bits = s->tmp_bits;
|
||||
/* TODO: do only once on initialization. */
|
||||
/* TODO(eustas): do only once on initialization. */
|
||||
memset(tmp_depth, 0, BROTLI_NUM_COMMAND_SYMBOLS);
|
||||
|
||||
BrotliCreateHuffmanTree(histogram, 64, 15, s->tree, depth);
|
||||
@ -168,7 +168,7 @@ static void BuildAndStoreCommandPrefixCode(BrotliOnePassArena* s,
|
||||
tmp_depth[256 + 8 * i] = depth[48 + i];
|
||||
tmp_depth[448 + 8 * i] = depth[56 + i];
|
||||
}
|
||||
/* TODO: could/should full-length machinery be avoided? */
|
||||
/* TODO(eustas): could/should full-length machinery be avoided? */
|
||||
BrotliStoreHuffmanTree(
|
||||
tmp_depth, BROTLI_NUM_COMMAND_SYMBOLS, s->tree, storage_ix, storage);
|
||||
}
|
||||
@ -567,6 +567,8 @@ trawl:
|
||||
int distance = (int)(base - candidate); /* > 0 */
|
||||
size_t insert = (size_t)(base - next_emit);
|
||||
ip += matched;
|
||||
BROTLI_LOG(("[CompressFragment] pos = %d insert = %lu copy = %d\n",
|
||||
(int)(next_emit - base_ip), (unsigned long)insert, 2));
|
||||
BROTLI_DCHECK(0 == memcmp(base, candidate, matched));
|
||||
if (BROTLI_PREDICT_TRUE(insert < 6210)) {
|
||||
EmitInsertLen(insert, cmd_depth, cmd_bits, cmd_histo,
|
||||
@ -595,6 +597,12 @@ trawl:
|
||||
}
|
||||
EmitCopyLenLastDistance(matched, cmd_depth, cmd_bits, cmd_histo,
|
||||
storage_ix, storage);
|
||||
BROTLI_LOG(("[CompressFragment] pos = %d distance = %d\n"
|
||||
"[CompressFragment] pos = %d insert = %d copy = %d\n"
|
||||
"[CompressFragment] pos = %d distance = %d\n",
|
||||
(int)(base - base_ip), (int)distance,
|
||||
(int)(base - base_ip) + 2, 0, (int)matched - 2,
|
||||
(int)(base - base_ip) + 2, (int)distance));
|
||||
|
||||
next_emit = ip;
|
||||
if (BROTLI_PREDICT_FALSE(ip >= ip_limit)) {
|
||||
@ -632,6 +640,10 @@ trawl:
|
||||
storage_ix, storage);
|
||||
EmitDistance((size_t)last_distance, cmd_depth, cmd_bits,
|
||||
cmd_histo, storage_ix, storage);
|
||||
BROTLI_LOG(("[CompressFragment] pos = %d insert = %d copy = %d\n"
|
||||
"[CompressFragment] pos = %d distance = %d\n",
|
||||
(int)(base - base_ip), 0, (int)matched,
|
||||
(int)(base - base_ip), (int)last_distance));
|
||||
|
||||
next_emit = ip;
|
||||
if (BROTLI_PREDICT_FALSE(ip >= ip_limit)) {
|
||||
@ -682,6 +694,8 @@ trawl:
|
||||
/* Emit the remaining bytes as literals. */
|
||||
if (next_emit < ip_end) {
|
||||
const size_t insert = (size_t)(ip_end - next_emit);
|
||||
BROTLI_LOG(("[CompressFragment] pos = %d insert = %lu copy = %d\n",
|
||||
(int)(next_emit - base_ip), (unsigned long)insert, 2));
|
||||
if (BROTLI_PREDICT_TRUE(insert < 6210)) {
|
||||
EmitInsertLen(insert, cmd_depth, cmd_bits, cmd_histo,
|
||||
storage_ix, storage);
|
||||
|
@ -69,7 +69,7 @@ static void BuildAndStoreCommandPrefixCode(BrotliTwoPassArena* s,
|
||||
size_t* storage_ix,
|
||||
uint8_t* storage) {
|
||||
/* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */
|
||||
/* TODO: initialize once. */
|
||||
/* TODO(eustas): initialize once. */
|
||||
memset(s->tmp_depth, 0, sizeof(s->tmp_depth));
|
||||
BrotliCreateHuffmanTree(s->cmd_histo, 64, 15, s->tmp_tree, s->cmd_depth);
|
||||
BrotliCreateHuffmanTree(&s->cmd_histo[64], 64, 14, s->tmp_tree,
|
||||
@ -329,6 +329,8 @@ trawl:
|
||||
ip += matched;
|
||||
BROTLI_DCHECK(0 == memcmp(base, candidate, matched));
|
||||
EmitInsertLen((uint32_t)insert, commands);
|
||||
BROTLI_LOG(("[CompressFragment] pos = %d insert = %d copy = %d\n",
|
||||
(int)(next_emit - base_ip), insert, 2));
|
||||
memcpy(*literals, next_emit, (size_t)insert);
|
||||
*literals += insert;
|
||||
if (distance == last_distance) {
|
||||
@ -339,6 +341,12 @@ trawl:
|
||||
last_distance = distance;
|
||||
}
|
||||
EmitCopyLenLastDistance(matched, commands);
|
||||
BROTLI_LOG(("[CompressFragment] pos = %d distance = %d\n"
|
||||
"[CompressFragment] pos = %d insert = %d copy = %d\n"
|
||||
"[CompressFragment] pos = %d distance = %d\n",
|
||||
(int)(base - base_ip), (int)distance,
|
||||
(int)(base - base_ip) + 2, 0, (int)matched - 2,
|
||||
(int)(base - base_ip) + 2, (int)distance));
|
||||
|
||||
next_emit = ip;
|
||||
if (BROTLI_PREDICT_FALSE(ip >= ip_limit)) {
|
||||
@ -394,6 +402,10 @@ trawl:
|
||||
BROTLI_DCHECK(0 == memcmp(base, candidate, matched));
|
||||
EmitCopyLen(matched, commands);
|
||||
EmitDistance((uint32_t)last_distance, commands);
|
||||
BROTLI_LOG(("[CompressFragment] pos = %d insert = %d copy = %d\n"
|
||||
"[CompressFragment] pos = %d distance = %d\n",
|
||||
(int)(base - base_ip), 0, (int)matched,
|
||||
(int)(base - base_ip), (int)last_distance));
|
||||
|
||||
next_emit = ip;
|
||||
if (BROTLI_PREDICT_FALSE(ip >= ip_limit)) {
|
||||
@ -446,6 +458,8 @@ emit_remainder:
|
||||
if (next_emit < ip_end) {
|
||||
const uint32_t insert = (uint32_t)(ip_end - next_emit);
|
||||
EmitInsertLen(insert, commands);
|
||||
BROTLI_LOG(("[CompressFragment] pos = %d insert = %d copy = %d\n",
|
||||
(int)(next_emit - base_ip), insert, 2));
|
||||
memcpy(*literals, next_emit, insert);
|
||||
*literals += insert;
|
||||
}
|
||||
@ -472,9 +486,9 @@ static void StoreCommands(BrotliTwoPassArena* s,
|
||||
|
||||
size_t i;
|
||||
memset(s->lit_histo, 0, sizeof(s->lit_histo));
|
||||
/* TODO: is that necessary? */
|
||||
/* TODO(eustas): is that necessary? */
|
||||
memset(s->cmd_depth, 0, sizeof(s->cmd_depth));
|
||||
/* TODO: is that necessary? */
|
||||
/* TODO(eustas): is that necessary? */
|
||||
memset(s->cmd_bits, 0, sizeof(s->cmd_bits));
|
||||
memset(s->cmd_histo, 0, sizeof(s->cmd_histo));
|
||||
for (i = 0; i < num_literals; ++i) {
|
||||
|
@ -22,7 +22,7 @@
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* TODO: turn to macro. */
|
||||
/* TODO(eustas): turn to macro. */
|
||||
static const size_t kCompressFragmentTwoPassBlockSize = 1 << 17;
|
||||
|
||||
typedef struct BrotliTwoPassArena {
|
||||
|
@ -13,6 +13,7 @@
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* GENERATED CODE START */
|
||||
BROTLI_INTERNAL const uint16_t kStaticDictionaryHashWords[32768] = {
|
||||
1002,0,0,0,0,0,0,0,0,683,0,0,0,0,0,0,0,1265,0,0,0,0,0,1431,0,0,0,0,0,0,40,0,0,0,
|
||||
0,155,8,741,0,624,0,0,0,0,0,0,0,0,0,0,0,0,66,503,0,0,0,451,0,0,0,0,0,0,0,835,70,
|
||||
@ -1840,6 +1841,7 @@ BROTLI_INTERNAL const uint8_t kStaticDictionaryHashLengths[32768] = {
|
||||
0,0,0,0,6,0,0,0,0,0,0,0,0,0,0,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,0,0,0,0,0,0,0,0,
|
||||
10,7,0,0,0,0,0,0,0,0,9,0,0,0,0,4,0,0,0,0,0,0,0,0,0,5,11,0,0,0,0,0,0,0,8,6,0,0,9,
|
||||
7,0,0,12,4,0,0,0,0,0,0,12,6,0,6,0,7,0,0,8,5,0,0,0,0};
|
||||
/* GENERATED CODE END */
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
|
161
c/enc/encode.c
161
c/enc/encode.c
@ -138,7 +138,7 @@ BROTLI_BOOL BrotliEncoderSetParameter(
|
||||
BrotliEncoderState* state, BrotliEncoderParameter p, uint32_t value) {
|
||||
/* Changing parameters on the fly is not implemented yet. */
|
||||
if (state->is_initialized_) return BROTLI_FALSE;
|
||||
/* TODO: Validate/clamp parameters here. */
|
||||
/* TODO(eustas): Validate/clamp parameters here. */
|
||||
switch (p) {
|
||||
case BROTLI_PARAM_MODE:
|
||||
state->params.mode = (BrotliEncoderMode)value;
|
||||
@ -274,7 +274,7 @@ static void EncodeWindowBits(int lgwin, BROTLI_BOOL large_window,
|
||||
}
|
||||
}
|
||||
|
||||
/* TODO: move to compress_fragment.c? */
|
||||
/* TODO(eustas): move to compress_fragment.c? */
|
||||
/* Initializes the command and distance prefix codes for the first block. */
|
||||
static void InitCommandPrefixCodes(BrotliOnePassArena* s) {
|
||||
static const uint8_t kDefaultCommandDepths[128] = {
|
||||
@ -506,7 +506,7 @@ static void DecideOverLiteralContextModeling(const uint8_t* input,
|
||||
static BROTLI_BOOL ShouldCompress(
|
||||
const uint8_t* data, const size_t mask, const uint64_t last_flush_pos,
|
||||
const size_t bytes, const size_t num_literals, const size_t num_commands) {
|
||||
/* TODO: find more precise minimal block overhead. */
|
||||
/* TODO(eustas): find more precise minimal block overhead. */
|
||||
if (bytes <= 2) return BROTLI_FALSE;
|
||||
if (num_commands < (bytes >> 8) + 2) {
|
||||
if ((double)num_literals > 0.99 * (double)bytes) {
|
||||
@ -606,7 +606,7 @@ static void WriteMetaBlockInternal(MemoryManager* m,
|
||||
size_t num_literal_contexts = 1;
|
||||
const uint32_t* literal_context_map = NULL;
|
||||
if (!params->disable_literal_context_modeling) {
|
||||
/* TODO: pull to higher level and reuse. */
|
||||
/* TODO(eustas): pull to higher level and reuse. */
|
||||
uint32_t* arena = BROTLI_ALLOC(m, uint32_t, 14 * 32);
|
||||
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(arena)) return;
|
||||
DecideOverLiteralContextModeling(
|
||||
@ -1136,7 +1136,7 @@ static BROTLI_BOOL EncodeData(
|
||||
const size_t max_commands = max_length / 8;
|
||||
const size_t processed_bytes = (size_t)(s->input_pos_ - s->last_flush_pos_);
|
||||
/* If maximal possible additional block doesn't fit metablock, flush now. */
|
||||
/* TODO: Postpone decision until next block arrives? */
|
||||
/* TODO(eustas): Postpone decision until next block arrives? */
|
||||
const BROTLI_BOOL next_input_fits_metablock = TO_BROTLI_BOOL(
|
||||
processed_bytes + InputBlockSize(s) <= max_length);
|
||||
/* If block splitting is not used, then flush as soon as there is some
|
||||
@ -1337,7 +1337,7 @@ static BROTLI_NOINLINE BROTLI_BOOL BrotliCompressBufferQuality10(
|
||||
allocation here and not before the loop, because if the input is small,
|
||||
this will be allocated after the Zopfli cost model is freed, so this
|
||||
will not increase peak memory usage.
|
||||
TODO: If the first allocation is too small, increase command
|
||||
TODO(eustas): If the first allocation is too small, increase command
|
||||
buffer size exponentially. */
|
||||
new_cmd_alloc_size = BROTLI_MAX(size_t, expected_num_commands,
|
||||
num_commands + path_size + 1);
|
||||
@ -1540,7 +1540,7 @@ BROTLI_BOOL BrotliEncoderCompress(
|
||||
return BROTLI_TRUE;
|
||||
}
|
||||
if (quality == 10) {
|
||||
/* TODO: Implement this direct path for all quality levels. */
|
||||
/* TODO(eustas): Implement this direct path for all quality levels. */
|
||||
const int lg_win = BROTLI_MIN(int, BROTLI_LARGE_MAX_WINDOW_BITS,
|
||||
BROTLI_MAX(int, 16, lgwin));
|
||||
int ok = BrotliCompressBufferQuality10(lg_win, input_size, input_buffer,
|
||||
@ -2091,6 +2091,153 @@ BROTLI_BOOL BrotliEncoderAttachPreparedDictionary(BrotliEncoderState* state,
|
||||
return BROTLI_TRUE;
|
||||
}
|
||||
|
||||
size_t BrotliEncoderEstimatePeakMemoryUsage(int quality, int lgwin,
|
||||
size_t input_size) {
|
||||
BrotliEncoderParams params;
|
||||
BrotliEncoderInitParams(¶ms);
|
||||
params.quality = quality;
|
||||
params.lgwin = lgwin;
|
||||
params.size_hint = input_size;
|
||||
SanitizeParams(¶ms);
|
||||
params.lgblock = ComputeLgBlock(¶ms);
|
||||
ChooseHasher(¶ms, ¶ms.hasher);
|
||||
if (params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY ||
|
||||
params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY) {
|
||||
size_t state_size = sizeof(BrotliEncoderState);
|
||||
size_t block_size = BROTLI_MIN(size_t, input_size, (1ul << params.lgwin));
|
||||
size_t hash_table_size =
|
||||
HashTableSize(MaxHashTableSize(params.quality), block_size);
|
||||
size_t hash_size =
|
||||
(hash_table_size < (1u << 10)) ? 0 : sizeof(int) * hash_table_size;
|
||||
size_t cmdbuf_size = params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY ?
|
||||
5 * BROTLI_MIN(size_t, block_size, 1ul << 17) : 0;
|
||||
if (params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY) {
|
||||
state_size += sizeof(BrotliOnePassArena);
|
||||
} else {
|
||||
state_size += sizeof(BrotliTwoPassArena);
|
||||
}
|
||||
return hash_size + cmdbuf_size + state_size;
|
||||
} else {
|
||||
size_t short_ringbuffer_size = (size_t)1 << params.lgblock;
|
||||
int ringbuffer_bits = ComputeRbBits(¶ms);
|
||||
size_t ringbuffer_size = input_size < short_ringbuffer_size ?
|
||||
input_size : (1u << ringbuffer_bits) + short_ringbuffer_size;
|
||||
size_t hash_size[4] = {0};
|
||||
size_t metablock_size =
|
||||
BROTLI_MIN(size_t, input_size, MaxMetablockSize(¶ms));
|
||||
size_t inputblock_size =
|
||||
BROTLI_MIN(size_t, input_size, (size_t)1 << params.lgblock);
|
||||
size_t cmdbuf_size = metablock_size * 2 + inputblock_size * 6;
|
||||
size_t outbuf_size = metablock_size * 2 + 503;
|
||||
size_t histogram_size = 0;
|
||||
HasherSize(¶ms, BROTLI_TRUE, input_size, hash_size);
|
||||
if (params.quality < MIN_QUALITY_FOR_BLOCK_SPLIT) {
|
||||
cmdbuf_size = BROTLI_MIN(size_t, cmdbuf_size,
|
||||
MAX_NUM_DELAYED_SYMBOLS * sizeof(Command) + inputblock_size * 12);
|
||||
}
|
||||
if (params.quality >= MIN_QUALITY_FOR_HQ_BLOCK_SPLITTING) {
|
||||
/* Only a very rough estimation, based on enwik8. */
|
||||
histogram_size = 200 << 20;
|
||||
} else if (params.quality >= MIN_QUALITY_FOR_BLOCK_SPLIT) {
|
||||
size_t literal_histograms =
|
||||
BROTLI_MIN(size_t, metablock_size / 6144, 256);
|
||||
size_t command_histograms =
|
||||
BROTLI_MIN(size_t, metablock_size / 6144, 256);
|
||||
size_t distance_histograms =
|
||||
BROTLI_MIN(size_t, metablock_size / 6144, 256);
|
||||
histogram_size = literal_histograms * sizeof(HistogramLiteral) +
|
||||
command_histograms * sizeof(HistogramCommand) +
|
||||
distance_histograms * sizeof(HistogramDistance);
|
||||
}
|
||||
return (ringbuffer_size +
|
||||
hash_size[0] + hash_size[1] + hash_size[2] + hash_size[3] +
|
||||
cmdbuf_size +
|
||||
outbuf_size +
|
||||
histogram_size);
|
||||
}
|
||||
}
|
||||
size_t BrotliEncoderGetPreparedDictionarySize(
|
||||
const BrotliEncoderPreparedDictionary* prepared_dictionary) {
|
||||
/* First field of dictionary structs */
|
||||
const BrotliEncoderPreparedDictionary* prepared = prepared_dictionary;
|
||||
uint32_t magic = *((const uint32_t*)prepared);
|
||||
size_t overhead = 0;
|
||||
if (magic == kManagedDictionaryMagic) {
|
||||
const ManagedDictionary* managed = (const ManagedDictionary*)prepared;
|
||||
overhead = sizeof(ManagedDictionary);
|
||||
magic = *managed->dictionary;
|
||||
prepared = (const BrotliEncoderPreparedDictionary*)managed->dictionary;
|
||||
}
|
||||
|
||||
if (magic == kPreparedDictionaryMagic) {
|
||||
const PreparedDictionary* dictionary =
|
||||
(const PreparedDictionary*)prepared;
|
||||
/* Keep in sync with step 3 of CreatePreparedDictionary */
|
||||
return sizeof(PreparedDictionary) + dictionary->source_size +
|
||||
(sizeof(uint32_t) << dictionary->slot_bits) +
|
||||
(sizeof(uint16_t) << dictionary->bucket_bits) +
|
||||
(sizeof(uint32_t) * dictionary->source_offset) + overhead;
|
||||
} else if (magic == kSharedDictionaryMagic) {
|
||||
const SharedEncoderDictionary* dictionary =
|
||||
(const SharedEncoderDictionary*)prepared;
|
||||
const CompoundDictionary* compound = &dictionary->compound;
|
||||
const ContextualEncoderDictionary* contextual = &dictionary->contextual;
|
||||
size_t result = sizeof(*dictionary);
|
||||
size_t i;
|
||||
size_t num_instances;
|
||||
const BrotliEncoderDictionary* instances;
|
||||
for (i = 0; i < compound->num_prepared_instances_; i++) {
|
||||
size_t size = BrotliEncoderGetPreparedDictionarySize(
|
||||
(const BrotliEncoderPreparedDictionary*)
|
||||
compound->prepared_instances_[i]);
|
||||
if (!size) return 0; /* error */
|
||||
result += size;
|
||||
}
|
||||
if (contextual->context_based) {
|
||||
num_instances = contextual->num_instances_;
|
||||
instances = contextual->instances_;
|
||||
result += sizeof(*instances) * num_instances;
|
||||
} else {
|
||||
num_instances = 1;
|
||||
instances = &contextual->instance_;
|
||||
}
|
||||
for (i = 0; i < num_instances; i++) {
|
||||
const BrotliEncoderDictionary* dict = &instances[i];
|
||||
result += dict->trie.pool_capacity * sizeof(BrotliTrieNode);
|
||||
if (dict->hash_table_data_words_) {
|
||||
result += sizeof(kStaticDictionaryHashWords);
|
||||
}
|
||||
if (dict->hash_table_data_lengths_) {
|
||||
result += sizeof(kStaticDictionaryHashLengths);
|
||||
}
|
||||
if (dict->buckets_data_) {
|
||||
result += sizeof(*dict->buckets_data_) * dict->buckets_alloc_size_;
|
||||
}
|
||||
if (dict->dict_words_data_) {
|
||||
result += sizeof(*dict->dict_words) * dict->dict_words_alloc_size_;
|
||||
}
|
||||
if (dict->words_instance_) {
|
||||
result += sizeof(*dict->words_instance_);
|
||||
/* data_size not added here: it is never allocated by the
|
||||
SharedEncoderDictionary, instead it always points to the file
|
||||
already loaded in memory. So if the caller wants to include
|
||||
this memory as well, add the size of the loaded dictionary
|
||||
file to this. */
|
||||
}
|
||||
}
|
||||
return result + overhead;
|
||||
}
|
||||
return 0; /* error */
|
||||
}
|
||||
|
||||
#if defined(BROTLI_TEST)
|
||||
size_t MakeUncompressedStreamForTest(const uint8_t*, size_t, uint8_t*);
|
||||
size_t MakeUncompressedStreamForTest(
|
||||
const uint8_t* input, size_t input_size, uint8_t* output) {
|
||||
return MakeUncompressedStream(input, input_size, output);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
@ -367,7 +367,7 @@ static void BuildDictionaryHashTable(uint16_t* hash_table_words,
|
||||
/* The order of the loops is such that in case of collision, words with
|
||||
shorter length are preferred, and in case of same length, words with
|
||||
smaller index. There is only a single word per bucket. */
|
||||
/* TODO: consider adding optional user-supplied frequency_map to use
|
||||
/* TODO(lode): consider adding optional user-supplied frequency_map to use
|
||||
for preferred words instead, this can make the encoder better for
|
||||
quality 9 and below without affecting the decoder */
|
||||
memset(hash_table_words, 0, sizeof(kStaticDictionaryHashWords));
|
||||
@ -424,7 +424,7 @@ static void ComputeCutoffTransforms(
|
||||
((cutoffTransforms >> (N * 6)) & 0x3F), so for example the identity
|
||||
transform code must be 0-63, for N=1 the transform code must be 4-67, ...,
|
||||
for N=9 it must be 36-99.
|
||||
TODO: consider a simple flexible uint8_t[10] instead of the uint64_t
|
||||
TODO(lode): consider a simple flexible uint8_t[10] instead of the uint64_t
|
||||
for the cutoff transforms, so that shared dictionaries can have the
|
||||
OmitLast transforms anywhere without loss. */
|
||||
*count = 0;
|
||||
@ -501,7 +501,7 @@ void BrotliInitSharedEncoderDictionary(SharedEncoderDictionary* dict) {
|
||||
dict->max_quality = BROTLI_MAX_QUALITY;
|
||||
}
|
||||
|
||||
/* TODO: make sure that tooling will warn user if not all the cutoff
|
||||
/* TODO(eustas): make sure that tooling will warn user if not all the cutoff
|
||||
transforms are available (for low-quality encoder). */
|
||||
static BROTLI_BOOL InitCustomSharedEncoderDictionary(
|
||||
MemoryManager* m, const BrotliSharedDictionary* decoded_dict,
|
||||
@ -623,6 +623,14 @@ void BrotliDestroyManagedDictionary(ManagedDictionary* dictionary) {
|
||||
BrotliBootstrapFree(dictionary, &dictionary->memory_manager_);
|
||||
}
|
||||
|
||||
/* Escalate internal functions visibility; for testing purposes only. */
|
||||
#if defined(BROTLI_TEST)
|
||||
void InitEncoderDictionaryForTest(BrotliEncoderDictionary*);
|
||||
void InitEncoderDictionaryForTest(BrotliEncoderDictionary* d) {
|
||||
InitEncoderDictionary(d);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
#endif
|
||||
|
@ -76,6 +76,7 @@ static const uint8_t kStaticDistanceCodeDepth[64] = {
|
||||
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
|
||||
};
|
||||
|
||||
/* GENERATED CODE START */
|
||||
static const uint32_t kCodeLengthBits[18] = {
|
||||
0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 15, 31, 0, 11, 7,
|
||||
};
|
||||
@ -531,6 +532,7 @@ static BROTLI_INLINE void StoreStaticDistanceHuffmanTree(
|
||||
size_t* storage_ix, uint8_t* storage) {
|
||||
BrotliWriteBits(28, 0x0369DC03u, storage_ix, storage);
|
||||
}
|
||||
/* GENERATED CODE END */
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
|
@ -48,7 +48,7 @@ static void FN(Initialize)(HasherCommon* common,
|
||||
self->hb_common = *self->common;
|
||||
self->fresh = BROTLI_TRUE;
|
||||
self->params = params;
|
||||
/* TODO: Initialize of the hashers is deferred to Prepare (and params
|
||||
/* TODO(lode): Initialize of the hashers is deferred to Prepare (and params
|
||||
remembered here) because we don't get the one_shot and input_size params
|
||||
here that are needed to know the memory size of them. Instead provide
|
||||
those params to all hashers FN(Initialize) */
|
||||
|
@ -63,7 +63,7 @@ void BrotliBuildHistogramsWithContext(
|
||||
BlockSplitIteratorNext(&insert_and_copy_it);
|
||||
HistogramAddCommand(&insert_and_copy_histograms[insert_and_copy_it.type_],
|
||||
cmd->cmd_prefix_);
|
||||
/* TODO: unwrap iterator blocks. */
|
||||
/* TODO(eustas): unwrap iterator blocks. */
|
||||
for (j = cmd->insert_len_; j != 0; --j) {
|
||||
size_t context;
|
||||
BlockSplitIteratorNext(&literal_it);
|
||||
|
@ -40,7 +40,7 @@ typedef struct BrotliEncoderParams {
|
||||
BROTLI_BOOL large_window;
|
||||
BrotliHasherParams hasher;
|
||||
BrotliDistanceParams dist;
|
||||
/* TODO: rename to BrotliShared... */
|
||||
/* TODO(eustas): rename to BrotliShared... */
|
||||
SharedEncoderDictionary dictionary;
|
||||
} BrotliEncoderParams;
|
||||
|
||||
|
@ -527,7 +527,7 @@ BROTLI_BOOL BrotliFindAllStaticDictionaryMatches(
|
||||
uint32_t skipdist = (uint32_t)((uint32_t)(1 << dictionary->words->
|
||||
size_bits_by_length[len_code]) & ~1u) *
|
||||
(uint32_t)dictionary->num_transforms;
|
||||
/* TODO: check for dist overflow */
|
||||
/* TODO(lode): check for dist overflow */
|
||||
dist += skipdist;
|
||||
AddMatch(dist, (size_t)l, len_code, matches);
|
||||
}
|
||||
|
@ -22,6 +22,7 @@ typedef struct DictWord {
|
||||
uint16_t idx;
|
||||
} DictWord;
|
||||
|
||||
/* GENERATED CODE START */
|
||||
static const int kDictNumBits = 15;
|
||||
static const uint32_t kDictHashMul32 = 0x1E35A7BD;
|
||||
|
||||
@ -5856,6 +5857,7 @@ static const DictWord kStaticDictionaryWords[31705] = {
|
||||
,0,1735},{5,0,598},{7,0,791},{8,0,108},{9,0,123},{7,10,1570},{140,10,542},{142,
|
||||
11,410},{9,11,660},{138,11,347}
|
||||
};
|
||||
/* GENERATED CODE END */
|
||||
|
||||
#if defined(__cplusplus) || defined(c_plusplus)
|
||||
} /* extern "C" */
|
||||
|
@ -479,6 +479,13 @@ BROTLI_ENC_API BROTLI_BOOL BrotliEncoderHasMoreOutput(
|
||||
BROTLI_ENC_API const uint8_t* BrotliEncoderTakeOutput(
|
||||
BrotliEncoderState* state, size_t* size);
|
||||
|
||||
/* Returns the estimated peak memory usage (in bytes) of the BrotliCompress()
|
||||
function, not counting the memory needed for the input and output. */
|
||||
BROTLI_ENC_EXTRA_API size_t BrotliEncoderEstimatePeakMemoryUsage(
|
||||
int quality, int lgwin, size_t input_size);
|
||||
/* Returns 0 if dictionary is not valid; otherwise returns allocation size. */
|
||||
BROTLI_ENC_EXTRA_API size_t BrotliEncoderGetPreparedDictionarySize(
|
||||
const BrotliEncoderPreparedDictionary* dictionary);
|
||||
|
||||
/**
|
||||
* Gets an encoder library version.
|
||||
|
@ -247,6 +247,24 @@
|
||||
#define BROTLI_PUBLIC
|
||||
#endif
|
||||
|
||||
/* BROTLI_INTERNAL could be defined to override visibility, e.g. for tests. */
|
||||
#if !defined(BROTLI_INTERNAL)
|
||||
#if defined(_WIN32) || defined(__CYGWIN__)
|
||||
#define BROTLI_INTERNAL
|
||||
#elif BROTLI_GNUC_VERSION_CHECK(3, 3, 0) || \
|
||||
BROTLI_TI_VERSION_CHECK(8, 0, 0) || \
|
||||
BROTLI_INTEL_VERSION_CHECK(16, 0, 0) || \
|
||||
BROTLI_ARM_VERSION_CHECK(4, 1, 0) || \
|
||||
BROTLI_IBM_VERSION_CHECK(13, 1, 0) || \
|
||||
BROTLI_SUNPRO_VERSION_CHECK(5, 11, 0) || \
|
||||
(BROTLI_TI_VERSION_CHECK(7, 3, 0) && \
|
||||
defined(__TI_GNU_ATTRIBUTE_SUPPORT__) && defined(__TI_EABI__))
|
||||
#define BROTLI_INTERNAL __attribute__ ((visibility ("hidden")))
|
||||
#else
|
||||
#define BROTLI_INTERNAL
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \
|
||||
!defined(__STDC_NO_VLA__) && !defined(__cplusplus) && \
|
||||
!defined(__PGI) && !defined(__PGIC__) && !defined(__TINYC__) && \
|
||||
@ -286,4 +304,10 @@
|
||||
#define BROTLI_ENC_API
|
||||
#endif
|
||||
|
||||
#if defined(BROTLI_BUILD_ENC_EXTRA_API)
|
||||
#define BROTLI_ENC_EXTRA_API BROTLI_ENC_API
|
||||
#else
|
||||
#define BROTLI_ENC_EXTRA_API BROTLI_INTERNAL
|
||||
#endif
|
||||
|
||||
#endif /* BROTLI_COMMON_PORT_H_ */
|
||||
|
@ -664,7 +664,7 @@ static int64_t FileSize(const char* path) {
|
||||
}
|
||||
|
||||
/* Copy file times and permissions.
|
||||
TODO: this is a "best effort" implementation; honest cross-platform
|
||||
TODO(eustas): this is a "best effort" implementation; honest cross-platform
|
||||
fully featured implementation is way too hacky; add more hacks by request. */
|
||||
static void CopyStat(const char* input_path, const char* output_path) {
|
||||
struct stat statbuf;
|
||||
|
@ -108,7 +108,7 @@ func (w *Writer) writeChunk(p []byte, op C.BrotliEncoderOperation) (n int, err e
|
||||
if length != 0 {
|
||||
// It is a workaround for non-copying-wrapping of native memory.
|
||||
// C-encoder never pushes output block longer than ((2 << 25) + 502).
|
||||
// TODO: use natural wrapper, when it becomes available, see
|
||||
// TODO(eustas): use natural wrapper, when it becomes available, see
|
||||
// https://golang.org/issue/13656.
|
||||
output := (*[1 << 30]byte)(unsafe.Pointer(result.output_data))[:length:length]
|
||||
_, err = w.dst.Write(output)
|
||||
|
@ -88,7 +88,7 @@ final class Decode {
|
||||
0x020000, 0x020004, 0x020003, 0x030002, 0x020000, 0x020004, 0x020003, 0x040005
|
||||
};
|
||||
|
||||
// TODO: generalize.
|
||||
// TODO(eustas): generalize.
|
||||
static final int MAX_TRANSFORMED_WORD_LENGTH = 5 + 24 + 8;
|
||||
|
||||
private static final int MAX_DISTANCE_BITS = 24;
|
||||
@ -152,7 +152,7 @@ final class Decode {
|
||||
return NUM_DISTANCE_SHORT_CODES + ndirect + 2 * (maxndistbits << npostfix);
|
||||
}
|
||||
|
||||
// TODO: add a correctness test for this function when
|
||||
// TODO(eustas): add a correctness test for this function when
|
||||
// large-window and dictionary are implemented.
|
||||
private static int calculateDistanceAlphabetLimit(int maxDistance, int npostfix, int ndirect) {
|
||||
if (maxDistance < ndirect + (2 << npostfix)) {
|
||||
@ -265,7 +265,7 @@ final class Decode {
|
||||
s.isLargeWindow = 1;
|
||||
}
|
||||
|
||||
// TODO: do we need byte views?
|
||||
// TODO(eustas): do we need byte views?
|
||||
static void attachDictionaryChunk(State s, byte[] data) {
|
||||
if (s.runningState != INITIALIZED) {
|
||||
throw new IllegalStateException("State MUST be freshly initialized");
|
||||
@ -485,7 +485,7 @@ final class Decode {
|
||||
if (space != 0) {
|
||||
throw new BrotliRuntimeException("Unused space"); // COV_NF_LINE
|
||||
}
|
||||
// TODO: Pass max_symbol to Huffman table builder instead?
|
||||
// TODO(eustas): Pass max_symbol to Huffman table builder instead?
|
||||
Utils.fillIntsWithZeroes(codeLengths, symbol, numSymbols);
|
||||
}
|
||||
|
||||
@ -504,7 +504,7 @@ final class Decode {
|
||||
*/
|
||||
private static int readSimpleHuffmanCode(int alphabetSizeMax, int alphabetSizeLimit,
|
||||
int[] tableGroup, int tableIdx, State s) {
|
||||
// TODO: Avoid allocation?
|
||||
// TODO(eustas): Avoid allocation?
|
||||
int[] codeLengths = new int[alphabetSizeLimit];
|
||||
int[] symbols = new int[4];
|
||||
|
||||
@ -560,7 +560,7 @@ final class Decode {
|
||||
break;
|
||||
}
|
||||
|
||||
// TODO: Use specialized version?
|
||||
// TODO(eustas): Use specialized version?
|
||||
return Huffman.buildHuffmanTable(
|
||||
tableGroup, tableIdx, HUFFMAN_TABLE_BITS, codeLengths, alphabetSizeLimit);
|
||||
}
|
||||
@ -568,7 +568,7 @@ final class Decode {
|
||||
// Decode Huffman-coded code lengths.
|
||||
private static int readComplexHuffmanCode(int alphabetSizeLimit, int skip,
|
||||
int[] tableGroup, int tableIdx, State s) {
|
||||
// TODO: Avoid allocation?
|
||||
// TODO(eustas): Avoid allocation?
|
||||
int[] codeLengths = new int[alphabetSizeLimit];
|
||||
int[] codeLengthCodeLengths = new int[CODE_LENGTH_CODES];
|
||||
int space = 32;
|
||||
@ -577,7 +577,7 @@ final class Decode {
|
||||
int codeLenIdx = CODE_LENGTH_CODE_ORDER[i];
|
||||
BitReader.fillBitWindow(s);
|
||||
int p = BitReader.peekBits(s) & 15;
|
||||
// TODO: Demultiplex FIXED_TABLE.
|
||||
// TODO(eustas): Demultiplex FIXED_TABLE.
|
||||
s.bitOffset += FIXED_TABLE[p] >> 16;
|
||||
int v = FIXED_TABLE[p] & 0xFFFF;
|
||||
codeLengthCodeLengths[codeLenIdx] = v;
|
||||
@ -709,7 +709,7 @@ final class Decode {
|
||||
private static void maybeReallocateRingBuffer(State s) {
|
||||
int newSize = s.maxRingBufferSize;
|
||||
if (newSize > s.expectedTotalSize) {
|
||||
/* TODO: Handle 2GB+ cases more gracefully. */
|
||||
/* TODO(eustas): Handle 2GB+ cases more gracefully. */
|
||||
int minimalNewSize = s.expectedTotalSize;
|
||||
while ((newSize >> 1) > minimalNewSize) {
|
||||
newSize >>= 1;
|
||||
@ -736,7 +736,7 @@ final class Decode {
|
||||
s.runningState = INIT_WRITE;
|
||||
return;
|
||||
}
|
||||
// TODO: Reset? Do we need this?
|
||||
// TODO(eustas): Reset? Do we need this?
|
||||
s.literalTreeGroup = new int[0];
|
||||
s.commandTreeGroup = new int[0];
|
||||
s.distanceTreeGroup = new int[0];
|
||||
@ -831,7 +831,7 @@ final class Decode {
|
||||
BitReader.fillBitWindow(s);
|
||||
s.distancePostfixBits = BitReader.readFewBits(s, 2);
|
||||
s.numDirectDistanceCodes = BitReader.readFewBits(s, 4) << s.distancePostfixBits;
|
||||
// TODO: Reuse?
|
||||
// TODO(eustas): Reuse?
|
||||
s.contextModes = new byte[s.numLiteralBlockTypes];
|
||||
for (int i = 0; i < s.numLiteralBlockTypes;) {
|
||||
/* Ensure that less than 256 bits read between readMoreInput. */
|
||||
@ -843,7 +843,7 @@ final class Decode {
|
||||
BitReader.readMoreInput(s);
|
||||
}
|
||||
|
||||
// TODO: Reuse?
|
||||
// TODO(eustas): Reuse?
|
||||
s.contextMap = new byte[s.numLiteralBlockTypes << LITERAL_CONTEXT_BITS];
|
||||
int numLiteralTrees = decodeContextMap(s.numLiteralBlockTypes << LITERAL_CONTEXT_BITS,
|
||||
s.contextMap, s);
|
||||
@ -855,7 +855,7 @@ final class Decode {
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Reuse?
|
||||
// TODO(eustas): Reuse?
|
||||
s.distContextMap = new byte[s.numDistanceBlockTypes << DISTANCE_CONTEXT_BITS];
|
||||
int numDistTrees = decodeContextMap(s.numDistanceBlockTypes << DISTANCE_CONTEXT_BITS,
|
||||
s.distContextMap, s);
|
||||
@ -1091,7 +1091,7 @@ final class Decode {
|
||||
byte[] ringBuffer = s.ringBuffer;
|
||||
|
||||
while (s.runningState != FINISHED) {
|
||||
// TODO: extract cases to methods for the better readability.
|
||||
// TODO(eustas): extract cases to methods for the better readability.
|
||||
switch (s.runningState) {
|
||||
case BLOCK_START:
|
||||
if (s.metaBlockLength < 0) {
|
||||
|
@ -66,7 +66,7 @@ final class Huffman {
|
||||
int tableOffset = tableGroup[tableIdx];
|
||||
int key; // Reversed prefix code.
|
||||
int[] sorted = new int[codeLengthsSize]; // Symbols sorted by code length.
|
||||
// TODO: fill with zeroes?
|
||||
// TODO(eustas): fill with zeroes?
|
||||
int[] count = new int[MAX_LENGTH + 1]; // Number of codes of each length.
|
||||
int[] offset = new int[MAX_LENGTH + 1]; // Offsets in sorted table for each length.
|
||||
int symbol;
|
||||
|
@ -11,7 +11,8 @@ def brotli_java_test(name, main_class = None, jvm_flags = None, **kwargs):
|
||||
jvm_flags = []
|
||||
jvm_flags = jvm_flags + _TEST_JVM_FLAGS
|
||||
|
||||
test_package = native.package_name().replace("/", ".")
|
||||
test_package = native.package_name().replace("/", ".").replace("javatests.", "")
|
||||
|
||||
if main_class == None:
|
||||
test_class = test_package + "." + name
|
||||
else:
|
||||
|
@ -73,7 +73,7 @@ public class PreparedDictionaryGenerator {
|
||||
accumulator |= (src.get(i) & 0xFFL) << (8 * i);
|
||||
}
|
||||
accumulator <<= 8;
|
||||
/* TODO: apply custom "store" order. */
|
||||
/* TODO(eustas): apply custom "store" order. */
|
||||
for (int i = 0; i + 7 < sourceSize; ++i) {
|
||||
accumulator = (accumulator >>> 8) | ((src.get(i + 7) & 0xFFL) << 56);
|
||||
long h = (accumulator & hashMask) * HASH_MULTIPLIER;
|
||||
|
@ -7,7 +7,7 @@ filegroup(
|
||||
srcs = [
|
||||
"decoder_jni.cc",
|
||||
"decoder_jni.h",
|
||||
# TODO: investigate, why this prevents JNI library loading.
|
||||
# TODO(eustas): investigate, why this prevents JNI library loading.
|
||||
#"decoder_jni_onload.cc",
|
||||
],
|
||||
)
|
||||
|
@ -126,7 +126,7 @@ public class DecoderJNI {
|
||||
@Override
|
||||
protected void finalize() throws Throwable {
|
||||
if (context[0] != 0) {
|
||||
/* TODO: log resource leak? */
|
||||
/* TODO(eustas): log resource leak? */
|
||||
destroy();
|
||||
}
|
||||
super.finalize();
|
||||
|
@ -79,7 +79,7 @@ Java_org_brotli_wrapper_dec_DecoderJNI_nativeCreate(
|
||||
}
|
||||
|
||||
if (ok) {
|
||||
/* TODO: future versions (e.g. when 128-bit architecture comes)
|
||||
/* TODO(eustas): future versions (e.g. when 128-bit architecture comes)
|
||||
might require thread-safe cookie<->handle mapping. */
|
||||
context[0] = reinterpret_cast<jlong>(handle);
|
||||
} else if (!!handle) {
|
||||
|
@ -169,7 +169,7 @@ class EncoderJNI {
|
||||
@Override
|
||||
protected void finalize() throws Throwable {
|
||||
if (context[0] != 0) {
|
||||
/* TODO: log resource leak? */
|
||||
/* TODO(eustas): log resource leak? */
|
||||
destroy();
|
||||
}
|
||||
super.finalize();
|
||||
|
@ -93,7 +93,7 @@ Java_org_brotli_wrapper_enc_EncoderJNI_nativeCreate(
|
||||
}
|
||||
|
||||
if (ok) {
|
||||
/* TODO: future versions (e.g. when 128-bit architecture comes)
|
||||
/* TODO(eustas): future versions (e.g. when 128-bit architecture comes)
|
||||
might require thread-safe cookie<->handle mapping. */
|
||||
context[0] = reinterpret_cast<jlong>(handle);
|
||||
} else if (!!handle) {
|
||||
|
@ -10,12 +10,16 @@ import unittest
|
||||
|
||||
|
||||
project_dir = os.path.abspath(os.path.join(__file__, '..', '..', '..'))
|
||||
src_dir = os.path.join(project_dir, 'python')
|
||||
test_dir = os.path.join(project_dir, 'tests')
|
||||
test_dir = os.getenv("BROTLI_TESTS_PATH")
|
||||
BRO_ARGS = [os.getenv("BROTLI_WRAPPER")]
|
||||
|
||||
python_exe = sys.executable or 'python'
|
||||
bro_path = os.path.join(src_dir, 'bro.py')
|
||||
BRO_ARGS = [python_exe, bro_path]
|
||||
# Fallbacks
|
||||
if test_dir is None:
|
||||
test_dir = os.path.join(project_dir, 'tests')
|
||||
if BRO_ARGS[0] is None:
|
||||
python_exe = sys.executable or 'python'
|
||||
bro_path = os.path.join(project_dir, 'python', 'bro.py')
|
||||
BRO_ARGS = [python_exe, bro_path]
|
||||
|
||||
# Get the platform/version-specific build folder.
|
||||
# By default, the distutils build base is in the same location as setup.py.
|
||||
|
@ -97,7 +97,7 @@ static void buildFullText(std::vector<std::vector<TextChar>>* data,
|
||||
}
|
||||
|
||||
/* Build longest-common-prefix based on suffix array and text.
|
||||
TODO: borrowed -> unknown efficiency. */
|
||||
TODO(eustas): borrowed -> unknown efficiency. */
|
||||
static void buildLcp(std::vector<TextChar>* text, std::vector<TextIdx>* sa,
|
||||
std::vector<TextIdx>* lcp, std::vector<TextIdx>* invese_sa) {
|
||||
TextIdx size = static_cast<TextIdx>(text->size());
|
||||
|
@ -219,7 +219,7 @@ int main(int argc, char const* argv[]) {
|
||||
}
|
||||
} else if (arg1 == 's') {
|
||||
sliceLen = readInt(arg2);
|
||||
// TODO: investigate why sliceLen == 4..5 greatly slows down
|
||||
// TODO(eustas): investigate why sliceLen == 4..5 greatly slows down
|
||||
// durschlag engine, but only from command line;
|
||||
// durschlag_runner seems to work fine with those.
|
||||
if (sliceLen < 4 || sliceLen > 256) {
|
||||
|
@ -401,7 +401,7 @@ DurchschlagIndex durchschlag_index(const std::vector<uint8_t>& data) {
|
||||
std::vector<TextIdx> isa(total);
|
||||
for (TextIdx i = 0; i < total; ++i) isa[sa[i]] = i;
|
||||
|
||||
// TODO: borrowed -> unknown efficiency.
|
||||
// TODO(eustas): borrowed -> unknown efficiency.
|
||||
std::vector<TextIdx> lcp(total);
|
||||
TextIdx k = 0;
|
||||
lcp[total - 1] = 0;
|
||||
|
Loading…
Reference in New Issue
Block a user