Prepare for copybara (#939)

Co-authored-by: Eugene Kliuchnikov <eustas@chromium.org>

c/common/dictionary.c

@@ -13,6 +13,7 @@ extern "C" {
 
 #if !defined(BROTLI_EXTERNAL_DICTIONARY_DATA)
 static const uint8_t kBrotliDictionaryData[] =
+/* GENERATED CODE START */
 {
 116,105,109,101,100,111,119,110,108,105,102,101,108,101,102,116,98,97,99,107,99,
 111,100,101,100,97,116,97,115,104,111,119,111,110,108,121,115,105,116,101,99,105
@@ -5860,6 +5861,7 @@ static const uint8_t kBrotliDictionaryData[] =
 ,164,181,224,164,190,224,164,136,224,164,184,224,164,149,224,165,141,224,164,176
 ,224,164,191,224,164,175,224,164,164,224,164,190
 }
+/* GENERATED CODE END */
 ;
 #endif  /* !BROTLI_EXTERNAL_DICTIONARY_DATA */
 

c/common/platform.h

@@ -156,24 +156,6 @@ OR:
 #define BROTLI_NOINLINE
 #endif
 
-/* BROTLI_INTERNAL could be defined to override visibility, e.g. for tests. */
-#if !defined(BROTLI_INTERNAL)
-#if defined(_WIN32) || defined(__CYGWIN__)
-#define BROTLI_INTERNAL
-#elif BROTLI_GNUC_VERSION_CHECK(3, 3, 0) ||                         \
-    BROTLI_TI_VERSION_CHECK(8, 0, 0) ||                             \
-    BROTLI_INTEL_VERSION_CHECK(16, 0, 0) ||                         \
-    BROTLI_ARM_VERSION_CHECK(4, 1, 0) ||                            \
-    BROTLI_IBM_VERSION_CHECK(13, 1, 0) ||                           \
-    BROTLI_SUNPRO_VERSION_CHECK(5, 11, 0) ||                        \
-    (BROTLI_TI_VERSION_CHECK(7, 3, 0) &&                            \
-     defined(__TI_GNU_ATTRIBUTE_SUPPORT__) && defined(__TI_EABI__))
-#define BROTLI_INTERNAL __attribute__ ((visibility ("hidden")))
-#else
-#define BROTLI_INTERNAL
-#endif
-#endif
-
 /* <<< <<< <<< end of hedley macros. */
 
 #if BROTLI_GNUC_HAS_ATTRIBUTE(unused, 2, 7, 0) || \
@@ -485,11 +467,11 @@ static BROTLI_INLINE void BrotliDump(const char* f, int l, const char* fn) {
 #define BROTLI_DUMP() (void)(0)
 #endif
 
-/* TODO: add appropriate icc/sunpro/arm/ibm/ti checks. */
+/* TODO(eustas): add appropriate icc/sunpro/arm/ibm/ti checks. */
 #if (BROTLI_GNUC_VERSION_CHECK(3, 0, 0) || defined(__llvm__)) && \
     !defined(BROTLI_BUILD_NO_RBIT)
 #if defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8_ANY)
-/* TODO: detect ARMv6T2 and enable this code for it. */
+/* TODO(eustas): detect ARMv6T2 and enable this code for it. */
 static BROTLI_INLINE brotli_reg_t BrotliRBit(brotli_reg_t input) {
   brotli_reg_t output;
   __asm__("rbit %0, %1\n" : "=r"(output) : "r"(input));

c/common/shared_dictionary.c

@@ -416,7 +416,7 @@ static BROTLI_BOOL ParseDictionary(const uint8_t* encoded, size_t size,
    BrotliSharedDictionary already contains data, compound dictionaries
    will be appended, but an error will be returned if it already has
    custom words or transforms.
-   TODO: link to RFC for shared brotli once published. */
+   TODO(lode): link to RFC for shared brotli once published. */
 static BROTLI_BOOL DecodeSharedDictionary(
     const uint8_t* encoded, size_t size, BrotliSharedDictionary* dict) {
   uint32_t num_prefix = 0;
@@ -492,7 +492,7 @@ BrotliSharedDictionary* BrotliSharedDictionaryCreateInstance(
     return 0;
   }
 
-  /* TODO: explicitly initialize all the fields? */
+  /* TODO(eustas): explicitly initialize all the fields? */
   memset(dict, 0, sizeof(BrotliSharedDictionary));
 
   dict->context_based = BROTLI_FALSE;

c/dec/decode.c

@@ -218,7 +218,7 @@ static BROTLI_NOINLINE BrotliDecoderErrorCode DecodeVarLenUint8(
 
     default:
       return
-          BROTLI_FAILURE(BROTLI_DECODER_ERROR_UNREACHABLE);
+          BROTLI_FAILURE(BROTLI_DECODER_ERROR_UNREACHABLE);  /* COV_NF_LINE */
   }
 }
 
@@ -339,7 +339,7 @@ static BrotliDecoderErrorCode BROTLI_NOINLINE DecodeMetaBlockLength(
 
       default:
         return
-            BROTLI_FAILURE(BROTLI_DECODER_ERROR_UNREACHABLE);
+            BROTLI_FAILURE(BROTLI_DECODER_ERROR_UNREACHABLE);  /* COV_NF_LINE */
     }
   }
 }
@@ -865,7 +865,7 @@ static BrotliDecoderErrorCode ReadHuffmanCode(uint32_t alphabet_size_max,
 
       default:
         return
-            BROTLI_FAILURE(BROTLI_DECODER_ERROR_UNREACHABLE);
+            BROTLI_FAILURE(BROTLI_DECODER_ERROR_UNREACHABLE);  /* COV_NF_LINE */
     }
   }
 }
@@ -1112,7 +1112,7 @@ static BrotliDecoderErrorCode DecodeContextMap(uint32_t context_map_size,
 
     default:
       return
-          BROTLI_FAILURE(BROTLI_DECODER_ERROR_UNREACHABLE);
+          BROTLI_FAILURE(BROTLI_DECODER_ERROR_UNREACHABLE);  /* COV_NF_LINE */
   }
 }
 
@@ -1356,7 +1356,7 @@ static BROTLI_BOOL BROTLI_NOINLINE BrotliEnsureRingBuffer(
 static BrotliDecoderErrorCode BROTLI_NOINLINE CopyUncompressedBlockToOutput(
     size_t* available_out, uint8_t** next_out, size_t* total_out,
     BrotliDecoderState* s) {
-  /* TODO: avoid allocation for single uncompressed block. */
+  /* TODO(eustas): avoid allocation for single uncompressed block. */
   if (!BrotliEnsureRingBuffer(s)) {
     return BROTLI_FAILURE(BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_1);
   }
@@ -1866,7 +1866,7 @@ static BROTLI_INLINE BrotliDecoderErrorCode ProcessCommandsInternal(
   } else if (s->state == BROTLI_STATE_COMMAND_POST_WRAP_COPY) {
     goto CommandPostWrapCopy;
   } else {
-    return BROTLI_FAILURE(BROTLI_DECODER_ERROR_UNREACHABLE);
+    return BROTLI_FAILURE(BROTLI_DECODER_ERROR_UNREACHABLE);  /* COV_NF_LINE */
   }
 
 CommandBegin:
@@ -2600,7 +2600,7 @@ BrotliDecoderResult BrotliDecoderDecompressStream(
           case 1: hgroup = &s->insert_copy_hgroup; break;
           case 2: hgroup = &s->distance_hgroup; break;
           default: return SaveErrorCode(s, BROTLI_FAILURE(
-              BROTLI_DECODER_ERROR_UNREACHABLE));
+              BROTLI_DECODER_ERROR_UNREACHABLE));  /* COV_NF_LINE */
         }
         result = HuffmanTreeGroupDecode(hgroup, s);
         if (result != BROTLI_DECODER_SUCCESS) break;
@@ -2779,6 +2779,23 @@ uint32_t BrotliDecoderVersion() {
   return BROTLI_VERSION;
 }
 
+/* Escalate internal functions visibility; for testing purposes only. */
+#if defined(BROTLI_TEST)
+BROTLI_BOOL SafeReadSymbolForTest(
+    const HuffmanCode*, BrotliBitReader*, uint32_t*);
+BROTLI_BOOL SafeReadSymbolForTest(
+    const HuffmanCode* table, BrotliBitReader* br, uint32_t* result) {
+  return SafeReadSymbol(table, br, result);
+}
+
+void InverseMoveToFrontTransformForTest(
+    uint8_t*, uint32_t, BrotliDecoderState*);
+void InverseMoveToFrontTransformForTest(
+    uint8_t* v, uint32_t l, BrotliDecoderState* s) {
+  InverseMoveToFrontTransform(v, l, s);
+}
+#endif
+
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */
 #endif

c/enc/block_splitter.c

@@ -170,7 +170,7 @@ void BrotliSplitBlock(MemoryManager* m,
         kCommandStrideLength, kCommandBlockSwitchCost, params,
         insert_and_copy_split);
     if (BROTLI_IS_OOM(m)) return;
-    /* TODO: reuse for distances? */
+    /* TODO(eustas): reuse for distances? */
     BROTLI_FREE(m, insert_and_copy_codes);
   }
 
@@ -197,6 +197,20 @@ void BrotliSplitBlock(MemoryManager* m,
   }
 }
 
+#if defined(BROTLI_TEST)
+size_t CountLiteralsForTest(const Command*, const size_t);
+size_t CountLiteralsForTest(const Command* cmds, const size_t num_commands) {
+  return CountLiterals(cmds, num_commands);
+}
+
+void CopyLiteralsToByteArrayForTest(const Command*,
+    const size_t, const uint8_t*, const size_t, const size_t, uint8_t*);
+void CopyLiteralsToByteArrayForTest(const Command* cmds,
+    const size_t num_commands, const uint8_t* data, const size_t offset,
+    const size_t mask, uint8_t* literals) {
+  CopyLiteralsToByteArray(cmds, num_commands, data, offset, mask, literals);
+}
+#endif
 
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */

c/enc/block_splitter_inc.h

@@ -234,7 +234,7 @@ static void FN(ClusterBlocks)(MemoryManager* m,
   uint32_t* BROTLI_RESTRICT const remap = u32 + 3 * HISTOGRAMS_PER_BATCH;
   uint32_t* BROTLI_RESTRICT const block_lengths =
       u32 + 4 * HISTOGRAMS_PER_BATCH;
-  /* TODO: move to arena? */
+  /* TODO(eustas): move to arena? */
   HistogramType* tmp = BROTLI_ALLOC(m, HistogramType, 2);
 
   if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(histogram_symbols) ||
@@ -335,7 +335,7 @@ static void FN(ClusterBlocks)(MemoryManager* m,
         FN(HistogramAdd)(tmp, data[pos++]);
       }
       /* Among equally good histograms prefer last used. */
-      /* TODO: should we give a block-switch discount here? */
+      /* TODO(eustas): should we give a block-switch discount here? */
       best_out = (i == 0) ? histogram_symbols[0] : histogram_symbols[i - 1];
       best_bits = FN(BrotliHistogramBitCostDistance)(
           tmp, &all_histograms[best_out], tmp + 1);

c/enc/brotli_bit_stream.c

@@ -286,7 +286,7 @@ void BrotliStoreHuffmanTree(const uint8_t* depths, size_t num,
   /* Write the Huffman tree into the brotli-representation.
      The command alphabet is the largest, so this allocation will fit all
      alphabets. */
-  /* TODO: fix me */
+  /* TODO(eustas): fix me */
   uint8_t huffman_tree[BROTLI_NUM_COMMAND_SYMBOLS];
   uint8_t huffman_tree_extra_bits[BROTLI_NUM_COMMAND_SYMBOLS];
   size_t huffman_tree_size = 0;
@@ -778,7 +778,7 @@ static void BuildAndStoreBlockSplitCode(const uint8_t* types,
     ++length_histo[BlockLengthPrefixCode(lengths[i])];
   }
   StoreVarLenUint8(num_types - 1, storage_ix, storage);
-  if (num_types > 1) {  /* TODO: else? could StoreBlockSwitch occur? */
+  if (num_types > 1) {  /* TODO(eustas): else? could StoreBlockSwitch occur? */
     BuildAndStoreHuffmanTree(&type_histo[0], num_types + 2, num_types + 2, tree,
                              &code->type_depths[0], &code->type_bits[0],
                              storage_ix, storage);
@@ -1150,12 +1150,12 @@ static void StoreDataWithHuffmanCodes(const uint8_t* input,
   }
 }
 
-/* TODO: pull alloc/dealloc to caller? */
+/* TODO(eustas): pull alloc/dealloc to caller? */
 typedef struct MetablockArena {
   HistogramLiteral lit_histo;
   HistogramCommand cmd_histo;
   HistogramDistance dist_histo;
-  /* TODO: merge bits and depth? */
+  /* TODO(eustas): merge bits and depth? */
   uint8_t lit_depth[BROTLI_NUM_LITERAL_SYMBOLS];
   uint16_t lit_bits[BROTLI_NUM_LITERAL_SYMBOLS];
   uint8_t cmd_depth[BROTLI_NUM_COMMAND_SYMBOLS];
@@ -1323,6 +1323,13 @@ void BrotliStoreUncompressedMetaBlock(BROTLI_BOOL is_final_block,
   }
 }
 
+#if defined(BROTLI_TEST)
+void GetBlockLengthPrefixCodeForTest(uint32_t len, size_t* code,
+                                     uint32_t* n_extra, uint32_t* extra) {
+  GetBlockLengthPrefixCode(len, code, n_extra, extra);
+}
+#endif
+
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */
 #endif

c/enc/brotli_bit_stream.h

@@ -77,6 +77,10 @@ BROTLI_INTERNAL void BrotliStoreUncompressedMetaBlock(
     size_t position, size_t mask, size_t len,
     size_t* BROTLI_RESTRICT storage_ix, uint8_t* BROTLI_RESTRICT storage);
 
+#if defined(BROTLI_TEST)
+void GetBlockLengthPrefixCodeForTest(uint32_t, size_t*, uint32_t*, uint32_t*);
+#endif
+
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */
 #endif

c/enc/cluster_inc.h

@@ -228,7 +228,7 @@ BROTLI_INTERNAL size_t FN(BrotliHistogramReindex)(MemoryManager* m,
       ++next_index;
     }
   }
-  /* TODO: by using idea of "cycle-sort" we can avoid allocation of
+  /* TODO(eustas): by using idea of "cycle-sort" we can avoid allocation of
      tmp and reduce the number of copying by the factor of 2. */
   tmp = BROTLI_ALLOC(m, HistogramType, next_index);
   if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(tmp)) return 0;
@@ -259,7 +259,7 @@ BROTLI_INTERNAL void FN(BrotliClusterHistograms)(
   size_t pairs_capacity = max_input_histograms * max_input_histograms / 2;
   /* For the first pass of clustering, we allow all pairs. */
   HistogramPair* pairs = BROTLI_ALLOC(m, HistogramPair, pairs_capacity + 1);
-  /* TODO: move to "persistent" arena? */
+  /* TODO(eustas): move to "persistent" arena? */
   HistogramType* tmp = BROTLI_ALLOC(m, HistogramType, 1);
   size_t i;
 

c/enc/compound_dictionary.c

@@ -52,7 +52,7 @@ static PreparedDictionary* CreatePreparedDictionaryWithParams(MemoryManager* m,
   next_bucket = (uint32_t*)(&bucket_heads[num_buckets]);
   memset(num, 0, num_buckets * sizeof(num[0]));
 
-  /* TODO: apply custom "store" order. */
+  /* TODO(eustas): apply custom "store" order. */
   for (i = 0; i + 7 < source_size; ++i) {
     const uint64_t h = (BROTLI_UNALIGNED_LOAD64LE(&source[i]) & hash_mask) *
         kPreparedDictionaryHashMul64Long;

c/enc/compress_fragment.c

@@ -130,7 +130,7 @@ static void BuildAndStoreCommandPrefixCode(BrotliOnePassArena* s,
   uint16_t* const bits = s->cmd_bits;
   uint8_t* BROTLI_RESTRICT const tmp_depth = s->tmp_depth;
   uint16_t* BROTLI_RESTRICT const tmp_bits = s->tmp_bits;
-  /* TODO: do only once on initialization. */
+  /* TODO(eustas): do only once on initialization. */
   memset(tmp_depth, 0, BROTLI_NUM_COMMAND_SYMBOLS);
 
   BrotliCreateHuffmanTree(histogram, 64, 15, s->tree, depth);
@@ -168,7 +168,7 @@ static void BuildAndStoreCommandPrefixCode(BrotliOnePassArena* s,
       tmp_depth[256 + 8 * i] = depth[48 + i];
       tmp_depth[448 + 8 * i] = depth[56 + i];
     }
-    /* TODO: could/should full-length machinery be avoided? */
+    /* TODO(eustas): could/should full-length machinery be avoided? */
     BrotliStoreHuffmanTree(
         tmp_depth, BROTLI_NUM_COMMAND_SYMBOLS, s->tree, storage_ix, storage);
   }
@@ -567,6 +567,8 @@ trawl:
         int distance = (int)(base - candidate);  /* > 0 */
         size_t insert = (size_t)(base - next_emit);
         ip += matched;
+        BROTLI_LOG(("[CompressFragment] pos = %d insert = %lu copy = %d\n",
+                    (int)(next_emit - base_ip), (unsigned long)insert, 2));
         BROTLI_DCHECK(0 == memcmp(base, candidate, matched));
         if (BROTLI_PREDICT_TRUE(insert < 6210)) {
           EmitInsertLen(insert, cmd_depth, cmd_bits, cmd_histo,
@@ -595,6 +597,12 @@ trawl:
         }
         EmitCopyLenLastDistance(matched, cmd_depth, cmd_bits, cmd_histo,
                                 storage_ix, storage);
+        BROTLI_LOG(("[CompressFragment] pos = %d distance = %d\n"
+                    "[CompressFragment] pos = %d insert = %d copy = %d\n"
+                    "[CompressFragment] pos = %d distance = %d\n",
+                    (int)(base - base_ip), (int)distance,
+                    (int)(base - base_ip) + 2, 0, (int)matched - 2,
+                    (int)(base - base_ip) + 2, (int)distance));
 
         next_emit = ip;
         if (BROTLI_PREDICT_FALSE(ip >= ip_limit)) {
@@ -632,6 +640,10 @@ trawl:
                     storage_ix, storage);
         EmitDistance((size_t)last_distance, cmd_depth, cmd_bits,
                      cmd_histo, storage_ix, storage);
+        BROTLI_LOG(("[CompressFragment] pos = %d insert = %d copy = %d\n"
+                    "[CompressFragment] pos = %d distance = %d\n",
+                    (int)(base - base_ip), 0, (int)matched,
+                    (int)(base - base_ip), (int)last_distance));
 
         next_emit = ip;
         if (BROTLI_PREDICT_FALSE(ip >= ip_limit)) {
@@ -682,6 +694,8 @@ trawl:
   /* Emit the remaining bytes as literals. */
   if (next_emit < ip_end) {
     const size_t insert = (size_t)(ip_end - next_emit);
+    BROTLI_LOG(("[CompressFragment] pos = %d insert = %lu copy = %d\n",
+                (int)(next_emit - base_ip), (unsigned long)insert, 2));
     if (BROTLI_PREDICT_TRUE(insert < 6210)) {
       EmitInsertLen(insert, cmd_depth, cmd_bits, cmd_histo,
                     storage_ix, storage);

c/enc/compress_fragment_two_pass.c

@@ -69,7 +69,7 @@ static void BuildAndStoreCommandPrefixCode(BrotliTwoPassArena* s,
                                            size_t* storage_ix,
                                            uint8_t* storage) {
   /* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */
-  /* TODO: initialize once. */
+  /* TODO(eustas): initialize once. */
   memset(s->tmp_depth, 0, sizeof(s->tmp_depth));
   BrotliCreateHuffmanTree(s->cmd_histo, 64, 15, s->tmp_tree, s->cmd_depth);
   BrotliCreateHuffmanTree(&s->cmd_histo[64], 64, 14, s->tmp_tree,
@@ -329,6 +329,8 @@ trawl:
         ip += matched;
         BROTLI_DCHECK(0 == memcmp(base, candidate, matched));
         EmitInsertLen((uint32_t)insert, commands);
+        BROTLI_LOG(("[CompressFragment] pos = %d insert = %d copy = %d\n",
+                    (int)(next_emit - base_ip), insert, 2));
         memcpy(*literals, next_emit, (size_t)insert);
         *literals += insert;
         if (distance == last_distance) {
@@ -339,6 +341,12 @@ trawl:
           last_distance = distance;
         }
         EmitCopyLenLastDistance(matched, commands);
+        BROTLI_LOG(("[CompressFragment] pos = %d distance = %d\n"
+                    "[CompressFragment] pos = %d insert = %d copy = %d\n"
+                    "[CompressFragment] pos = %d distance = %d\n",
+                    (int)(base - base_ip), (int)distance,
+                    (int)(base - base_ip) + 2, 0, (int)matched - 2,
+                    (int)(base - base_ip) + 2, (int)distance));
 
         next_emit = ip;
         if (BROTLI_PREDICT_FALSE(ip >= ip_limit)) {
@@ -394,6 +402,10 @@ trawl:
         BROTLI_DCHECK(0 == memcmp(base, candidate, matched));
         EmitCopyLen(matched, commands);
         EmitDistance((uint32_t)last_distance, commands);
+        BROTLI_LOG(("[CompressFragment] pos = %d insert = %d copy = %d\n"
+                    "[CompressFragment] pos = %d distance = %d\n",
+                    (int)(base - base_ip), 0, (int)matched,
+                    (int)(base - base_ip), (int)last_distance));
 
         next_emit = ip;
         if (BROTLI_PREDICT_FALSE(ip >= ip_limit)) {
@@ -446,6 +458,8 @@ emit_remainder:
   if (next_emit < ip_end) {
     const uint32_t insert = (uint32_t)(ip_end - next_emit);
     EmitInsertLen(insert, commands);
+    BROTLI_LOG(("[CompressFragment] pos = %d insert = %d copy = %d\n",
+                (int)(next_emit - base_ip), insert, 2));
     memcpy(*literals, next_emit, insert);
     *literals += insert;
   }
@@ -472,9 +486,9 @@ static void StoreCommands(BrotliTwoPassArena* s,
 
   size_t i;
   memset(s->lit_histo, 0, sizeof(s->lit_histo));
-  /* TODO: is that necessary? */
+  /* TODO(eustas): is that necessary? */
   memset(s->cmd_depth, 0, sizeof(s->cmd_depth));
-  /* TODO: is that necessary? */
+  /* TODO(eustas): is that necessary? */
   memset(s->cmd_bits, 0, sizeof(s->cmd_bits));
   memset(s->cmd_histo, 0, sizeof(s->cmd_histo));
   for (i = 0; i < num_literals; ++i) {

c/enc/compress_fragment_two_pass.h

@@ -22,7 +22,7 @@
 extern "C" {
 #endif
 
-/* TODO: turn to macro. */
+/* TODO(eustas): turn to macro. */
 static const size_t kCompressFragmentTwoPassBlockSize = 1 << 17;
 
 typedef struct BrotliTwoPassArena {

c/enc/dictionary_hash.c

@@ -13,6 +13,7 @@
 extern "C" {
 #endif
 
+/* GENERATED CODE START */
 BROTLI_INTERNAL const uint16_t kStaticDictionaryHashWords[32768] = {
 1002,0,0,0,0,0,0,0,0,683,0,0,0,0,0,0,0,1265,0,0,0,0,0,1431,0,0,0,0,0,0,40,0,0,0,
 0,155,8,741,0,624,0,0,0,0,0,0,0,0,0,0,0,0,66,503,0,0,0,451,0,0,0,0,0,0,0,835,70,
@@ -1840,6 +1841,7 @@ BROTLI_INTERNAL const uint8_t kStaticDictionaryHashLengths[32768] = {
 0,0,0,0,6,0,0,0,0,0,0,0,0,0,0,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,0,0,0,0,0,0,0,0,
 10,7,0,0,0,0,0,0,0,0,9,0,0,0,0,4,0,0,0,0,0,0,0,0,0,5,11,0,0,0,0,0,0,0,8,6,0,0,9,
 7,0,0,12,4,0,0,0,0,0,0,12,6,0,6,0,7,0,0,8,5,0,0,0,0};
+/* GENERATED CODE END */
 
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */

c/enc/encode.c

@@ -138,7 +138,7 @@ BROTLI_BOOL BrotliEncoderSetParameter(
     BrotliEncoderState* state, BrotliEncoderParameter p, uint32_t value) {
   /* Changing parameters on the fly is not implemented yet. */
   if (state->is_initialized_) return BROTLI_FALSE;
-  /* TODO: Validate/clamp parameters here. */
+  /* TODO(eustas): Validate/clamp parameters here. */
   switch (p) {
     case BROTLI_PARAM_MODE:
       state->params.mode = (BrotliEncoderMode)value;
@@ -274,7 +274,7 @@ static void EncodeWindowBits(int lgwin, BROTLI_BOOL large_window,
   }
 }
 
-/* TODO: move to compress_fragment.c? */
+/* TODO(eustas): move to compress_fragment.c? */
 /* Initializes the command and distance prefix codes for the first block. */
 static void InitCommandPrefixCodes(BrotliOnePassArena* s) {
   static const uint8_t kDefaultCommandDepths[128] = {
@@ -506,7 +506,7 @@ static void DecideOverLiteralContextModeling(const uint8_t* input,
 static BROTLI_BOOL ShouldCompress(
     const uint8_t* data, const size_t mask, const uint64_t last_flush_pos,
     const size_t bytes, const size_t num_literals, const size_t num_commands) {
-  /* TODO: find more precise minimal block overhead. */
+  /* TODO(eustas): find more precise minimal block overhead. */
   if (bytes <= 2) return BROTLI_FALSE;
   if (num_commands < (bytes >> 8) + 2) {
     if ((double)num_literals > 0.99 * (double)bytes) {
@@ -606,7 +606,7 @@ static void WriteMetaBlockInternal(MemoryManager* m,
       size_t num_literal_contexts = 1;
       const uint32_t* literal_context_map = NULL;
       if (!params->disable_literal_context_modeling) {
-        /* TODO: pull to higher level and reuse. */
+        /* TODO(eustas): pull to higher level and reuse. */
         uint32_t* arena = BROTLI_ALLOC(m, uint32_t, 14 * 32);
         if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(arena)) return;
         DecideOverLiteralContextModeling(
@@ -1136,7 +1136,7 @@ static BROTLI_BOOL EncodeData(
     const size_t max_commands = max_length / 8;
     const size_t processed_bytes = (size_t)(s->input_pos_ - s->last_flush_pos_);
     /* If maximal possible additional block doesn't fit metablock, flush now. */
-    /* TODO: Postpone decision until next block arrives? */
+    /* TODO(eustas): Postpone decision until next block arrives? */
     const BROTLI_BOOL next_input_fits_metablock = TO_BROTLI_BOOL(
         processed_bytes + InputBlockSize(s) <= max_length);
     /* If block splitting is not used, then flush as soon as there is some
@@ -1337,7 +1337,7 @@ static BROTLI_NOINLINE BROTLI_BOOL BrotliCompressBufferQuality10(
          allocation here and not before the loop, because if the input is small,
          this will be allocated after the Zopfli cost model is freed, so this
          will not increase peak memory usage.
-         TODO: If the first allocation is too small, increase command
+         TODO(eustas): If the first allocation is too small, increase command
          buffer size exponentially. */
       new_cmd_alloc_size = BROTLI_MAX(size_t, expected_num_commands,
                                       num_commands + path_size + 1);
@@ -1540,7 +1540,7 @@ BROTLI_BOOL BrotliEncoderCompress(
     return BROTLI_TRUE;
   }
   if (quality == 10) {
-    /* TODO: Implement this direct path for all quality levels. */
+    /* TODO(eustas): Implement this direct path for all quality levels. */
     const int lg_win = BROTLI_MIN(int, BROTLI_LARGE_MAX_WINDOW_BITS,
                                        BROTLI_MAX(int, 16, lgwin));
     int ok = BrotliCompressBufferQuality10(lg_win, input_size, input_buffer,
@@ -2091,6 +2091,153 @@ BROTLI_BOOL BrotliEncoderAttachPreparedDictionary(BrotliEncoderState* state,
   return BROTLI_TRUE;
 }
 
+size_t BrotliEncoderEstimatePeakMemoryUsage(int quality, int lgwin,
+                                            size_t input_size) {
+  BrotliEncoderParams params;
+  BrotliEncoderInitParams(&params);
+  params.quality = quality;
+  params.lgwin = lgwin;
+  params.size_hint = input_size;
+  SanitizeParams(&params);
+  params.lgblock = ComputeLgBlock(&params);
+  ChooseHasher(&params, &params.hasher);
+  if (params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY ||
+      params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY) {
+    size_t state_size = sizeof(BrotliEncoderState);
+    size_t block_size = BROTLI_MIN(size_t, input_size, (1ul << params.lgwin));
+    size_t hash_table_size =
+        HashTableSize(MaxHashTableSize(params.quality), block_size);
+    size_t hash_size =
+        (hash_table_size < (1u << 10)) ? 0 : sizeof(int) * hash_table_size;
+    size_t cmdbuf_size = params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY ?
+        5 * BROTLI_MIN(size_t, block_size, 1ul << 17) : 0;
+    if (params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY) {
+      state_size += sizeof(BrotliOnePassArena);
+    } else {
+      state_size += sizeof(BrotliTwoPassArena);
+    }
+    return hash_size + cmdbuf_size + state_size;
+  } else {
+    size_t short_ringbuffer_size = (size_t)1 << params.lgblock;
+    int ringbuffer_bits = ComputeRbBits(&params);
+    size_t ringbuffer_size = input_size < short_ringbuffer_size ?
+        input_size : (1u << ringbuffer_bits) + short_ringbuffer_size;
+    size_t hash_size[4] = {0};
+    size_t metablock_size =
+        BROTLI_MIN(size_t, input_size, MaxMetablockSize(&params));
+    size_t inputblock_size =
+        BROTLI_MIN(size_t, input_size, (size_t)1 << params.lgblock);
+    size_t cmdbuf_size = metablock_size * 2 + inputblock_size * 6;
+    size_t outbuf_size = metablock_size * 2 + 503;
+    size_t histogram_size = 0;
+    HasherSize(&params, BROTLI_TRUE, input_size, hash_size);
+    if (params.quality < MIN_QUALITY_FOR_BLOCK_SPLIT) {
+      cmdbuf_size = BROTLI_MIN(size_t, cmdbuf_size,
+          MAX_NUM_DELAYED_SYMBOLS * sizeof(Command) + inputblock_size * 12);
+    }
+    if (params.quality >= MIN_QUALITY_FOR_HQ_BLOCK_SPLITTING) {
+      /* Only a very rough estimation, based on enwik8. */
+      histogram_size = 200 << 20;
+    } else if (params.quality >= MIN_QUALITY_FOR_BLOCK_SPLIT) {
+      size_t literal_histograms =
+          BROTLI_MIN(size_t, metablock_size / 6144, 256);
+      size_t command_histograms =
+          BROTLI_MIN(size_t, metablock_size / 6144, 256);
+      size_t distance_histograms =
+          BROTLI_MIN(size_t, metablock_size / 6144, 256);
+      histogram_size = literal_histograms * sizeof(HistogramLiteral) +
+                       command_histograms * sizeof(HistogramCommand) +
+                       distance_histograms * sizeof(HistogramDistance);
+    }
+    return (ringbuffer_size +
+            hash_size[0] + hash_size[1] + hash_size[2] + hash_size[3] +
+            cmdbuf_size +
+            outbuf_size +
+            histogram_size);
+  }
+}
+size_t BrotliEncoderGetPreparedDictionarySize(
+    const BrotliEncoderPreparedDictionary* prepared_dictionary) {
+  /* First field of dictionary structs */
+  const BrotliEncoderPreparedDictionary* prepared = prepared_dictionary;
+  uint32_t magic = *((const uint32_t*)prepared);
+  size_t overhead = 0;
+  if (magic == kManagedDictionaryMagic) {
+    const ManagedDictionary* managed = (const ManagedDictionary*)prepared;
+    overhead = sizeof(ManagedDictionary);
+    magic = *managed->dictionary;
+    prepared = (const BrotliEncoderPreparedDictionary*)managed->dictionary;
+  }
+
+  if (magic == kPreparedDictionaryMagic) {
+    const PreparedDictionary* dictionary =
+        (const PreparedDictionary*)prepared;
+    /* Keep in sync with step 3 of CreatePreparedDictionary */
+    return sizeof(PreparedDictionary) + dictionary->source_size +
+        (sizeof(uint32_t) << dictionary->slot_bits) +
+        (sizeof(uint16_t) << dictionary->bucket_bits) +
+        (sizeof(uint32_t) * dictionary->source_offset) + overhead;
+  } else if (magic == kSharedDictionaryMagic) {
+    const SharedEncoderDictionary* dictionary =
+        (const SharedEncoderDictionary*)prepared;
+    const CompoundDictionary* compound = &dictionary->compound;
+    const ContextualEncoderDictionary* contextual = &dictionary->contextual;
+    size_t result = sizeof(*dictionary);
+    size_t i;
+    size_t num_instances;
+    const BrotliEncoderDictionary* instances;
+    for (i = 0; i < compound->num_prepared_instances_; i++) {
+      size_t size = BrotliEncoderGetPreparedDictionarySize(
+          (const BrotliEncoderPreparedDictionary*)
+          compound->prepared_instances_[i]);
+      if (!size) return 0;  /* error */
+      result += size;
+    }
+    if (contextual->context_based) {
+      num_instances = contextual->num_instances_;
+      instances = contextual->instances_;
+      result += sizeof(*instances) * num_instances;
+    } else {
+      num_instances = 1;
+      instances = &contextual->instance_;
+    }
+    for (i = 0; i < num_instances; i++) {
+      const BrotliEncoderDictionary* dict = &instances[i];
+      result += dict->trie.pool_capacity * sizeof(BrotliTrieNode);
+      if (dict->hash_table_data_words_) {
+        result += sizeof(kStaticDictionaryHashWords);
+      }
+      if (dict->hash_table_data_lengths_) {
+        result += sizeof(kStaticDictionaryHashLengths);
+      }
+      if (dict->buckets_data_) {
+        result += sizeof(*dict->buckets_data_) * dict->buckets_alloc_size_;
+      }
+      if (dict->dict_words_data_) {
+        result += sizeof(*dict->dict_words) * dict->dict_words_alloc_size_;
+      }
+      if (dict->words_instance_) {
+        result += sizeof(*dict->words_instance_);
+        /* data_size not added here: it is never allocated by the
+           SharedEncoderDictionary, instead it always points to the file
+           already loaded in memory. So if the caller wants to include
+           this memory as well, add the size of the loaded dictionary
+           file to this. */
+      }
+    }
+    return result + overhead;
+  }
+  return 0;  /* error */
+}
+
+#if defined(BROTLI_TEST)
+size_t MakeUncompressedStreamForTest(const uint8_t*, size_t, uint8_t*);
+size_t MakeUncompressedStreamForTest(
+    const uint8_t* input, size_t input_size, uint8_t* output) {
+  return MakeUncompressedStream(input, input_size, output);
+}
+#endif
+
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */
 #endif

c/enc/encoder_dict.c

@@ -367,7 +367,7 @@ static void BuildDictionaryHashTable(uint16_t* hash_table_words,
   /* The order of the loops is such that in case of collision, words with
      shorter length are preferred, and in case of same length, words with
      smaller index. There is only a single word per bucket. */
-  /* TODO: consider adding optional user-supplied frequency_map to use
+  /* TODO(lode): consider adding optional user-supplied frequency_map to use
      for preferred words instead, this can make the encoder better for
      quality 9 and below without affecting the decoder */
   memset(hash_table_words, 0, sizeof(kStaticDictionaryHashWords));
@@ -424,7 +424,7 @@ static void ComputeCutoffTransforms(
      ((cutoffTransforms >> (N * 6)) & 0x3F), so for example the identity
      transform code must be 0-63, for N=1 the transform code must be 4-67, ...,
      for N=9 it must be 36-99.
-     TODO: consider a simple flexible uint8_t[10] instead of the uint64_t
+     TODO(lode): consider a simple flexible uint8_t[10] instead of the uint64_t
      for the cutoff transforms, so that shared dictionaries can have the
      OmitLast transforms anywhere without loss. */
   *count = 0;
@@ -501,7 +501,7 @@ void BrotliInitSharedEncoderDictionary(SharedEncoderDictionary* dict) {
   dict->max_quality = BROTLI_MAX_QUALITY;
 }
 
-/* TODO: make sure that tooling will warn user if not all the cutoff
+/* TODO(eustas): make sure that tooling will warn user if not all the cutoff
    transforms are available (for low-quality encoder). */
 static BROTLI_BOOL InitCustomSharedEncoderDictionary(
     MemoryManager* m, const BrotliSharedDictionary* decoded_dict,
@@ -623,6 +623,14 @@ void BrotliDestroyManagedDictionary(ManagedDictionary* dictionary) {
   BrotliBootstrapFree(dictionary, &dictionary->memory_manager_);
 }
 
+/* Escalate internal functions visibility; for testing purposes only. */
+#if defined(BROTLI_TEST)
+void InitEncoderDictionaryForTest(BrotliEncoderDictionary*);
+void InitEncoderDictionaryForTest(BrotliEncoderDictionary* d) {
+  InitEncoderDictionary(d);
+}
+#endif
+
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */
 #endif

c/enc/entropy_encode_static.h

@@ -76,6 +76,7 @@ static const uint8_t kStaticDistanceCodeDepth[64] = {
   6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
 };
 
+/* GENERATED CODE START */
 static const uint32_t kCodeLengthBits[18] = {
   0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 15, 31, 0, 11, 7,
 };
@@ -531,6 +532,7 @@ static BROTLI_INLINE void StoreStaticDistanceHuffmanTree(
     size_t* storage_ix, uint8_t* storage) {
   BrotliWriteBits(28, 0x0369DC03u, storage_ix, storage);
 }
+/* GENERATED CODE END */
 
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */

c/enc/hash_composite_inc.h

@@ -48,7 +48,7 @@ static void FN(Initialize)(HasherCommon* common,
   self->hb_common = *self->common;
   self->fresh = BROTLI_TRUE;
   self->params = params;
-  /* TODO: Initialize of the hashers is deferred to Prepare (and params
+  /* TODO(lode): Initialize of the hashers is deferred to Prepare (and params
      remembered here) because we don't get the one_shot and input_size params
      here that are needed to know the memory size of them. Instead provide
      those params to all hashers FN(Initialize) */

c/enc/histogram.c

@@ -63,7 +63,7 @@ void BrotliBuildHistogramsWithContext(
     BlockSplitIteratorNext(&insert_and_copy_it);
     HistogramAddCommand(&insert_and_copy_histograms[insert_and_copy_it.type_],
         cmd->cmd_prefix_);
-    /* TODO: unwrap iterator blocks. */
+    /* TODO(eustas): unwrap iterator blocks. */
     for (j = cmd->insert_len_; j != 0; --j) {
       size_t context;
       BlockSplitIteratorNext(&literal_it);

c/enc/params.h

@@ -40,7 +40,7 @@ typedef struct BrotliEncoderParams {
   BROTLI_BOOL large_window;
   BrotliHasherParams hasher;
   BrotliDistanceParams dist;
-  /* TODO: rename to BrotliShared... */
+  /* TODO(eustas): rename to BrotliShared... */
   SharedEncoderDictionary dictionary;
 } BrotliEncoderParams;
 

c/enc/static_dict.c

@@ -527,7 +527,7 @@ BROTLI_BOOL BrotliFindAllStaticDictionaryMatches(
         uint32_t skipdist = (uint32_t)((uint32_t)(1 << dictionary->words->
             size_bits_by_length[len_code]) & ~1u) *
             (uint32_t)dictionary->num_transforms;
-        /* TODO: check for dist overflow */
+        /* TODO(lode): check for dist overflow */
         dist += skipdist;
         AddMatch(dist, (size_t)l, len_code, matches);
       }

c/enc/static_dict_lut.h

@@ -22,6 +22,7 @@ typedef struct DictWord {
   uint16_t idx;
 } DictWord;
 
+/* GENERATED CODE START */
 static const int kDictNumBits = 15;
 static const uint32_t kDictHashMul32 = 0x1E35A7BD;
 
@@ -5856,6 +5857,7 @@ static const DictWord kStaticDictionaryWords[31705] = {
 ,0,1735},{5,0,598},{7,0,791},{8,0,108},{9,0,123},{7,10,1570},{140,10,542},{142,
 11,410},{9,11,660},{138,11,347}
 };
+/* GENERATED CODE END */
 
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */

c/include/brotli/encode.h

@@ -479,6 +479,13 @@ BROTLI_ENC_API BROTLI_BOOL BrotliEncoderHasMoreOutput(
 BROTLI_ENC_API const uint8_t* BrotliEncoderTakeOutput(
     BrotliEncoderState* state, size_t* size);
 
+/* Returns the estimated peak memory usage (in bytes) of the BrotliCompress()
+   function, not counting the memory needed for the input and output. */
+BROTLI_ENC_EXTRA_API size_t BrotliEncoderEstimatePeakMemoryUsage(
+    int quality, int lgwin, size_t input_size);
+/* Returns 0 if dictionary is not valid; otherwise returns allocation size. */
+BROTLI_ENC_EXTRA_API size_t BrotliEncoderGetPreparedDictionarySize(
+    const BrotliEncoderPreparedDictionary* dictionary);
 
 /**
  * Gets an encoder library version.

c/include/brotli/port.h

@@ -247,6 +247,24 @@
 #define BROTLI_PUBLIC
 #endif
 
+/* BROTLI_INTERNAL could be defined to override visibility, e.g. for tests. */
+#if !defined(BROTLI_INTERNAL)
+#if defined(_WIN32) || defined(__CYGWIN__)
+#define BROTLI_INTERNAL
+#elif BROTLI_GNUC_VERSION_CHECK(3, 3, 0) ||                         \
+    BROTLI_TI_VERSION_CHECK(8, 0, 0) ||                             \
+    BROTLI_INTEL_VERSION_CHECK(16, 0, 0) ||                         \
+    BROTLI_ARM_VERSION_CHECK(4, 1, 0) ||                            \
+    BROTLI_IBM_VERSION_CHECK(13, 1, 0) ||                           \
+    BROTLI_SUNPRO_VERSION_CHECK(5, 11, 0) ||                        \
+    (BROTLI_TI_VERSION_CHECK(7, 3, 0) &&                            \
+     defined(__TI_GNU_ATTRIBUTE_SUPPORT__) && defined(__TI_EABI__))
+#define BROTLI_INTERNAL __attribute__ ((visibility ("hidden")))
+#else
+#define BROTLI_INTERNAL
+#endif
+#endif
+
 #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) &&   \
     !defined(__STDC_NO_VLA__) && !defined(__cplusplus) &&           \
     !defined(__PGI) && !defined(__PGIC__) && !defined(__TINYC__) && \
@@ -286,4 +304,10 @@
 #define BROTLI_ENC_API
 #endif
 
+#if defined(BROTLI_BUILD_ENC_EXTRA_API)
+#define BROTLI_ENC_EXTRA_API BROTLI_ENC_API
+#else
+#define BROTLI_ENC_EXTRA_API BROTLI_INTERNAL
+#endif
+
 #endif  /* BROTLI_COMMON_PORT_H_ */

c/tools/brotli.c

@@ -664,7 +664,7 @@ static int64_t FileSize(const char* path) {
 }
 
 /* Copy file times and permissions.
-   TODO: this is a "best effort" implementation; honest cross-platform
+   TODO(eustas): this is a "best effort" implementation; honest cross-platform
    fully featured implementation is way too hacky; add more hacks by request. */
 static void CopyStat(const char* input_path, const char* output_path) {
   struct stat statbuf;

go/cbrotli/writer.go

@@ -108,7 +108,7 @@ func (w *Writer) writeChunk(p []byte, op C.BrotliEncoderOperation) (n int, err e
 		if length != 0 {
 			// It is a workaround for non-copying-wrapping of native memory.
 			// C-encoder never pushes output block longer than ((2 << 25) + 502).
-			// TODO: use natural wrapper, when it becomes available, see
+			// TODO(eustas): use natural wrapper, when it becomes available, see
 			//               https://golang.org/issue/13656.
 			output := (*[1 << 30]byte)(unsafe.Pointer(result.output_data))[:length:length]
 			_, err = w.dst.Write(output)

java/org/brotli/dec/Decode.java

@@ -88,7 +88,7 @@ final class Decode {
       0x020000, 0x020004, 0x020003, 0x030002, 0x020000, 0x020004, 0x020003, 0x040005
   };
 
-  // TODO: generalize.
+  // TODO(eustas): generalize.
   static final int MAX_TRANSFORMED_WORD_LENGTH = 5 + 24 + 8;
 
   private static final int MAX_DISTANCE_BITS = 24;
@@ -152,7 +152,7 @@ final class Decode {
     return NUM_DISTANCE_SHORT_CODES + ndirect + 2 * (maxndistbits << npostfix);
   }
 
-  // TODO: add a correctness test for this function when
+  // TODO(eustas): add a correctness test for this function when
   //               large-window and dictionary are implemented.
   private static int calculateDistanceAlphabetLimit(int maxDistance, int npostfix, int ndirect) {
     if (maxDistance < ndirect + (2 << npostfix)) {
@@ -265,7 +265,7 @@ final class Decode {
     s.isLargeWindow = 1;
   }
 
-  // TODO: do we need byte views?
+  // TODO(eustas): do we need byte views?
   static void attachDictionaryChunk(State s, byte[] data) {
     if (s.runningState != INITIALIZED) {
       throw new IllegalStateException("State MUST be freshly initialized");
@@ -485,7 +485,7 @@ final class Decode {
     if (space != 0) {
       throw new BrotliRuntimeException("Unused space"); // COV_NF_LINE
     }
-    // TODO: Pass max_symbol to Huffman table builder instead?
+    // TODO(eustas): Pass max_symbol to Huffman table builder instead?
     Utils.fillIntsWithZeroes(codeLengths, symbol, numSymbols);
   }
 
@@ -504,7 +504,7 @@ final class Decode {
    */
   private static int readSimpleHuffmanCode(int alphabetSizeMax, int alphabetSizeLimit,
       int[] tableGroup, int tableIdx, State s) {
-    // TODO: Avoid allocation?
+    // TODO(eustas): Avoid allocation?
     int[] codeLengths = new int[alphabetSizeLimit];
     int[] symbols = new int[4];
 
@@ -560,7 +560,7 @@ final class Decode {
         break;
     }
 
-    // TODO: Use specialized version?
+    // TODO(eustas): Use specialized version?
     return Huffman.buildHuffmanTable(
         tableGroup, tableIdx, HUFFMAN_TABLE_BITS, codeLengths, alphabetSizeLimit);
   }
@@ -568,7 +568,7 @@ final class Decode {
   // Decode Huffman-coded code lengths.
   private static int readComplexHuffmanCode(int alphabetSizeLimit, int skip,
       int[] tableGroup, int tableIdx, State s) {
-    // TODO: Avoid allocation?
+    // TODO(eustas): Avoid allocation?
     int[] codeLengths = new int[alphabetSizeLimit];
     int[] codeLengthCodeLengths = new int[CODE_LENGTH_CODES];
     int space = 32;
@@ -577,7 +577,7 @@ final class Decode {
       int codeLenIdx = CODE_LENGTH_CODE_ORDER[i];
       BitReader.fillBitWindow(s);
       int p = BitReader.peekBits(s) & 15;
-      // TODO: Demultiplex FIXED_TABLE.
+      // TODO(eustas): Demultiplex FIXED_TABLE.
       s.bitOffset += FIXED_TABLE[p] >> 16;
       int v = FIXED_TABLE[p] & 0xFFFF;
       codeLengthCodeLengths[codeLenIdx] = v;
@@ -709,7 +709,7 @@ final class Decode {
   private static void maybeReallocateRingBuffer(State s) {
     int newSize = s.maxRingBufferSize;
     if (newSize > s.expectedTotalSize) {
-      /* TODO: Handle 2GB+ cases more gracefully. */
+      /* TODO(eustas): Handle 2GB+ cases more gracefully. */
       int minimalNewSize = s.expectedTotalSize;
       while ((newSize >> 1) > minimalNewSize) {
         newSize >>= 1;
@@ -736,7 +736,7 @@ final class Decode {
       s.runningState = INIT_WRITE;
       return;
     }
-    // TODO: Reset? Do we need this?
+    // TODO(eustas): Reset? Do we need this?
     s.literalTreeGroup = new int[0];
     s.commandTreeGroup = new int[0];
     s.distanceTreeGroup = new int[0];
@@ -831,7 +831,7 @@ final class Decode {
     BitReader.fillBitWindow(s);
     s.distancePostfixBits = BitReader.readFewBits(s, 2);
     s.numDirectDistanceCodes = BitReader.readFewBits(s, 4) << s.distancePostfixBits;
-    // TODO: Reuse?
+    // TODO(eustas): Reuse?
     s.contextModes = new byte[s.numLiteralBlockTypes];
     for (int i = 0; i < s.numLiteralBlockTypes;) {
       /* Ensure that less than 256 bits read between readMoreInput. */
@@ -843,7 +843,7 @@ final class Decode {
       BitReader.readMoreInput(s);
     }
 
-    // TODO: Reuse?
+    // TODO(eustas): Reuse?
     s.contextMap = new byte[s.numLiteralBlockTypes << LITERAL_CONTEXT_BITS];
     int numLiteralTrees = decodeContextMap(s.numLiteralBlockTypes << LITERAL_CONTEXT_BITS,
         s.contextMap, s);
@@ -855,7 +855,7 @@ final class Decode {
       }
     }
 
-    // TODO: Reuse?
+    // TODO(eustas): Reuse?
     s.distContextMap = new byte[s.numDistanceBlockTypes << DISTANCE_CONTEXT_BITS];
     int numDistTrees = decodeContextMap(s.numDistanceBlockTypes << DISTANCE_CONTEXT_BITS,
         s.distContextMap, s);
@@ -1091,7 +1091,7 @@ final class Decode {
     byte[] ringBuffer = s.ringBuffer;
 
     while (s.runningState != FINISHED) {
-      // TODO: extract cases to methods for the better readability.
+      // TODO(eustas): extract cases to methods for the better readability.
       switch (s.runningState) {
         case BLOCK_START:
           if (s.metaBlockLength < 0) {

java/org/brotli/dec/Huffman.java

@@ -66,7 +66,7 @@ final class Huffman {
     int tableOffset = tableGroup[tableIdx];
     int key; // Reversed prefix code.
     int[] sorted = new int[codeLengthsSize]; // Symbols sorted by code length.
-    // TODO: fill with zeroes?
+    // TODO(eustas): fill with zeroes?
     int[] count = new int[MAX_LENGTH + 1]; // Number of codes of each length.
     int[] offset = new int[MAX_LENGTH + 1]; // Offsets in sorted table for each length.
     int symbol;

java/org/brotli/dec/build_defs.bzl

@@ -11,7 +11,8 @@ def brotli_java_test(name, main_class = None, jvm_flags = None, **kwargs):
         jvm_flags = []
     jvm_flags = jvm_flags + _TEST_JVM_FLAGS
 
-    test_package = native.package_name().replace("/", ".")
+    test_package = native.package_name().replace("/", ".").replace("javatests.", "")
+
     if main_class == None:
         test_class = test_package + "." + name
     else:

java/org/brotli/enc/PreparedDictionaryGenerator.java

@@ -73,7 +73,7 @@ public class PreparedDictionaryGenerator {
       accumulator |= (src.get(i) & 0xFFL) << (8 * i);
     }
     accumulator <<= 8;
-    /* TODO: apply custom "store" order. */
+    /* TODO(eustas): apply custom "store" order. */
     for (int i = 0; i + 7 < sourceSize; ++i) {
       accumulator = (accumulator >>> 8) | ((src.get(i + 7) & 0xFFL) << 56);
       long h = (accumulator & hashMask) * HASH_MULTIPLIER;

java/org/brotli/wrapper/dec/BUILD

@@ -7,7 +7,7 @@ filegroup(
     srcs = [
         "decoder_jni.cc",
         "decoder_jni.h",
-        # TODO: investigate, why this prevents JNI library loading.
+        # TODO(eustas): investigate, why this prevents JNI library loading.
         #"decoder_jni_onload.cc",
     ],
 )

java/org/brotli/wrapper/dec/DecoderJNI.java

@@ -126,7 +126,7 @@ public class DecoderJNI {
     @Override
     protected void finalize() throws Throwable {
       if (context[0] != 0) {
-        /* TODO: log resource leak? */
+        /* TODO(eustas): log resource leak? */
         destroy();
       }
       super.finalize();

java/org/brotli/wrapper/dec/decoder_jni.cc

@@ -79,7 +79,7 @@ Java_org_brotli_wrapper_dec_DecoderJNI_nativeCreate(
   }
 
   if (ok) {
-    /* TODO: future versions (e.g. when 128-bit architecture comes)
+    /* TODO(eustas): future versions (e.g. when 128-bit architecture comes)
                      might require thread-safe cookie<->handle mapping. */
     context[0] = reinterpret_cast<jlong>(handle);
   } else if (!!handle) {

java/org/brotli/wrapper/enc/EncoderJNI.java

@@ -169,7 +169,7 @@ class EncoderJNI {
     @Override
     protected void finalize() throws Throwable {
       if (context[0] != 0) {
-        /* TODO: log resource leak? */
+        /* TODO(eustas): log resource leak? */
         destroy();
       }
       super.finalize();

java/org/brotli/wrapper/enc/encoder_jni.cc

@@ -93,7 +93,7 @@ Java_org_brotli_wrapper_enc_EncoderJNI_nativeCreate(
   }
 
   if (ok) {
-    /* TODO: future versions (e.g. when 128-bit architecture comes)
+    /* TODO(eustas): future versions (e.g. when 128-bit architecture comes)
                      might require thread-safe cookie<->handle mapping. */
     context[0] = reinterpret_cast<jlong>(handle);
   } else if (!!handle) {

python/tests/_test_utils.py

@@ -10,12 +10,16 @@ import unittest
 
 
 project_dir = os.path.abspath(os.path.join(__file__, '..', '..', '..'))
-src_dir = os.path.join(project_dir, 'python')
+test_dir = os.getenv("BROTLI_TESTS_PATH")
-test_dir = os.path.join(project_dir, 'tests')
+BRO_ARGS = [os.getenv("BROTLI_WRAPPER")]
 
-python_exe = sys.executable or 'python'
+# Fallbacks
-bro_path = os.path.join(src_dir, 'bro.py')
+if test_dir is None:
-BRO_ARGS = [python_exe, bro_path]
+  test_dir = os.path.join(project_dir, 'tests')
+if BRO_ARGS[0] is None:
+  python_exe = sys.executable or 'python'
+  bro_path = os.path.join(project_dir, 'python', 'bro.py')
+  BRO_ARGS = [python_exe, bro_path]
 
 # Get the platform/version-specific build folder.
 # By default, the distutils build base is in the same location as setup.py.

research/deorummolae.cc

@@ -97,7 +97,7 @@ static void buildFullText(std::vector<std::vector<TextChar>>* data,
 }
 
 /* Build longest-common-prefix based on suffix array and text.
-   TODO: borrowed -> unknown efficiency. */
+   TODO(eustas): borrowed -> unknown efficiency. */
 static void buildLcp(std::vector<TextChar>* text, std::vector<TextIdx>* sa,
     std::vector<TextIdx>* lcp, std::vector<TextIdx>* invese_sa) {
   TextIdx size = static_cast<TextIdx>(text->size());

research/dictionary_generator.cc

@@ -219,7 +219,7 @@ int main(int argc, char const* argv[]) {
         }
       } else if (arg1 == 's') {
         sliceLen = readInt(arg2);
-        // TODO: investigate why sliceLen == 4..5 greatly slows down
+        // TODO(eustas): investigate why sliceLen == 4..5 greatly slows down
         //               durschlag engine, but only from command line;
         //               durschlag_runner seems to work fine with those.
         if (sliceLen < 4 || sliceLen > 256) {

research/durchschlag.cc

@@ -401,7 +401,7 @@ DurchschlagIndex durchschlag_index(const std::vector<uint8_t>& data) {
   std::vector<TextIdx> isa(total);
   for (TextIdx i = 0; i < total; ++i) isa[sa[i]] = i;
 
-  // TODO: borrowed -> unknown efficiency.
+  // TODO(eustas): borrowed -> unknown efficiency.
   std::vector<TextIdx> lcp(total);
   TextIdx k = 0;
   lcp[total - 1] = 0;