Update: (#600)

* encoder: relax backward references candidates asserts
 * encoder: make RNG more platform-independent
 * encoder: remove "unused" param (context mode)
 * CLI: improve first-encounter experience
 * Java: update SynthTest
 * Java: refine proguard config
 * Java/JNI: fix one-shot compression workflow

c/enc/backward_references_hq.c

@@ -745,10 +745,8 @@ void BrotliCreateHqZopfliBackwardReferences(
         &matches[cur_match_pos]);
     cur_match_end = cur_match_pos + num_found_matches;
     for (j = cur_match_pos; j + 1 < cur_match_end; ++j) {
-      assert(BackwardMatchLength(&matches[j]) <
+      assert(BackwardMatchLength(&matches[j]) <=
           BackwardMatchLength(&matches[j + 1]));
-      assert(matches[j].distance > max_distance ||
-             matches[j].distance <= matches[j + 1].distance);
     }
     num_matches[i] = (uint32_t)num_found_matches;
     if (num_found_matches > 0) {

c/enc/block_splitter.c

@@ -74,11 +74,9 @@ static void CopyLiteralsToByteArray(const Command* cmds,
   }
 }
 
-static BROTLI_INLINE unsigned int MyRand(unsigned int* seed) {
+static BROTLI_INLINE uint32_t MyRand(uint32_t* seed) {
+  /* Initial seed should be 7. In this case, loop length is (1 << 29). */
   *seed *= 16807U;
-  if (*seed == 0) {
-    *seed = 1;
-  }
   return *seed;
 }
 

c/enc/block_splitter_inc.h

@@ -13,7 +13,7 @@ static void FN(InitialEntropyCodes)(const DataType* data, size_t length,
                                     size_t stride,
                                     size_t num_histograms,
                                     HistogramType* histograms) {
-  unsigned int seed = 7;
+  uint32_t seed = 7;
   size_t block_length = length / num_histograms;
   size_t i;
   FN(ClearHistograms)(histograms, num_histograms);
@@ -29,14 +29,13 @@ static void FN(InitialEntropyCodes)(const DataType* data, size_t length,
   }
 }
 
-static void FN(RandomSample)(unsigned int* seed,
+static void FN(RandomSample)(uint32_t* seed,
                              const DataType* data,
                              size_t length,
                              size_t stride,
                              HistogramType* sample) {
   size_t pos = 0;
   if (stride >= length) {
-    pos = 0;
     stride = length;
   } else {
     pos = MyRand(seed) % (length - stride + 1);
@@ -50,7 +49,7 @@ static void FN(RefineEntropyCodes)(const DataType* data, size_t length,
                                    HistogramType* histograms) {
   size_t iters =
       kIterMulForRefining * length / stride + kMinItersForRefining;
-  unsigned int seed = 7;
+  uint32_t seed = 7;
   size_t iter;
   iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms;
   for (iter = 0; iter < iters; ++iter) {

c/enc/encode.c

@@ -385,8 +385,7 @@ static void ChooseContextMap(int quality,
    context values, based on the entropy reduction of histograms over the
    first 5 bits of literals. */
 static BROTLI_BOOL ShouldUseComplexStaticContextMap(const uint8_t* input,
-    size_t start_pos, size_t length, size_t mask, int quality,
-    size_t size_hint, ContextType* literal_context_mode,
+    size_t start_pos, size_t length, size_t mask, int quality, size_t size_hint,
     size_t* num_literal_contexts, const uint32_t** literal_context_map) {
   static const uint32_t kStaticContextMapComplexUTF8[64] = {
     11, 11, 12, 12, /* 0 special */
@@ -457,7 +456,6 @@ static BROTLI_BOOL ShouldUseComplexStaticContextMap(const uint8_t* input,
     if (entropy[2] > 3.0 || entropy[1] - entropy[2] < 0.2) {
       return BROTLI_FALSE;
     } else {
-      *literal_context_mode = CONTEXT_UTF8;
       *num_literal_contexts = 13;
       *literal_context_map = kStaticContextMapComplexUTF8;
       return BROTLI_TRUE;
@@ -466,13 +464,12 @@ static BROTLI_BOOL ShouldUseComplexStaticContextMap(const uint8_t* input,
 }
 
 static void DecideOverLiteralContextModeling(const uint8_t* input,
-    size_t start_pos, size_t length, size_t mask, int quality,
-    size_t size_hint, ContextType* literal_context_mode,
+    size_t start_pos, size_t length, size_t mask, int quality, size_t size_hint,
     size_t* num_literal_contexts, const uint32_t** literal_context_map) {
   if (quality < MIN_QUALITY_FOR_CONTEXT_MODELING || length < 64) {
     return;
   } else if (ShouldUseComplexStaticContextMap(
-      input, start_pos, length, mask, quality, size_hint, literal_context_mode,
+      input, start_pos, length, mask, quality, size_hint,
       num_literal_contexts, literal_context_map)) {
     /* Context map was already set, nothing else to do. */
   } else {
@@ -492,7 +489,6 @@ static void DecideOverLiteralContextModeling(const uint8_t* input,
         prev = lut[literal >> 6] * 3;
       }
     }
-    *literal_context_mode = CONTEXT_UTF8;
     ChooseContextMap(quality, &bigram_prefix_histo[0], num_literal_contexts,
                      literal_context_map);
   }
@@ -596,7 +592,7 @@ static void WriteMetaBlockInternal(MemoryManager* m,
       if (!params->disable_literal_context_modeling) {
         DecideOverLiteralContextModeling(
             data, wrapped_last_flush_pos, bytes, mask, params->quality,
-            params->size_hint, &literal_context_mode, &num_literal_contexts,
+            params->size_hint, &num_literal_contexts,
             &literal_context_map);
       }
       BrotliBuildMetaBlockGreedy(m, data, wrapped_last_flush_pos, mask,

c/tools/brotli.c

@@ -15,6 +15,7 @@
 #include <sys/types.h>
 #include <time.h>
 
+#include "../common/constants.h"
 #include "../common/version.h"
 #include <brotli/decode.h>
 #include <brotli/encode.h>
@@ -38,6 +39,7 @@
 #endif
 
 #define fdopen _fdopen
+#define isatty _isatty
 #define unlink _unlink
 #define utimbuf _utimbuf
 #define utime _utime
@@ -685,6 +687,11 @@ static BROTLI_BOOL DecompressFiles(Context* context) {
       return BROTLI_FALSE;
     }
     is_ok = OpenFiles(context);
+    if (is_ok && !context->current_input_path &&
+        !context->force_overwrite && isatty(STDIN_FILENO)) {
+      fprintf(stderr, "Use -h help. Use -f to force input from a terminal.\n");
+      is_ok = BROTLI_FALSE;
+    }
     if (is_ok) is_ok = DecompressFile(context, s);
     BrotliDecoderDestroyInstance(s);
     if (!CloseFiles(context, is_ok)) is_ok = BROTLI_FALSE;
@@ -750,6 +757,11 @@ static BROTLI_BOOL CompressFiles(Context* context) {
     BrotliEncoderSetParameter(s,
         BROTLI_PARAM_LGWIN, (uint32_t)context->lgwin);
     is_ok = OpenFiles(context);
+    if (is_ok && !context->current_output_path &&
+        !context->force_overwrite && isatty(STDOUT_FILENO)) {
+      fprintf(stderr, "Use -h help. Use -f to force output to a terminal.\n");
+      is_ok = BROTLI_FALSE;
+    }
     if (is_ok) is_ok = CompressFile(context, s);
     BrotliEncoderDestroyInstance(s);
     if (!CloseFiles(context, is_ok)) is_ok = BROTLI_FALSE;

java/org/brotli/dec/proguard.cfg

@@ -1,2 +1,6 @@
 # DictionaryData is an optionally / dynamically loaded built-in dictionary.
 -keep class org.brotli.dec.DictionaryData
+
+# We get the fully-qualified name of DictionaryData from Dictionary, so avoid
+# renaming it.
+-keepnames class org.brotli.dec.Dictionary

java/org/brotli/wrapper/enc/Encoder.java

@@ -171,7 +171,7 @@ public class Encoder {
           buffer.get(chunk);
           output.add(chunk);
           totalOutputSize += chunk.length;
-        } else if (encoder.hasRemainingInput()) {
+        } else if (!encoder.isFinished()) {
           encoder.push(EncoderJNI.Operation.FINISH, 0);
         } else {
           break;

java/org/brotli/wrapper/enc/EncoderJNI.java

@@ -25,7 +25,7 @@ class EncoderJNI {
   }
 
   static class Wrapper {
-    protected final long[] context = new long[4];
+    protected final long[] context = new long[5];
     private final ByteBuffer inputBuffer;
 
     Wrapper(int inputBufferSize, int quality, int lgwin)
@@ -71,6 +71,10 @@ class EncoderJNI {
       return context[3] != 0;
     }
 
+    boolean isFinished() {
+      return context[4] != 0;
+    }
+
     ByteBuffer getInputBuffer() {
       return inputBuffer;
     }

java/org/brotli/wrapper/enc/EncoderTest.java

@@ -69,15 +69,18 @@ public class EncoderTest {
       throw new RuntimeException("Can't read bundle entry: " + entryName);
     }
 
-    byte[] compressed = Encoder.compress(original, new Encoder.Parameters().setQuality(6));
+    for (int window = 10; window <= 22; window++) {
+      byte[] compressed =
+          Encoder.compress(original, new Encoder.Parameters().setQuality(6).setWindow(window));
 
-    InputStream decoder = new BrotliInputStream(new ByteArrayInputStream(compressed));
-    try {
-      long originalCrc = BundleHelper.fingerprintStream(new ByteArrayInputStream(original));
-      long crc = BundleHelper.fingerprintStream(decoder);
-      assertEquals(originalCrc, crc);
-    } finally {
-      decoder.close();
+      InputStream decoder = new BrotliInputStream(new ByteArrayInputStream(compressed));
+      try {
+        long originalCrc = BundleHelper.fingerprintStream(new ByteArrayInputStream(original));
+        long crc = BundleHelper.fingerprintStream(decoder);
+        assertEquals(originalCrc, crc);
+      } finally {
+        decoder.close();
+      }
     }
   }
 }

java/org/brotli/wrapper/enc/encoder_jni.cc

@@ -45,8 +45,8 @@ Java_org_brotli_wrapper_enc_EncoderJNI_nativeCreate(
     JNIEnv* env, jobject /*jobj*/, jlongArray ctx) {
   bool ok = true;
   EncoderHandle* handle = nullptr;
-  jlong context[4];
-  env->GetLongArrayRegion(ctx, 0, 4, context);
+  jlong context[5];
+  env->GetLongArrayRegion(ctx, 0, 5, context);
   size_t input_size = context[1];
   context[0] = 0;
   handle = new (std::nothrow) EncoderHandle();
@@ -110,12 +110,12 @@ Java_org_brotli_wrapper_enc_EncoderJNI_nativeCreate(
 JNIEXPORT void JNICALL
 Java_org_brotli_wrapper_enc_EncoderJNI_nativePush(
     JNIEnv* env, jobject /*jobj*/, jlongArray ctx, jint input_length) {
-  jlong context[4];
-  env->GetLongArrayRegion(ctx, 0, 4, context);
+  jlong context[5];
+  env->GetLongArrayRegion(ctx, 0, 5, context);
   EncoderHandle* handle = getHandle(reinterpret_cast<void*>(context[0]));
   int operation = context[1];
   context[1] = 0;  /* ERROR */
-  env->SetLongArrayRegion(ctx, 0, 4, context);
+  env->SetLongArrayRegion(ctx, 0, 5, context);
 
   BrotliEncoderOperation op;
   switch (operation) {
@@ -145,8 +145,9 @@ Java_org_brotli_wrapper_enc_EncoderJNI_nativePush(
     context[1] = 1;
     context[2] = BrotliEncoderHasMoreOutput(handle->state) ? 1 : 0;
     context[3] = (handle->input_offset != handle->input_last) ? 1 : 0;
+    context[4] = BrotliEncoderIsFinished(handle->state) ? 1 : 0;
   }
-  env->SetLongArrayRegion(ctx, 0, 4, context);
+  env->SetLongArrayRegion(ctx, 0, 5, context);
 }
 
 /**
@@ -160,15 +161,16 @@ Java_org_brotli_wrapper_enc_EncoderJNI_nativePush(
 JNIEXPORT jobject JNICALL
 Java_org_brotli_wrapper_enc_EncoderJNI_nativePull(
     JNIEnv* env, jobject /*jobj*/, jlongArray ctx) {
-  jlong context[4];
-  env->GetLongArrayRegion(ctx, 0, 4, context);
+  jlong context[5];
+  env->GetLongArrayRegion(ctx, 0, 5, context);
   EncoderHandle* handle = getHandle(reinterpret_cast<void*>(context[0]));
   size_t data_length = 0;
   const uint8_t* data = BrotliEncoderTakeOutput(handle->state, &data_length);
   context[1] = 1;
   context[2] = BrotliEncoderHasMoreOutput(handle->state) ? 1 : 0;
   context[3] = (handle->input_offset != handle->input_last) ? 1 : 0;
-  env->SetLongArrayRegion(ctx, 0, 4, context);
+  context[4] = BrotliEncoderIsFinished(handle->state) ? 1 : 0;
+  env->SetLongArrayRegion(ctx, 0, 5, context);
   return env->NewDirectByteBuffer(const_cast<uint8_t*>(data), data_length);
 }