Update (#706)

Update
 * add ASAN/MSAN unaligned read specializations
 * add "brotli" prefix to u_uint64 type
 * increment version to 1.0.06
 * fix CoverityScan "unused assignment" warning
 * fix JDK 8<->9 incompatibility
 * add encoder optimization for empty input
 * regenerate JS decoder
 * unbreak Travis builds

.travis.yml

@@ -115,11 +115,10 @@ matrix:
     ## We'll just test 4.4 and the most recent version.
     ###
     - os: osx
-      env: BUILD_SYSTEM=cmake C_COMPILER=gcc-6 CXX_COMPILER=g++-6
+      env: BUILD_SYSTEM=cmake C_COMPILER=gcc CXX_COMPILER=g++
     - os: osx
       env: BUILD_SYSTEM=cmake C_COMPILER=gcc-4.9 CXX_COMPILER=g++-4.9
     - os: osx
-      osx_image: xcode9.3
       env: BUILD_SYSTEM=cmake
 
     ###
@@ -166,6 +165,7 @@ matrix:
     - os: linux
       sudo: required
       language: java
+      jdk: oraclejdk9
       env: BUILD_SYSTEM=bazel
       addons:
         apt:
@@ -174,11 +174,13 @@ matrix:
               key_url: "https://storage.googleapis.com/bazel-apt/doc/apt-key.pub.gpg"
             - ubuntu-toolchain-r-test
           packages:
-            - oracle-java8-installer
             - bazel
 
     - os: osx
       env: BUILD_SYSTEM=bazel
+      # Latest image with Java 1.8 (required to install Bazel).
+      osx_image: xcode9.3
+      language: java
 
 before_install:
 ###

c/common/platform.h

@@ -298,6 +298,33 @@ static BROTLI_INLINE void BrotliUnalignedWrite64(void* p, uint64_t v) {
 }
 #else  /* BROTLI_ALIGNED_READ */
 /* Unaligned memory access is allowed: just cast pointer to requested type. */
+#if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER) || \
+    defined(MEMORY_SANITIZER)
+/* Consider we have an unaligned load/store of 4 bytes from address 0x...05.
+   AddressSanitizer will treat it as a 3-byte access to the range 05:07 and
+   will miss a bug if 08 is the first unaddressable byte.
+   ThreadSanitizer will also treat this as a 3-byte access to 05:07 and will
+   miss a race between this access and some other accesses to 08.
+   MemorySanitizer will correctly propagate the shadow on unaligned stores
+   and correctly report bugs on unaligned loads, but it may not properly
+   update and report the origin of the uninitialized memory.
+   For all three tools, replacing an unaligned access with a tool-specific
+   callback solves the problem. */
+#if defined(__cplusplus)
+extern "C" {
+#endif  /* __cplusplus */
+  uint16_t __sanitizer_unaligned_load16(const void* p);
+  uint32_t __sanitizer_unaligned_load32(const void* p);
+  uint64_t __sanitizer_unaligned_load64(const void* p);
+  void __sanitizer_unaligned_store64(void* p, uint64_t v);
+#if defined(__cplusplus)
+}  /* extern "C" */
+#endif  /* __cplusplus */
+#define BrotliUnalignedRead16 __sanitizer_unaligned_load16
+#define BrotliUnalignedRead32 __sanitizer_unaligned_load32
+#define BrotliUnalignedRead64 __sanitizer_unaligned_load64
+#define BrotliUnalignedWrite64 __sanitizer_unaligned_store64
+#else
 static BROTLI_INLINE uint16_t BrotliUnalignedRead16(const void* p) {
   return *(const uint16_t*)p;
 }
@@ -316,14 +343,14 @@ static BROTLI_INLINE void BrotliUnalignedWrite64(void* p, uint64_t v) {
 /* If __attribute__(aligned) is available, use that. Otherwise, memcpy. */
 
 #if BROTLI_GNUC_HAS_ATTRIBUTE(aligned, 2, 7, 0)
-typedef  __attribute__((aligned(1))) uint64_t unaligned_uint64_t;
+typedef  __attribute__((aligned(1))) uint64_t brotli_unaligned_uint64_t;
 
 static BROTLI_INLINE uint64_t BrotliUnalignedRead64(const void* p) {
-  return (uint64_t) ((unaligned_uint64_t*) p)[0];
+  return (uint64_t) ((brotli_unaligned_uint64_t*) p)[0];
 }
 static BROTLI_INLINE void BrotliUnalignedWrite64(void* p, uint64_t v) {
-  unaligned_uint64_t* dwords = (unaligned_uint64_t*) p;
-  dwords[0] = (unaligned_uint64_t) v;
+  brotli_unaligned_uint64_t* dwords = (brotli_unaligned_uint64_t*) p;
+  dwords[0] = (brotli_unaligned_uint64_t) v;
 }
 #else /* BROTLI_GNUC_HAS_ATTRIBUTE(aligned, 2, 7, 0) */
 static BROTLI_INLINE uint64_t BrotliUnalignedRead64(const void* p) {
@@ -337,6 +364,7 @@ static BROTLI_INLINE void BrotliUnalignedWrite64(void* p, uint64_t v) {
 }
 #endif  /* BROTLI_GNUC_HAS_ATTRIBUTE(aligned, 2, 7, 0) */
 #endif  /* BROTLI_64_BITS */
+#endif  /* ASAN / TSAN / MSAN */
 #endif  /* BROTLI_ALIGNED_READ */
 
 #if BROTLI_LITTLE_ENDIAN

c/common/version.h

@@ -14,13 +14,13 @@
    BrotliEncoderVersion methods. */
 
 /* Semantic version, calculated as (MAJOR << 24) | (MINOR << 12) | PATCH */
-#define BROTLI_VERSION 0x1000005
+#define BROTLI_VERSION 0x1000006
 
 /* This macro is used by build system to produce Libtool-friendly soname. See
    https://www.gnu.org/software/libtool/manual/html_node/Libtool-versioning.html
  */
 
 /* ABI version, calculated as (CURRENT << 24) | (REVISION << 12) | AGE */
-#define BROTLI_ABI_VERSION 0x1005000
+#define BROTLI_ABI_VERSION 0x1006000
 
 #endif  /* BROTLI_COMMON_VERSION_H_ */

c/enc/metablock.c

@@ -181,7 +181,8 @@ void BrotliBuildMetaBlock(MemoryManager* m,
     ComputeDistanceCost(cmds, num_commands,
                         &orig_params.dist, &orig_params.dist, &dist_cost);
     if (dist_cost < best_dist_cost) {
-      best_dist_cost = dist_cost;
+      /* NB: currently unused; uncomment when more param tuning is added. */
+      /* best_dist_cost = dist_cost; */
       params->dist = orig_params.dist;
     }
   }

java/org/brotli/dec/DictionaryData.java

@@ -6,7 +6,6 @@
 
 package org.brotli.dec;
 
-import java.io.UnsupportedEncodingException;
 import java.nio.ByteBuffer;
 
 /**
@@ -22,14 +21,7 @@ final class DictionaryData {
   private static void unpackDictionaryData(
       ByteBuffer dictionary, String data0, String data1, String skipFlip) {
     // Initialize lower 7 bits of every byte in the dictionary.
-    byte[] dict;
-    try {
-      // NB: String#getBytes(String) is present in JDK 1.1, while other variants require JDK 1.6 and
-      // above.
-      dict = (data0 + data1).getBytes("US-ASCII");
-    } catch (UnsupportedEncodingException e) {
-      throw new RuntimeException(e); // cannot happen
-    }
+    byte[] dict = Utils.toUsAsciiBytes(data0 + data1);
     if (dict.length != dictionary.capacity()) {
       throw new RuntimeException("Corrupted brotli dictionary");
     }
@@ -53,7 +45,7 @@ final class DictionaryData {
   static {
     ByteBuffer dictionary = ByteBuffer.allocateDirect(122784);
     unpackDictionaryData(dictionary, DATA0, DATA1, SKIP_FLIP);
-    dictionary.flip();
+    Utils.flipBuffer(dictionary);
     Dictionary.setData(dictionary.asReadOnlyBuffer());
   }
 }

java/org/brotli/dec/Utils.java

@@ -8,6 +8,8 @@ package org.brotli.dec;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.nio.Buffer;
 
 /**
  * A set of utility methods.
@@ -71,4 +73,19 @@ final class Utils {
   static void closeInput(InputStream src) throws IOException {
     src.close();
   }
+
+  static byte[] toUsAsciiBytes(String src) {
+    try {
+      // NB: String#getBytes(String) is present in JDK 1.1, while other variants require JDK 1.6 and
+      // above.
+      return src.getBytes("US-ASCII");
+    } catch (UnsupportedEncodingException e) {
+      throw new RuntimeException(e); // cannot happen
+    }
+  }
+
+  // Crazy pills factory: code compiled for JDK8 does not work on JRE9.
+  static void flipBuffer(Buffer buffer) {
+    buffer.flip();
+  }
 }

java/org/brotli/wrapper/dec/Decoder.java

@@ -7,6 +7,7 @@
 package org.brotli.wrapper.dec;
 
 import java.io.IOException;
+import java.nio.Buffer;
 import java.nio.ByteBuffer;
 import java.nio.channels.ReadableByteChannel;
 import java.util.ArrayList;
@@ -83,7 +84,7 @@ public class Decoder {
             break;
           }
           ByteBuffer inputBuffer = decoder.getInputBuffer();
-          inputBuffer.clear();
+          ((Buffer) inputBuffer).clear();
           int bytesRead = source.read(inputBuffer);
           if (bytesRead == -1) {
             fail("unexpected end of input");
@@ -107,7 +108,7 @@ public class Decoder {
   }
 
   void discard(int length) {
-    buffer.position(buffer.position() + length);
+    ((Buffer) buffer).position(buffer.position() + length);
     if (!buffer.hasRemaining()) {
       buffer = null;
     }
@@ -116,7 +117,7 @@ public class Decoder {
   int consume(ByteBuffer dst) {
     ByteBuffer slice = buffer.slice();
     int limit = Math.min(slice.remaining(), dst.remaining());
-    slice.limit(limit);
+    ((Buffer) slice).limit(limit);
     dst.put(slice);
     discard(limit);
     return limit;

java/org/brotli/wrapper/enc/BrotliEncoderChannel.java

@@ -7,6 +7,7 @@
 package org.brotli.wrapper.enc;
 
 import java.io.IOException;
+import java.nio.Buffer;
 import java.nio.ByteBuffer;
 import java.nio.channels.ClosedChannelException;
 import java.nio.channels.WritableByteChannel;
@@ -65,10 +66,10 @@ public class BrotliEncoderChannel extends Encoder implements WritableByteChannel
       while (src.hasRemaining() && encode(EncoderJNI.Operation.PROCESS)) {
         int limit = Math.min(src.remaining(), inputBuffer.remaining());
         ByteBuffer slice = src.slice();
-        slice.limit(limit);
+        ((Buffer) slice).limit(limit);
         inputBuffer.put(slice);
         result += limit;
-        src.position(src.position() + limit);
+        ((Buffer) src).position(src.position() + limit);
       }
       return result;
     }

java/org/brotli/wrapper/enc/BrotliEncoderChannelTest.java

@@ -10,6 +10,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.Buffer;
 import java.nio.ByteBuffer;
 import java.nio.channels.Channels;
 import java.nio.channels.WritableByteChannel;
@@ -95,8 +96,8 @@ public class BrotliEncoderChannelTest extends BrotliJniTestBase {
           while (src.hasRemaining()) {
             int limit = Math.min(CHUNK_SIZE, src.remaining());
             ByteBuffer slice = src.slice();
-            slice.limit(limit);
-            src.position(src.position() + limit);
+            ((Buffer) slice).limit(limit);
+            ((Buffer) src).position(src.position() + limit);
             encoder.write(slice);
           }
           break;

java/org/brotli/wrapper/enc/EmptyInputTest.java

@@ -0,0 +1,29 @@
+/* Copyright 2018 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+package org.brotli.wrapper.enc;
+
+import static org.junit.Assert.assertEquals;
+
+import org.brotli.integration.BrotliJniTestBase;
+import org.brotli.wrapper.dec.Decoder;
+import java.io.IOException;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link org.brotli.wrapper.enc.Encoder}. */
+@RunWith(JUnit4.class)
+public class EmptyInputTest extends BrotliJniTestBase {
+  @Test
+  public void testEmptyInput() throws IOException {
+    byte[] data = new byte[0];
+    byte[] encoded = Encoder.compress(data);
+    assertEquals(1, encoded.length);
+    byte[] decoded = Decoder.decompress(encoded);
+    assertEquals(0, decoded.length);
+  }
+}

java/org/brotli/wrapper/enc/Encoder.java

@@ -7,6 +7,7 @@
 package org.brotli.wrapper.enc;
 
 import java.io.IOException;
+import java.nio.Buffer;
 import java.nio.ByteBuffer;
 import java.nio.channels.WritableByteChannel;
 import java.util.ArrayList;
@@ -17,8 +18,8 @@ import java.util.ArrayList;
 public class Encoder {
   private final WritableByteChannel destination;
   private final EncoderJNI.Wrapper encoder;
+  private ByteBuffer buffer;
   final ByteBuffer inputBuffer;
-  ByteBuffer buffer;
   boolean closed;
 
   /**
@@ -111,7 +112,7 @@ public class Encoder {
   boolean encode(EncoderJNI.Operation op) throws IOException {
     boolean force = (op != EncoderJNI.Operation.PROCESS);
     if (force) {
-      inputBuffer.limit(inputBuffer.position());
+      ((Buffer) inputBuffer).limit(inputBuffer.position());
     } else if (inputBuffer.hasRemaining()) {
       return true;
     }
@@ -129,7 +130,7 @@ public class Encoder {
         encoder.push(op, inputBuffer.limit());
         hasInput = false;
       } else {
-        inputBuffer.clear();
+        ((Buffer) inputBuffer).clear();
         return true;
       }
     }
@@ -156,6 +157,12 @@ public class Encoder {
    * Encodes the given data buffer.
    */
   public static byte[] compress(byte[] data, Parameters params) throws IOException {
+    if (data.length == 0) {
+      byte[] empty = new byte[1];
+      empty[0] = 6;
+      return empty;
+    }
+    /* data.length > 0 */
     EncoderJNI.Wrapper encoder = new EncoderJNI.Wrapper(data.length, params.quality, params.lgwin);
     ArrayList<byte[]> output = new ArrayList<byte[]>();
     int totalOutputSize = 0;

js/WORKSPACE

@@ -2,11 +2,11 @@ workspace(name = "org_brotli_js")
 
 http_archive(
     name = "io_bazel_rules_closure",
-    sha256 = "a80acb69c63d5f6437b099c111480a4493bad4592015af2127a2f49fb7512d8d",
-    strip_prefix = "rules_closure-0.7.0",
+    sha256 = "b29a8bc2cb10513c864cb1084d6f38613ef14a143797cea0af0f91cd385f5e8c",
+    strip_prefix = "rules_closure-0.8.0",
     urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/0.7.0.tar.gz",
-        "https://github.com/bazelbuild/rules_closure/archive/0.7.0.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/0.8.0.tar.gz",
+        "https://github.com/bazelbuild/rules_closure/archive/0.8.0.tar.gz",
     ],
 )