Update (#918)

Prepare to use copybara worklow.

.gitmodules

@@ -1,6 +0,0 @@
-[submodule "research/esaxx"]
-	path = research/esaxx
-	url = https://github.com/hillbig/esaxx
-[submodule "research/libdivsufsort"]
-	path = research/libdivsufsort
-	url = https://github.com/y-256/libdivsufsort.git

.travis.yml

@@ -166,9 +166,9 @@ matrix:
     - os: osx
       osx_image: xcode12.2
       env: BUILD_SYSTEM=cmake C_COMPILER=gcc CXX_COMPILER=g++
-    - os: osx
-      osx_image: xcode12.2
-      env: BUILD_SYSTEM=cmake C_COMPILER=gcc-4.9 CXX_COMPILER=g++-4.9
+    #- os: osx
+    #  osx_image: xcode12.2
+    #  env: BUILD_SYSTEM=cmake C_COMPILER=gcc-4.9 CXX_COMPILER=g++-4.9
     - os: osx
       osx_image: xcode12.2
       env: BUILD_SYSTEM=cmake

BUILD

@@ -1,6 +1,8 @@
 # Description:
 #   Brotli is a generic-purpose lossless compression algorithm.
 
+load(":compiler_config_setting.bzl", "create_msvc_config")
+
 package(
     default_visibility = ["//visibility:public"],
 )
@@ -39,8 +41,6 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
-load(":compiler_config_setting.bzl", "create_msvc_config")
-
 create_msvc_config()
 
 STRICT_C_OPTIONS = select({

c/enc/encode.c

@@ -1238,7 +1238,7 @@ static size_t WriteMetadataHeader(
   return (storage_ix + 7u) >> 3;
 }
 
-static BROTLI_BOOL BrotliCompressBufferQuality10(
+static BROTLI_NOINLINE BROTLI_BOOL BrotliCompressBufferQuality10(
     int lgwin, size_t input_size, const uint8_t* input_buffer,
     size_t* encoded_size, uint8_t* encoded_buffer) {
   MemoryManager* m =

c/enc/params.h

@@ -40,7 +40,7 @@ typedef struct BrotliEncoderParams {
   BROTLI_BOOL large_window;
   BrotliHasherParams hasher;
   BrotliDistanceParams dist;
-  /* TODO(eustas): rename to BrotliShared... */
+  /* TODO: rename to BrotliShared... */
   SharedEncoderDictionary dictionary;
 } BrotliEncoderParams;
 

c/include/brotli/port.h

@@ -247,9 +247,10 @@
 #define BROTLI_PUBLIC
 #endif
 
-#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \
-    !defined(__STDC_NO_VLA__) && !defined(__cplusplus) &&         \
-    !defined(__PGI) && !defined(__PGIC__) && !defined(__TINYC__)
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) &&   \
+    !defined(__STDC_NO_VLA__) && !defined(__cplusplus) &&           \
+    !defined(__PGI) && !defined(__PGIC__) && !defined(__TINYC__) && \
+    !defined(__clang__)
 #define BROTLI_ARRAY_PARAM(name) (name)
 #else
 #define BROTLI_ARRAY_PARAM(name)

go/cbrotli/BUILD

@@ -1,9 +1,9 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
 package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])  # MIT
 
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
-
 go_library(
     name = "cbrotli",
     srcs = [

java/org/brotli/dec/BUILD

@@ -1,6 +1,8 @@
 # Description:
 #   Java port of Brotli decoder.
 
+load(":build_defs.bzl", "brotli_java_test")
+
 package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])  # MIT
@@ -14,14 +16,15 @@ java_library(
     name = "dec",
     srcs = glob(
         ["*.java"],
-        exclude = ["*Test*.java"],
+        exclude = [
+            "Decoder.java",
+            "*Test*.java",
+        ],
     ),
     proguard_specs = ["proguard.pgcfg"],
     resource_jars = ["//:license"],
 )
 
-load(":build_defs.bzl", "brotli_java_test")
-
 brotli_java_test(
     name = "BitReaderTest",
     srcs = ["BitReaderTest.java"],

java/org/brotli/dec/Decoder.java

@@ -0,0 +1,61 @@
+package org.brotli.dec;
+
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+public class Decoder {
+  private static long decodeBytes(InputStream input, OutputStream output, byte[] buffer)
+      throws IOException {
+    long totalOut = 0;
+    int readBytes;
+    BrotliInputStream in = new BrotliInputStream(input);
+    in.enableLargeWindow();
+    try {
+      while ((readBytes = in.read(buffer)) >= 0) {
+        output.write(buffer, 0, readBytes);
+        totalOut += readBytes;
+      }
+    } finally {
+      in.close();
+    }
+    return totalOut;
+  }
+
+  public static void main(String... args) throws IOException {
+    if (args.length != 2) {
+      System.out.println("Usage: decoder <compressed_in> <decompressed_out>");
+      return;
+    }
+
+    byte[] buffer = new byte[1024 * 1024];
+    long start;
+    long bytesDecoded;
+    long end;
+    InputStream in = null;
+    OutputStream out = null;
+    try {
+      in = new FileInputStream(args[0]);
+      out = new FileOutputStream(args[1]);
+      start = System.nanoTime();
+      bytesDecoded = decodeBytes(in, out, buffer);
+      end = System.nanoTime();
+    } finally {
+      if (in != null) {
+        in.close();  // Hopefully, does not throw exception.
+      }
+      if (out != null) {
+        out.close();
+      }
+    }
+
+    double timeDelta = (end - start) / 1000000000.0;
+    if (timeDelta <= 0) {
+      return;
+    }
+    double mbDecoded = bytesDecoded / (1024.0 * 1024.0);
+    System.out.println(mbDecoded / timeDelta + " MiB/s");
+  }
+}

java/org/brotli/integration/Benchmark.java

@@ -0,0 +1,117 @@
+package org.brotli.integration;
+
+import org.brotli.dec.BrotliInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+
+/**
+ * Measures decompression speed on the given corpus.
+ */
+public class Benchmark {
+  private static byte[] readFile(String fileName) throws IOException {
+    int bufferLength = 65536;
+    byte[] buffer = new byte[bufferLength];
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    FileInputStream fin = new FileInputStream(fileName);
+    try {
+      int bytesRead;
+      while ((bytesRead = fin.read(buffer)) >= 0) {
+        baos.write(buffer, 0, bytesRead);
+      }
+    } finally {
+      fin.close();
+    }
+    return baos.toByteArray();
+  }
+  
+  private static long decodeBytes(InputStream input, OutputStream output, byte[] buffer)
+      throws IOException {
+    long totalOut = 0;
+    int readBytes;
+    BrotliInputStream in = new BrotliInputStream(input);
+    try {
+      while ((readBytes = in.read(buffer)) >= 0) {
+        output.write(buffer, 0, readBytes);
+        totalOut += readBytes;
+      }
+    } finally {
+      in.close();
+    }
+    return totalOut;
+  }
+
+  public static void main(String... args) throws IOException {
+    if (args.length == 0) {
+      System.out.println("Usage: benchmark <corpus>");
+      return;
+    }
+    File corpusDir = new File(args[0]);
+    ArrayList<String> fileNames = new ArrayList<String>();
+    File[] filesList = corpusDir.listFiles();
+    if (filesList == null) {
+      System.out.println("'" + args[0] + "' is not a directory");
+      return;
+    }
+    for (File file : filesList) {
+      if (!file.isFile()) {
+        continue;
+      }
+      String fileName = file.getAbsolutePath();
+      if (!fileName.endsWith(".brotli")) {
+        continue;
+      }
+      fileNames.add(fileName);
+    }
+    int fileCount = fileNames.size();
+    if (fileCount == 0) {
+      System.out.println("No files found");
+      return;
+    }
+    byte[][] files = new byte[fileCount][];
+    for (int i = 0; i < fileCount; ++i) {
+      files[i] = readFile(fileNames.get(i));
+    }
+
+    ByteArrayOutputStream baos = new ByteArrayOutputStream(65536);
+    byte[] buffer = new byte[65536];
+
+    int warmupRepeat = 10;
+    long bytesDecoded = 0;
+    for (int i = 0; i < warmupRepeat; ++i) {
+      bytesDecoded = 0;
+      for (int j = 0; j < fileCount; ++j) {
+        baos.reset();
+        bytesDecoded += decodeBytes(new ByteArrayInputStream(files[j]), baos, buffer);
+      }
+    }
+
+    int repeat = 100;
+
+    long bestTime = Long.MAX_VALUE;
+    for (int i = 0; i < repeat; ++i) {
+      long start = System.nanoTime();
+      for (int j = 0; j < fileCount; ++j) {
+        baos.reset();
+        decodeBytes(new ByteArrayInputStream(files[j]), baos, buffer);
+      }
+      long end = System.nanoTime();
+      long timeDelta = end - start;
+      if (timeDelta < bestTime) {
+        bestTime = timeDelta;
+      }
+    }
+
+    double timeDeltaSeconds = bestTime / 1000000000.0;
+    if (timeDeltaSeconds <= 0) {
+      return;
+    }
+    double mbDecoded = bytesDecoded / (1024.0 * 1024.0);
+    System.out.println(mbDecoded / timeDeltaSeconds);
+  }
+}

java/org/brotli/wrapper/android/JniHelper.java

@@ -0,0 +1,83 @@
+package org.brotli.wrapper.android;
+
+import android.content.Context;
+import android.os.Build;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.zip.ZipFile;
+
+public class JniHelper {
+
+  // Should be set on application start.
+  static Context context = null;
+
+  private static final String LIB_NAME = "native";
+
+  private static void tryInitialize() {
+    try {
+      System.loadLibrary(LIB_NAME);
+    } catch (UnsatisfiedLinkError e) {
+      if (context == null) {
+        throw e;
+      }
+      int sdk = Build.VERSION.SDK_INT;
+      boolean tryFallback =
+          (sdk >= Build.VERSION_CODES.JELLY_BEAN) && (sdk <= Build.VERSION_CODES.KITKAT);
+      if (!tryFallback) {
+        throw e;
+      }
+      try {
+        String libraryFileName = "lib" + LIB_NAME + ".so";
+        String libraryFullPath = context.getFilesDir() + File.separator + libraryFileName;
+        File file = new File(libraryFullPath);
+        if (!file.exists()) {
+          String apkPath = context.getApplicationInfo().sourceDir;
+          String pathInApk = "lib/" + Build.CPU_ABI + "/lib" + LIB_NAME + ".so";
+          unzip(apkPath, pathInApk, libraryFullPath);
+        }
+        System.load(libraryFullPath);
+      } catch (UnsatisfiedLinkError unsatisfiedLinkError) {
+        throw unsatisfiedLinkError;
+      } catch (Throwable t) {
+        UnsatisfiedLinkError unsatisfiedLinkError = new UnsatisfiedLinkError(
+            "Exception while extracting native library: " + t.getMessage());
+        unsatisfiedLinkError.initCause(t);
+        throw unsatisfiedLinkError;
+      }
+    }
+  }
+
+  private static final Object mutex = new Object();
+  private static volatile boolean alreadyInitialized;
+
+  public static void ensureInitialized() {
+    synchronized (mutex) {
+      if (!alreadyInitialized) {
+        // If failed, do not retry.
+        alreadyInitialized = true;
+        tryInitialize();
+      }
+    }
+  }
+
+  private static void unzip(String zipFileName, String entryName, String outputFileName)
+      throws IOException {
+    ZipFile zipFile = new ZipFile(zipFileName);
+    try {
+      InputStream input = zipFile.getInputStream(zipFile.getEntry(entryName));
+      OutputStream output = new FileOutputStream(outputFileName);
+      byte[] data = new byte[16384];
+      int len;
+      while ((len = input.read(data)) != -1) {
+        output.write(data, 0, len);
+      }
+      output.close();
+      input.close();
+    } finally {
+      zipFile.close();
+    }
+  }
+}

java/org/brotli/wrapper/android/UseJni.java

@@ -0,0 +1,22 @@
+package org.brotli.wrapper.android;
+
+import org.brotli.wrapper.dec.Decoder;
+import org.brotli.wrapper.enc.Encoder;
+import java.io.IOException;
+
+public final class UseJni {
+
+  static {
+    JniHelper.ensureInitialized();
+  }
+
+  public static int deepThought() {
+    String theUltimateQuestion = "What do you get when you multiply six by 9";
+    try {
+        return Decoder.decompress(
+            Encoder.compress(new byte[theUltimateQuestion.length()])).length;
+    } catch (IOException ex) {
+        throw new RuntimeException("Please wait 7.5 million years");
+    }
+  }
+}

java/org/brotli/wrapper/android/UseJniTest.java

@@ -0,0 +1,23 @@
+package org.brotli.wrapper.android;
+
+import static junit.framework.Assert.assertEquals;
+
+import androidx.test.core.app.ApplicationProvider;
+import com.google.thirdparty.robolectric.GoogleRobolectricTestRunner;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+@RunWith(GoogleRobolectricTestRunner.class)
+public final class UseJniTest {
+
+  @Before
+  public void setup() {
+    JniHelper.context = ApplicationProvider.getApplicationContext();
+  }
+
+  @Test
+  public void testAnswerToTheUltimateQuestionOfLifeTheUniverseAndEverything() {
+    assertEquals(42, UseJni.deepThought());
+  }
+}

java/org/brotli/wrapper/dec/BUILD

@@ -4,7 +4,12 @@ licenses(["notice"])  # MIT
 
 filegroup(
     name = "jni_src",
-    srcs = ["decoder_jni.cc"],
+    srcs = [
+        "decoder_jni.cc",
+        "decoder_jni.h",
+        # TODO: investigate, why this prevents JNI library loading.
+        #"decoder_jni_onload.cc",
+    ],
 )
 
 java_library(

java/org/brotli/wrapper/dec/decoder_jni.cc

@@ -4,7 +4,7 @@
    See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
 */
 
-#include <jni.h>
+#include "./decoder_jni.h"
 
 #include <new>
 

java/org/brotli/wrapper/dec/decoder_jni.h

@@ -0,0 +1,75 @@
+/* Copyright 2017 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+#ifndef BROTLI_WRAPPER_DEC_DECODER_JNI_H_
+#define BROTLI_WRAPPER_DEC_DECODER_JNI_H_
+
+#include <jni.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Creates a new Decoder.
+ *
+ * Cookie to address created decoder is stored in out_cookie. In case of failure
+ * cookie is 0.
+ *
+ * @param ctx {out_cookie, in_directBufferSize} tuple
+ * @returns direct ByteBuffer if directBufferSize is not 0; otherwise null
+ */
+JNIEXPORT jobject JNICALL
+Java_org_brotli_wrapper_dec_DecoderJNI_nativeCreate(
+    JNIEnv* env, jobject /*jobj*/, jlongArray ctx);
+
+/**
+ * Push data to decoder.
+ *
+ * status codes:
+ *  - 0 error happened
+ *  - 1 stream is finished, no more input / output expected
+ *  - 2 needs more input to process further
+ *  - 3 needs more output to process further
+ *  - 4 ok, can proceed further without additional input
+ *
+ * @param ctx {in_cookie, out_status} tuple
+ * @param input_length number of bytes provided in input or direct input;
+ *                     0 to process further previous input
+ */
+JNIEXPORT void JNICALL
+Java_org_brotli_wrapper_dec_DecoderJNI_nativePush(
+    JNIEnv* env, jobject /*jobj*/, jlongArray ctx, jint input_length);
+
+/**
+ * Pull decompressed data from decoder.
+ *
+ * @param ctx {in_cookie, out_status} tuple
+ * @returns direct ByteBuffer; all the produced data MUST be consumed before
+ *          any further invocation; null in case of error
+ */
+JNIEXPORT jobject JNICALL
+Java_org_brotli_wrapper_dec_DecoderJNI_nativePull(
+    JNIEnv* env, jobject /*jobj*/, jlongArray ctx);
+
+/**
+ * Releases all used resources.
+ *
+ * @param ctx {in_cookie} tuple
+ */
+JNIEXPORT void JNICALL
+Java_org_brotli_wrapper_dec_DecoderJNI_nativeDestroy(
+    JNIEnv* env, jobject /*jobj*/, jlongArray ctx);
+
+JNIEXPORT jboolean JNICALL
+Java_org_brotli_wrapper_dec_DecoderJNI_nativeAttachDictionary(
+    JNIEnv* env, jobject /*jobj*/, jlongArray ctx, jobject dictionary);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // BROTLI_WRAPPER_DEC_DECODER_JNI_H_

java/org/brotli/wrapper/dec/decoder_jni_onload.cc

@@ -0,0 +1,55 @@
+/* Copyright 2017 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+
+#include <jni.h>
+
+#include "./decoder_jni.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+static const JNINativeMethod kDecoderMethods[] = {
+    {"nativeCreate", "([J)Ljava/nio/ByteBuffer;",
+     reinterpret_cast<void*>(
+         Java_org_brotli_wrapper_dec_DecoderJNI_nativeCreate)},
+    {"nativePush", "([JI)V",
+     reinterpret_cast<void*>(
+         Java_org_brotli_wrapper_dec_DecoderJNI_nativePush)},
+    {"nativePull", "([J)Ljava/nio/ByteBuffer;",
+     reinterpret_cast<void*>(
+         Java_org_brotli_wrapper_dec_DecoderJNI_nativePull)},
+    {"nativeDestroy", "([J)V",
+     reinterpret_cast<void*>(
+         Java_org_brotli_wrapper_dec_DecoderJNI_nativeDestroy)},
+    {"nativeAttachDictionary", "([JLjava/nio/ByteBuffer;)Z",
+     reinterpret_cast<void*>(
+         Java_org_brotli_wrapper_dec_DecoderJNI_nativeAttachDictionary)}};
+
+JNIEXPORT jint JNI_OnLoad(JavaVM* vm, void* reserved) {
+  JNIEnv* env;
+  if (vm->GetEnv(reinterpret_cast<void**>(&env), JNI_VERSION_1_6) != JNI_OK) {
+    return -1;
+  }
+
+  jclass clazz =
+      env->FindClass("com/google/compression/brotli/wrapper/dec/DecoderJNI");
+  if (clazz == nullptr) {
+    return -1;
+  }
+
+  if (env->RegisterNatives(
+          clazz, kDecoderMethods,
+          sizeof(kDecoderMethods) / sizeof(kDecoderMethods[0])) < 0) {
+    return -1;
+  }
+
+  return JNI_VERSION_1_6;
+}
+
+#ifdef __cplusplus
+}
+#endif

java/org/brotli/wrapper/enc/Encoder.java

@@ -69,6 +69,8 @@ public class Encoder {
     }
 
     /**
+     * Setup encoder quality.
+     *
      * @param quality compression quality, or -1 for default
      */
     public Parameters setQuality(int quality) {
@@ -80,6 +82,8 @@ public class Encoder {
     }
 
     /**
+     * Setup encoder window size.
+     *
      * @param lgwin log2(LZ window size), or -1 for default
      */
     public Parameters setWindow(int lgwin) {
@@ -91,6 +95,8 @@ public class Encoder {
     }
 
     /**
+     * Setup encoder compression mode.
+     *
      * @param mode compression mode, or {@code null} for default
      */
     public Parameters setMode(Mode mode) {
@@ -116,7 +122,8 @@ public class Encoder {
     }
     this.dictionaries = new ArrayList<PreparedDictionary>();
     this.destination = destination;
-    this.encoder = new EncoderJNI.Wrapper(inputBufferSize, params.quality, params.lgwin, params.mode);
+    this.encoder =
+        new EncoderJNI.Wrapper(inputBufferSize, params.quality, params.lgwin, params.mode);
     this.inputBuffer = this.encoder.getInputBuffer();
   }
 
@@ -212,7 +219,8 @@ public class Encoder {
       return empty;
     }
     /* data.length > 0 */
-    EncoderJNI.Wrapper encoder = new EncoderJNI.Wrapper(data.length, params.quality, params.lgwin, params.mode);
+    EncoderJNI.Wrapper encoder =
+        new EncoderJNI.Wrapper(data.length, params.quality, params.lgwin, params.mode);
     ArrayList<byte[]> output = new ArrayList<byte[]>();
     int totalOutputSize = 0;
     try {

js/BUILD

@@ -1,11 +1,11 @@
+load("@io_bazel_rules_closure//closure:defs.bzl", "closure_js_library", "closure_js_test")
+
 package(
     default_visibility = ["//visibility:public"],
 )
 
 licenses(["notice"])  # MIT
 
-load("@io_bazel_rules_closure//closure:defs.bzl", "closure_js_library")
-
 # Not a real polyfill. Do NOT use for anything, but tests.
 closure_js_library(
     name = "polyfill",
@@ -27,8 +27,6 @@ closure_js_library(
     deps = [":polyfill"],
 )
 
-load("@io_bazel_rules_closure//closure:defs.bzl", "closure_js_test")
-
 closure_js_test(
     name = "all_tests",
     srcs = ["decode_test.js"],

js/bundle_test.js

@@ -0,0 +1,72 @@
+/* Copyright 2017 Google Inc. All Rights Reserved.
+
+   Distributed under MIT license.
+   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
+*/
+import {BrotliDecode} from "./decode.js";
+import {makeTestData} from "./test_data.js";
+goog.require('goog.testing.asserts');
+const testSuite = goog.require('goog.testing.testSuite');
+
+const CRC_64_POLY = new Uint32Array([0xD7870F42, 0xC96C5795]);
+
+/**
+ * Calculates binary data footprint.
+ *
+ * @param {!Int8Array} data binary data
+ * @return {string} footprint
+ */
+function calculateCrc64(data) {
+  let crc = new Uint32Array([0xFFFFFFFF, 0xFFFFFFFF]);
+  let c = new Uint32Array(2);
+  for (let i = 0; i < data.length; ++i) {
+    c[1] = 0;
+    c[0] = (crc[0] ^ data[i]) & 0xFF;
+    for (let k = 0; k < 8; ++k) {
+      const isOdd = c[0] & 1;
+      c[0] = (c[0] >>> 1) | ((c[1] & 1) << 31);
+      c[1] = c[1] >>> 1;
+      if (isOdd) {
+        c[0] = c[0] ^ CRC_64_POLY[0];
+        c[1] = c[1] ^ CRC_64_POLY[1];
+      }
+    }
+    crc[0] = ((crc[0] >>> 8) | ((crc[1] & 0xFF) << 24)) ^ c[0];
+    crc[1] = (crc[1] >>> 8) ^ c[1];
+  }
+  crc[0] = ~crc[0];
+  crc[1] = ~crc[1];
+
+  let lo = crc[0].toString(16);
+  lo = "0".repeat(8 - lo.length) + lo;
+  let hi = crc[1].toString(16);
+  hi = "0".repeat(8 - hi.length) + hi;
+
+  return hi + lo;
+}
+
+/**
+ * Decompresses data and checks that output footprint is correct.
+ *
+ * @param {string} entry filename including footprint prefix
+ * @param {!Int8Array} data compressed data
+ */
+function checkEntry(entry, data) {
+  const expectedCrc = entry.substring(0, 16);
+  const decompressed = BrotliDecode(data);
+  const crc = calculateCrc64(decompressed);
+  assertEquals(expectedCrc, crc);
+}
+
+let allTests = {};
+const testData = makeTestData();
+for (let entry in testData) {
+  if (!testData.hasOwnProperty(entry)) {
+    continue;
+  }
+  const name = entry.substring(17);
+  const data = testData[entry];
+  allTests['test_' + name] = checkEntry.bind(null, entry, data);
+}
+
+testSuite(allTests);

js/cli.js

@@ -0,0 +1,59 @@
+#!/usr/bin/env node
+
+let defaults =
+    {input: 'input.br', output: 'output.txt', test_iters: 0, test_repeat: 100};
+
+/* Parse command line arguments. */
+let argv =
+    require('yargs')
+        .usage('Usage: $0 -i file -o file')
+        .option(
+            'input',
+            {alias: 'i', default: defaults.input, describe: 'compressed file'})
+        .option('output', {
+          alias: 'o',
+          default: defaults.output,
+          describe: 'decompressed file'
+        })
+        .option('test_iters', {
+          default: defaults.test_iters,
+          describe: '# of times to run performance test'
+        })
+        .option('test_repeat', {
+          default: defaults.test_repeat,
+          describe: '# of times to decompress file in performance test'
+        })
+        .argv;
+
+/* Read input. */
+const fs = require('fs');
+data = fs.readFileSync(argv.input);
+if (!Buffer.isBuffer(data)) throw 'not a buffer';
+const bytes = new Uint8Array(data);
+
+/* Load and map brotli decoder module. */
+global.window = {};
+require('./decode.js')
+const brotliDecode = window['BrotliDecode'];
+
+/* Load "performance" module. */
+const {PerformanceObserver, performance} = require('perf_hooks');
+
+/* Performance test. */
+for (let i = 0; i < argv.test_iters; ++i) {
+  const a = performance.now();
+  let result;
+  for (let j = 0; j < argv.test_repeat; ++j) {
+    result = brotliDecode(bytes);
+  }
+  const b = performance.now();
+  const total_length = argv.test_repeat * result.length / (1024 * 1024);
+  const total_time = (b - a) / 1000;
+
+  console.log(
+      total_length + 'MB / ' + total_time +
+      's = ' + (total_length / total_time) + 'MB/s');
+}
+
+/* Decode and write output file. */
+fs.writeFileSync(argv.output, new Buffer(brotliDecode(bytes)));

js/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "brotli",
+  "description": "Pure JavaScript Brotli implementation",
+  "bin": {
+    "brotli": "cli.js"
+  },
+  "files": [
+    "cli.js",
+    "decode.js",
+  ],
+  "dependencies": {
+    "yargs": "~8.0.2"
+  },
+}

python/brotli.py

@@ -7,7 +7,6 @@
 
 import _brotli
 
-
 # The library version.
 __version__ = _brotli.__version__
 

research/BUILD

@@ -7,10 +7,8 @@ licenses(["notice"])  # MIT
 cc_library(
     name = "dm",
     srcs = ["deorummolae.cc"],
-    hdrs = [
-        "deorummolae.h",
-        "esaxx/sais.hxx",
-    ],
+    deps = ["@esaxx//:sais"],
+    hdrs = ["deorummolae.h"],
 )
 
 cc_library(

research/BUILD.libdivsufsort

@@ -19,6 +19,7 @@ cc_library(
         "-DHAVE_CONFIG_H=1",
     ],
     includes = ["include"],
+    include_prefix = "third_party/libdivsufsort",
 )
 
 commom_awk_replaces = (

research/Makefile

@@ -1,17 +0,0 @@
-CC = g++
-CFLAGS += -O2
-CPPFLAGS += -std=c++11
-SOURCES = $(wildcard *.cc)
-EXECUTABLES = $(SOURCES:.cc=)
-BINDIR = bin
-
-all: $(EXECUTABLES)
-
-$(BINDIR):
-	mkdir -p $@
-
-$(EXECUTABLES): $(BINDIR)
-	$(CC) $(CFLAGS) $(CPPFLAGS) $(addsuffix .cc, $@) -o $(BINDIR)/$@ -lgflags_nothreads
-
-clean:
-	rm -rf $(BINDIR)

research/WORKSPACE

@@ -1,12 +1,30 @@
 workspace(name = "org_brotli_research")
 
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository")
+
 local_repository(
     name = "org_brotli",
     path = "..",
 )
 
-new_local_repository(
+new_git_repository(
     name = "divsufsort",
-    build_file = "BUILD.libdivsufsort",
-    path = "libdivsufsort",
+    build_file = "@//:BUILD.libdivsufsort",
+    commit = "5f60d6f026c30fb4ac296f696b3c8b0eb71bd428",
+    remote = "https://github.com/y-256/libdivsufsort",
+)
+
+new_git_repository(
+    name = "esaxx",
+    build_file_content = """
+package(default_visibility = ["//visibility:public"])
+cc_library(
+    name = "sais",
+    hdrs = ["sais.hxx"],
+    includes = ["."],
+    include_prefix = "third_party/esaxx",
+)
+""",
+    commit = "ca7cb332011ec37a8436487f210f396b84bd8273",
+    remote = "https://github.com/hillbig/esaxx",
 )

research/deorummolae.cc

@@ -3,7 +3,7 @@
 #include <array>
 #include <cstdio>
 
-#include "./esaxx/sais.hxx"
+#include "third_party/esaxx/sais.hxx"
 
 /* Used for quick SA-entry to file mapping. Each file is padded to size that
    is a multiple of chunk size. */

research/draw_histogram.cc

@@ -18,6 +18,7 @@
 #include <gflags/gflags.h>
 using gflags::ParseCommandLineFlags;
 
+#include "third_party/absl/flags/flag.h"
 #include "./read_dist.h"
 
 DEFINE_int32(height, 1000, "Height of the resulting histogam.");
@@ -32,7 +33,7 @@ DEFINE_bool(linear, false, "True if using linear distance mapping.");
 DEFINE_uint64(skip, 0, "Number of bytes to skip.");
 
 inline double DistanceTransform(double x) {
-  static bool linear = FLAGS_linear;
+  static bool linear = absl::GetFlag(FLAGS_linear);
   if (linear) {
     return x;
   } else {
@@ -48,14 +49,12 @@ inline double DensityTransform(double x) {
   return sqrt(255 * 255 - z * z);
 }
 
-inline int GetMaxDistance() {
-  return FLAGS_max_distance;
-}
+inline int GetMaxDistance() { return absl::GetFlag(FLAGS_max_distance); }
 
 void AdjustPosition(int* pos) {
   static uint32_t offset = 0;
   static int last = 0;
-  static uint32_t window_size = (1 << FLAGS_brotli_window);
+  static uint32_t window_size = (1 << absl::GetFlag(FLAGS_brotli_window));
   assert(*pos >= 0 && *pos < window_size);
   if (*pos < last) {
     offset += window_size;
@@ -65,10 +64,10 @@ void AdjustPosition(int* pos) {
 }
 
 void BuildHistogram(FILE* fin, int** histo) {
-  int height = FLAGS_height;
-  int width = FLAGS_width;
-  int skip = FLAGS_skip;
-  size_t min_distance = FLAGS_min_distance;
+  int height = absl::GetFlag(FLAGS_height);
+  int width = absl::GetFlag(FLAGS_width);
+  int skip = absl::GetFlag(FLAGS_skip);
+  size_t min_distance = absl::GetFlag(FLAGS_min_distance);
 
   printf("height = %d, width = %d\n", height, width);
 
@@ -78,7 +77,7 @@ void BuildHistogram(FILE* fin, int** histo) {
     }
   }
 
-  int max_pos = FLAGS_size - skip;
+  int max_pos = absl::GetFlag(FLAGS_size) - skip;
   double min_dist = min_distance > 0 ? DistanceTransform(min_distance) : 0;
   double max_dist = DistanceTransform(GetMaxDistance()) - min_dist;
   int copy, pos, distance, x, y;
@@ -86,7 +85,7 @@ void BuildHistogram(FILE* fin, int** histo) {
   while (ReadBackwardReference(fin, &copy, &pos, &distance)) {
     if (pos == -1) continue;  // In case when only insert is present.
     if (distance < min_distance || distance >= GetMaxDistance()) continue;
-    if (FLAGS_brotli_window != -1) {
+    if (absl::GetFlag(FLAGS_brotli_window) != -1) {
       AdjustPosition(&pos);
     }
     if (pos >= skip && distance <= pos) {
@@ -102,7 +101,7 @@ void BuildHistogram(FILE* fin, int** histo) {
         assert(y >= 0 && y < width);
       }
 
-      if (FLAGS_with_copies) {
+      if (absl::GetFlag(FLAGS_with_copies)) {
         int right = 1ul * (pos + copy - 1) * width / max_pos;
         if (right < 0) {
           printf("pos = %d, distance = %d, copy = %d, y = %d, right = %d\n",
@@ -131,8 +130,8 @@ void BuildHistogram(FILE* fin, int** histo) {
 }
 
 void ConvertToPixels(int** histo, uint8_t** pixel) {
-  int height = FLAGS_height;
-  int width = FLAGS_width;
+  int height = absl::GetFlag(FLAGS_height);
+  int width = absl::GetFlag(FLAGS_width);
 
   int maxs = 0;
   for (int i = 0; i < height; i++) {
@@ -141,7 +140,7 @@ void ConvertToPixels(int** histo, uint8_t** pixel) {
     }
   }
 
-  bool simple = FLAGS_simple;
+  bool simple = absl::GetFlag(FLAGS_simple);
   double max_histo = static_cast<double>(maxs);
   for (int i = 0; i < height; i++) {
     for (int j = 0; j < width; j++) {
@@ -156,8 +155,8 @@ void ConvertToPixels(int** histo, uint8_t** pixel) {
 }
 
 void DrawPixels(uint8_t** pixel, FILE* fout) {
-  int height = FLAGS_height;
-  int width = FLAGS_width;
+  int height = absl::GetFlag(FLAGS_height);
+  int width = absl::GetFlag(FLAGS_width);
 
   fprintf(fout, "P5\n%d %d\n255\n", width, height);
   for (int i = height - 1; i >= 0; i--) {
@@ -166,14 +165,14 @@ void DrawPixels(uint8_t** pixel, FILE* fout) {
 }
 
 int main(int argc, char* argv[]) {
-  ParseCommandLineFlags(&argc, &argv, true);
+  base::ParseCommandLine(&argc, &argv);
   if (argc != 3) {
     printf("usage: draw_histogram.cc data output_file\n");
     return 1;
   }
 
-  int height = FLAGS_height;
-  int width = FLAGS_width;
+  int height = absl::GetFlag(FLAGS_height);
+  int width = absl::GetFlag(FLAGS_width);
 
   FILE* fin = fopen(argv[1], "r");
   FILE* fout = fopen(argv[2], "wb");

research/durchschlag.cc

@@ -3,7 +3,7 @@
 #include <algorithm>
 #include <exception>  /* terminate */
 
-#include "divsufsort.h"
+#include "third_party/libdivsufsort/include/divsufsort.h"
 
 /* Pointer to position in text. */
 typedef DurchschlagTextIdx TextIdx;

research/esaxx

@@ -1 +0,0 @@
-Subproject commit ca7cb332011ec37a8436487f210f396b84bd8273

research/find_opt_references.cc

@@ -19,7 +19,8 @@
 #include <gflags/gflags.h>
 using gflags::ParseCommandLineFlags;
 
-#include "./esaxx/sais.hxx"
+#include "third_party/absl/flags/flag.h"
+#include "third_party/esaxx/sais.hxx"
 
 DEFINE_bool(advanced, false, "Advanced searching mode: finds all longest "
     "matches at positions that are not covered by matches of length at least "
@@ -91,7 +92,7 @@ inline void PrintReference(sarray_type* sarray, lcp_type* lcp, size_t size,
 inline void GoLeft(sarray_type* sarray, lcp_type* lcp, int idx, int left_ix,
                    int left_lcp, entry_type* entry) {
   entry->first = left_lcp;
-  if (left_lcp > FLAGS_long_length) return;
+  if (left_lcp > absl::GetFlag(FLAGS_long_length)) return;
   for (; left_ix >= 0; --left_ix) {
     if (lcp[left_ix] < left_lcp) break;
     if (sarray[left_ix] < idx) {
@@ -103,7 +104,7 @@ inline void GoLeft(sarray_type* sarray, lcp_type* lcp, int idx, int left_ix,
 inline void GoRight(sarray_type* sarray, lcp_type* lcp, int idx, size_t size,
                     int right_ix, int right_lcp, entry_type* entry) {
   entry->first = right_lcp;
-  if (right_lcp > FLAGS_long_length) return;
+  if (right_lcp > absl::GetFlag(FLAGS_long_length)) return;
   for (; right_ix < size - 1; ++right_ix) {
     if (lcp[right_ix] < right_lcp) break;
     if (sarray[right_ix] < idx) {
@@ -129,8 +130,8 @@ inline void StoreReference(sarray_type* sarray, lcp_type* lcp, size_t size,
 
 void ProcessReferences(sarray_type* sarray, lcp_type* lcp, size_t size,
                        uint32_t* pos, const Fn& process_output) {
-  int min_length = FLAGS_min_length;
-  for (int idx = FLAGS_skip; idx < size; ++idx) {
+  int min_length = absl::GetFlag(FLAGS_min_length);
+  for (int idx = absl::GetFlag(FLAGS_skip); idx < size; ++idx) {
     int left_lcp = -1;
     int left_ix;
     for (left_ix = pos[idx] - 1; left_ix >= 0; --left_ix) {
@@ -162,7 +163,7 @@ void ProcessReferences(sarray_type* sarray, lcp_type* lcp, size_t size,
 }
 
 void ProcessEntries(entry_type* entries, size_t size, FILE* fout) {
-  int long_length = FLAGS_long_length;
+  int long_length = absl::GetFlag(FLAGS_long_length);
   std::vector<std::pair<int, int> > segments;
   size_t idx;
   for (idx = 0; idx < size;) {
@@ -195,7 +196,7 @@ void ProcessEntries(entry_type* entries, size_t size, FILE* fout) {
 }
 
 int main(int argc, char* argv[]) {
-  ParseCommandLineFlags(&argc, &argv, true);
+  base::ParseCommandLine(&argc, &argv);
   if (argc != 3) {
     printf("usage: %s input_file output_file\n", argv[0]);
     return 1;
@@ -235,7 +236,7 @@ int main(int argc, char* argv[]) {
   using std::placeholders::_7;
   using std::placeholders::_8;
   entry_type* entries;
-  if (FLAGS_advanced) {
+  if (absl::GetFlag(FLAGS_advanced)) {
     entries = new entry_type[input_size];
     for (size_t i = 0; i < input_size; ++i) entries[i].first = -1;
   }
@@ -243,10 +244,10 @@ int main(int argc, char* argv[]) {
   Fn store = std::bind(StoreReference, _1, _2, _3, _4, _5, _6, _7, _8, entries);
 
   ProcessReferences(sarray, lcp, input_size, pos,
-                    FLAGS_advanced ? store : print);
+                    absl::GetFlag(FLAGS_advanced) ? store : print);
   printf("References processed.\n");
 
-  if (FLAGS_advanced) {
+  if (absl::GetFlag(FLAGS_advanced)) {
     int good_cnt = 0;
     uint64_t avg_cnt = 0;
     for (size_t i = 0; i < input_size; ++i) {

research/libdivsufsort

@@ -1 +0,0 @@
-Subproject commit 5f60d6f026c30fb4ac296f696b3c8b0eb71bd428