Prepare to use copybara worklow.
This commit is contained in:
Eugene Kliuchnikov 2021-08-18 19:15:07 +02:00 committed by GitHub
parent 19d86fb9a6
commit 68f1b90ad0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
37 changed files with 2984 additions and 88 deletions

6
.gitmodules vendored
View File

@ -1,6 +0,0 @@
[submodule "research/esaxx"]
path = research/esaxx
url = https://github.com/hillbig/esaxx
[submodule "research/libdivsufsort"]
path = research/libdivsufsort
url = https://github.com/y-256/libdivsufsort.git

View File

@ -166,9 +166,9 @@ matrix:
- os: osx
osx_image: xcode12.2
env: BUILD_SYSTEM=cmake C_COMPILER=gcc CXX_COMPILER=g++
- os: osx
osx_image: xcode12.2
env: BUILD_SYSTEM=cmake C_COMPILER=gcc-4.9 CXX_COMPILER=g++-4.9
#- os: osx
# osx_image: xcode12.2
# env: BUILD_SYSTEM=cmake C_COMPILER=gcc-4.9 CXX_COMPILER=g++-4.9
- os: osx
osx_image: xcode12.2
env: BUILD_SYSTEM=cmake

4
BUILD
View File

@ -1,6 +1,8 @@
# Description:
# Brotli is a generic-purpose lossless compression algorithm.
load(":compiler_config_setting.bzl", "create_msvc_config")
package(
default_visibility = ["//visibility:public"],
)
@ -39,8 +41,6 @@ config_setting(
visibility = ["//visibility:public"],
)
load(":compiler_config_setting.bzl", "create_msvc_config")
create_msvc_config()
STRICT_C_OPTIONS = select({

View File

@ -1238,7 +1238,7 @@ static size_t WriteMetadataHeader(
return (storage_ix + 7u) >> 3;
}
static BROTLI_BOOL BrotliCompressBufferQuality10(
static BROTLI_NOINLINE BROTLI_BOOL BrotliCompressBufferQuality10(
int lgwin, size_t input_size, const uint8_t* input_buffer,
size_t* encoded_size, uint8_t* encoded_buffer) {
MemoryManager* m =

View File

@ -40,7 +40,7 @@ typedef struct BrotliEncoderParams {
BROTLI_BOOL large_window;
BrotliHasherParams hasher;
BrotliDistanceParams dist;
/* TODO(eustas): rename to BrotliShared... */
/* TODO: rename to BrotliShared... */
SharedEncoderDictionary dictionary;
} BrotliEncoderParams;

View File

@ -249,7 +249,8 @@
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \
!defined(__STDC_NO_VLA__) && !defined(__cplusplus) && \
!defined(__PGI) && !defined(__PGIC__) && !defined(__TINYC__)
!defined(__PGI) && !defined(__PGIC__) && !defined(__TINYC__) && \
!defined(__clang__)
#define BROTLI_ARRAY_PARAM(name) (name)
#else
#define BROTLI_ARRAY_PARAM(name)

View File

@ -1,9 +1,9 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
package(default_visibility = ["//visibility:public"])
licenses(["notice"]) # MIT
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "cbrotli",
srcs = [

View File

@ -1,6 +1,8 @@
# Description:
# Java port of Brotli decoder.
load(":build_defs.bzl", "brotli_java_test")
package(default_visibility = ["//visibility:public"])
licenses(["notice"]) # MIT
@ -14,14 +16,15 @@ java_library(
name = "dec",
srcs = glob(
["*.java"],
exclude = ["*Test*.java"],
exclude = [
"Decoder.java",
"*Test*.java",
],
),
proguard_specs = ["proguard.pgcfg"],
resource_jars = ["//:license"],
)
load(":build_defs.bzl", "brotli_java_test")
brotli_java_test(
name = "BitReaderTest",
srcs = ["BitReaderTest.java"],

View File

@ -0,0 +1,61 @@
package org.brotli.dec;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
public class Decoder {
private static long decodeBytes(InputStream input, OutputStream output, byte[] buffer)
throws IOException {
long totalOut = 0;
int readBytes;
BrotliInputStream in = new BrotliInputStream(input);
in.enableLargeWindow();
try {
while ((readBytes = in.read(buffer)) >= 0) {
output.write(buffer, 0, readBytes);
totalOut += readBytes;
}
} finally {
in.close();
}
return totalOut;
}
public static void main(String... args) throws IOException {
if (args.length != 2) {
System.out.println("Usage: decoder <compressed_in> <decompressed_out>");
return;
}
byte[] buffer = new byte[1024 * 1024];
long start;
long bytesDecoded;
long end;
InputStream in = null;
OutputStream out = null;
try {
in = new FileInputStream(args[0]);
out = new FileOutputStream(args[1]);
start = System.nanoTime();
bytesDecoded = decodeBytes(in, out, buffer);
end = System.nanoTime();
} finally {
if (in != null) {
in.close(); // Hopefully, does not throw exception.
}
if (out != null) {
out.close();
}
}
double timeDelta = (end - start) / 1000000000.0;
if (timeDelta <= 0) {
return;
}
double mbDecoded = bytesDecoded / (1024.0 * 1024.0);
System.out.println(mbDecoded / timeDelta + " MiB/s");
}
}

View File

@ -0,0 +1,117 @@
package org.brotli.integration;
import org.brotli.dec.BrotliInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
/**
* Measures decompression speed on the given corpus.
*/
public class Benchmark {
private static byte[] readFile(String fileName) throws IOException {
int bufferLength = 65536;
byte[] buffer = new byte[bufferLength];
ByteArrayOutputStream baos = new ByteArrayOutputStream();
FileInputStream fin = new FileInputStream(fileName);
try {
int bytesRead;
while ((bytesRead = fin.read(buffer)) >= 0) {
baos.write(buffer, 0, bytesRead);
}
} finally {
fin.close();
}
return baos.toByteArray();
}
private static long decodeBytes(InputStream input, OutputStream output, byte[] buffer)
throws IOException {
long totalOut = 0;
int readBytes;
BrotliInputStream in = new BrotliInputStream(input);
try {
while ((readBytes = in.read(buffer)) >= 0) {
output.write(buffer, 0, readBytes);
totalOut += readBytes;
}
} finally {
in.close();
}
return totalOut;
}
public static void main(String... args) throws IOException {
if (args.length == 0) {
System.out.println("Usage: benchmark <corpus>");
return;
}
File corpusDir = new File(args[0]);
ArrayList<String> fileNames = new ArrayList<String>();
File[] filesList = corpusDir.listFiles();
if (filesList == null) {
System.out.println("'" + args[0] + "' is not a directory");
return;
}
for (File file : filesList) {
if (!file.isFile()) {
continue;
}
String fileName = file.getAbsolutePath();
if (!fileName.endsWith(".brotli")) {
continue;
}
fileNames.add(fileName);
}
int fileCount = fileNames.size();
if (fileCount == 0) {
System.out.println("No files found");
return;
}
byte[][] files = new byte[fileCount][];
for (int i = 0; i < fileCount; ++i) {
files[i] = readFile(fileNames.get(i));
}
ByteArrayOutputStream baos = new ByteArrayOutputStream(65536);
byte[] buffer = new byte[65536];
int warmupRepeat = 10;
long bytesDecoded = 0;
for (int i = 0; i < warmupRepeat; ++i) {
bytesDecoded = 0;
for (int j = 0; j < fileCount; ++j) {
baos.reset();
bytesDecoded += decodeBytes(new ByteArrayInputStream(files[j]), baos, buffer);
}
}
int repeat = 100;
long bestTime = Long.MAX_VALUE;
for (int i = 0; i < repeat; ++i) {
long start = System.nanoTime();
for (int j = 0; j < fileCount; ++j) {
baos.reset();
decodeBytes(new ByteArrayInputStream(files[j]), baos, buffer);
}
long end = System.nanoTime();
long timeDelta = end - start;
if (timeDelta < bestTime) {
bestTime = timeDelta;
}
}
double timeDeltaSeconds = bestTime / 1000000000.0;
if (timeDeltaSeconds <= 0) {
return;
}
double mbDecoded = bytesDecoded / (1024.0 * 1024.0);
System.out.println(mbDecoded / timeDeltaSeconds);
}
}

View File

@ -0,0 +1,83 @@
package org.brotli.wrapper.android;
import android.content.Context;
import android.os.Build;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.zip.ZipFile;
public class JniHelper {
// Should be set on application start.
static Context context = null;
private static final String LIB_NAME = "native";
private static void tryInitialize() {
try {
System.loadLibrary(LIB_NAME);
} catch (UnsatisfiedLinkError e) {
if (context == null) {
throw e;
}
int sdk = Build.VERSION.SDK_INT;
boolean tryFallback =
(sdk >= Build.VERSION_CODES.JELLY_BEAN) && (sdk <= Build.VERSION_CODES.KITKAT);
if (!tryFallback) {
throw e;
}
try {
String libraryFileName = "lib" + LIB_NAME + ".so";
String libraryFullPath = context.getFilesDir() + File.separator + libraryFileName;
File file = new File(libraryFullPath);
if (!file.exists()) {
String apkPath = context.getApplicationInfo().sourceDir;
String pathInApk = "lib/" + Build.CPU_ABI + "/lib" + LIB_NAME + ".so";
unzip(apkPath, pathInApk, libraryFullPath);
}
System.load(libraryFullPath);
} catch (UnsatisfiedLinkError unsatisfiedLinkError) {
throw unsatisfiedLinkError;
} catch (Throwable t) {
UnsatisfiedLinkError unsatisfiedLinkError = new UnsatisfiedLinkError(
"Exception while extracting native library: " + t.getMessage());
unsatisfiedLinkError.initCause(t);
throw unsatisfiedLinkError;
}
}
}
private static final Object mutex = new Object();
private static volatile boolean alreadyInitialized;
public static void ensureInitialized() {
synchronized (mutex) {
if (!alreadyInitialized) {
// If failed, do not retry.
alreadyInitialized = true;
tryInitialize();
}
}
}
private static void unzip(String zipFileName, String entryName, String outputFileName)
throws IOException {
ZipFile zipFile = new ZipFile(zipFileName);
try {
InputStream input = zipFile.getInputStream(zipFile.getEntry(entryName));
OutputStream output = new FileOutputStream(outputFileName);
byte[] data = new byte[16384];
int len;
while ((len = input.read(data)) != -1) {
output.write(data, 0, len);
}
output.close();
input.close();
} finally {
zipFile.close();
}
}
}

View File

@ -0,0 +1,22 @@
package org.brotli.wrapper.android;
import org.brotli.wrapper.dec.Decoder;
import org.brotli.wrapper.enc.Encoder;
import java.io.IOException;
public final class UseJni {
static {
JniHelper.ensureInitialized();
}
public static int deepThought() {
String theUltimateQuestion = "What do you get when you multiply six by 9";
try {
return Decoder.decompress(
Encoder.compress(new byte[theUltimateQuestion.length()])).length;
} catch (IOException ex) {
throw new RuntimeException("Please wait 7.5 million years");
}
}
}

View File

@ -0,0 +1,23 @@
package org.brotli.wrapper.android;
import static junit.framework.Assert.assertEquals;
import androidx.test.core.app.ApplicationProvider;
import com.google.thirdparty.robolectric.GoogleRobolectricTestRunner;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@RunWith(GoogleRobolectricTestRunner.class)
public final class UseJniTest {
@Before
public void setup() {
JniHelper.context = ApplicationProvider.getApplicationContext();
}
@Test
public void testAnswerToTheUltimateQuestionOfLifeTheUniverseAndEverything() {
assertEquals(42, UseJni.deepThought());
}
}

View File

@ -4,7 +4,12 @@ licenses(["notice"]) # MIT
filegroup(
name = "jni_src",
srcs = ["decoder_jni.cc"],
srcs = [
"decoder_jni.cc",
"decoder_jni.h",
# TODO: investigate, why this prevents JNI library loading.
#"decoder_jni_onload.cc",
],
)
java_library(

View File

@ -4,7 +4,7 @@
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
#include <jni.h>
#include "./decoder_jni.h"
#include <new>

View File

@ -0,0 +1,75 @@
/* Copyright 2017 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
#ifndef BROTLI_WRAPPER_DEC_DECODER_JNI_H_
#define BROTLI_WRAPPER_DEC_DECODER_JNI_H_
#include <jni.h>
#ifdef __cplusplus
extern "C" {
#endif
/**
* Creates a new Decoder.
*
* Cookie to address created decoder is stored in out_cookie. In case of failure
* cookie is 0.
*
* @param ctx {out_cookie, in_directBufferSize} tuple
* @returns direct ByteBuffer if directBufferSize is not 0; otherwise null
*/
JNIEXPORT jobject JNICALL
Java_org_brotli_wrapper_dec_DecoderJNI_nativeCreate(
JNIEnv* env, jobject /*jobj*/, jlongArray ctx);
/**
* Push data to decoder.
*
* status codes:
* - 0 error happened
* - 1 stream is finished, no more input / output expected
* - 2 needs more input to process further
* - 3 needs more output to process further
* - 4 ok, can proceed further without additional input
*
* @param ctx {in_cookie, out_status} tuple
* @param input_length number of bytes provided in input or direct input;
* 0 to process further previous input
*/
JNIEXPORT void JNICALL
Java_org_brotli_wrapper_dec_DecoderJNI_nativePush(
JNIEnv* env, jobject /*jobj*/, jlongArray ctx, jint input_length);
/**
* Pull decompressed data from decoder.
*
* @param ctx {in_cookie, out_status} tuple
* @returns direct ByteBuffer; all the produced data MUST be consumed before
* any further invocation; null in case of error
*/
JNIEXPORT jobject JNICALL
Java_org_brotli_wrapper_dec_DecoderJNI_nativePull(
JNIEnv* env, jobject /*jobj*/, jlongArray ctx);
/**
* Releases all used resources.
*
* @param ctx {in_cookie} tuple
*/
JNIEXPORT void JNICALL
Java_org_brotli_wrapper_dec_DecoderJNI_nativeDestroy(
JNIEnv* env, jobject /*jobj*/, jlongArray ctx);
JNIEXPORT jboolean JNICALL
Java_org_brotli_wrapper_dec_DecoderJNI_nativeAttachDictionary(
JNIEnv* env, jobject /*jobj*/, jlongArray ctx, jobject dictionary);
#ifdef __cplusplus
}
#endif
#endif // BROTLI_WRAPPER_DEC_DECODER_JNI_H_

View File

@ -0,0 +1,55 @@
/* Copyright 2017 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
#include <jni.h>
#include "./decoder_jni.h"
#ifdef __cplusplus
extern "C" {
#endif
static const JNINativeMethod kDecoderMethods[] = {
{"nativeCreate", "([J)Ljava/nio/ByteBuffer;",
reinterpret_cast<void*>(
Java_org_brotli_wrapper_dec_DecoderJNI_nativeCreate)},
{"nativePush", "([JI)V",
reinterpret_cast<void*>(
Java_org_brotli_wrapper_dec_DecoderJNI_nativePush)},
{"nativePull", "([J)Ljava/nio/ByteBuffer;",
reinterpret_cast<void*>(
Java_org_brotli_wrapper_dec_DecoderJNI_nativePull)},
{"nativeDestroy", "([J)V",
reinterpret_cast<void*>(
Java_org_brotli_wrapper_dec_DecoderJNI_nativeDestroy)},
{"nativeAttachDictionary", "([JLjava/nio/ByteBuffer;)Z",
reinterpret_cast<void*>(
Java_org_brotli_wrapper_dec_DecoderJNI_nativeAttachDictionary)}};
JNIEXPORT jint JNI_OnLoad(JavaVM* vm, void* reserved) {
JNIEnv* env;
if (vm->GetEnv(reinterpret_cast<void**>(&env), JNI_VERSION_1_6) != JNI_OK) {
return -1;
}
jclass clazz =
env->FindClass("com/google/compression/brotli/wrapper/dec/DecoderJNI");
if (clazz == nullptr) {
return -1;
}
if (env->RegisterNatives(
clazz, kDecoderMethods,
sizeof(kDecoderMethods) / sizeof(kDecoderMethods[0])) < 0) {
return -1;
}
return JNI_VERSION_1_6;
}
#ifdef __cplusplus
}
#endif

View File

@ -69,6 +69,8 @@ public class Encoder {
}
/**
* Setup encoder quality.
*
* @param quality compression quality, or -1 for default
*/
public Parameters setQuality(int quality) {
@ -80,6 +82,8 @@ public class Encoder {
}
/**
* Setup encoder window size.
*
* @param lgwin log2(LZ window size), or -1 for default
*/
public Parameters setWindow(int lgwin) {
@ -91,6 +95,8 @@ public class Encoder {
}
/**
* Setup encoder compression mode.
*
* @param mode compression mode, or {@code null} for default
*/
public Parameters setMode(Mode mode) {
@ -116,7 +122,8 @@ public class Encoder {
}
this.dictionaries = new ArrayList<PreparedDictionary>();
this.destination = destination;
this.encoder = new EncoderJNI.Wrapper(inputBufferSize, params.quality, params.lgwin, params.mode);
this.encoder =
new EncoderJNI.Wrapper(inputBufferSize, params.quality, params.lgwin, params.mode);
this.inputBuffer = this.encoder.getInputBuffer();
}
@ -212,7 +219,8 @@ public class Encoder {
return empty;
}
/* data.length > 0 */
EncoderJNI.Wrapper encoder = new EncoderJNI.Wrapper(data.length, params.quality, params.lgwin, params.mode);
EncoderJNI.Wrapper encoder =
new EncoderJNI.Wrapper(data.length, params.quality, params.lgwin, params.mode);
ArrayList<byte[]> output = new ArrayList<byte[]>();
int totalOutputSize = 0;
try {

View File

@ -1,11 +1,11 @@
load("@io_bazel_rules_closure//closure:defs.bzl", "closure_js_library", "closure_js_test")
package(
default_visibility = ["//visibility:public"],
)
licenses(["notice"]) # MIT
load("@io_bazel_rules_closure//closure:defs.bzl", "closure_js_library")
# Not a real polyfill. Do NOT use for anything, but tests.
closure_js_library(
name = "polyfill",
@ -27,8 +27,6 @@ closure_js_library(
deps = [":polyfill"],
)
load("@io_bazel_rules_closure//closure:defs.bzl", "closure_js_test")
closure_js_test(
name = "all_tests",
srcs = ["decode_test.js"],

72
js/bundle_test.js Normal file
View File

@ -0,0 +1,72 @@
/* Copyright 2017 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
import {BrotliDecode} from "./decode.js";
import {makeTestData} from "./test_data.js";
goog.require('goog.testing.asserts');
const testSuite = goog.require('goog.testing.testSuite');
const CRC_64_POLY = new Uint32Array([0xD7870F42, 0xC96C5795]);
/**
* Calculates binary data footprint.
*
* @param {!Int8Array} data binary data
* @return {string} footprint
*/
function calculateCrc64(data) {
let crc = new Uint32Array([0xFFFFFFFF, 0xFFFFFFFF]);
let c = new Uint32Array(2);
for (let i = 0; i < data.length; ++i) {
c[1] = 0;
c[0] = (crc[0] ^ data[i]) & 0xFF;
for (let k = 0; k < 8; ++k) {
const isOdd = c[0] & 1;
c[0] = (c[0] >>> 1) | ((c[1] & 1) << 31);
c[1] = c[1] >>> 1;
if (isOdd) {
c[0] = c[0] ^ CRC_64_POLY[0];
c[1] = c[1] ^ CRC_64_POLY[1];
}
}
crc[0] = ((crc[0] >>> 8) | ((crc[1] & 0xFF) << 24)) ^ c[0];
crc[1] = (crc[1] >>> 8) ^ c[1];
}
crc[0] = ~crc[0];
crc[1] = ~crc[1];
let lo = crc[0].toString(16);
lo = "0".repeat(8 - lo.length) + lo;
let hi = crc[1].toString(16);
hi = "0".repeat(8 - hi.length) + hi;
return hi + lo;
}
/**
* Decompresses data and checks that output footprint is correct.
*
* @param {string} entry filename including footprint prefix
* @param {!Int8Array} data compressed data
*/
function checkEntry(entry, data) {
const expectedCrc = entry.substring(0, 16);
const decompressed = BrotliDecode(data);
const crc = calculateCrc64(decompressed);
assertEquals(expectedCrc, crc);
}
let allTests = {};
const testData = makeTestData();
for (let entry in testData) {
if (!testData.hasOwnProperty(entry)) {
continue;
}
const name = entry.substring(17);
const data = testData[entry];
allTests['test_' + name] = checkEntry.bind(null, entry, data);
}
testSuite(allTests);

59
js/cli.js Normal file
View File

@ -0,0 +1,59 @@
#!/usr/bin/env node
let defaults =
{input: 'input.br', output: 'output.txt', test_iters: 0, test_repeat: 100};
/* Parse command line arguments. */
let argv =
require('yargs')
.usage('Usage: $0 -i file -o file')
.option(
'input',
{alias: 'i', default: defaults.input, describe: 'compressed file'})
.option('output', {
alias: 'o',
default: defaults.output,
describe: 'decompressed file'
})
.option('test_iters', {
default: defaults.test_iters,
describe: '# of times to run performance test'
})
.option('test_repeat', {
default: defaults.test_repeat,
describe: '# of times to decompress file in performance test'
})
.argv;
/* Read input. */
const fs = require('fs');
data = fs.readFileSync(argv.input);
if (!Buffer.isBuffer(data)) throw 'not a buffer';
const bytes = new Uint8Array(data);
/* Load and map brotli decoder module. */
global.window = {};
require('./decode.js')
const brotliDecode = window['BrotliDecode'];
/* Load "performance" module. */
const {PerformanceObserver, performance} = require('perf_hooks');
/* Performance test. */
for (let i = 0; i < argv.test_iters; ++i) {
const a = performance.now();
let result;
for (let j = 0; j < argv.test_repeat; ++j) {
result = brotliDecode(bytes);
}
const b = performance.now();
const total_length = argv.test_repeat * result.length / (1024 * 1024);
const total_time = (b - a) / 1000;
console.log(
total_length + 'MB / ' + total_time +
's = ' + (total_length / total_time) + 'MB/s');
}
/* Decode and write output file. */
fs.writeFileSync(argv.output, new Buffer(brotliDecode(bytes)));

2281
js/decode_synth_test.js Normal file

File diff suppressed because it is too large Load Diff

14
js/package.json Normal file
View File

@ -0,0 +1,14 @@
{
"name": "brotli",
"description": "Pure JavaScript Brotli implementation",
"bin": {
"brotli": "cli.js"
},
"files": [
"cli.js",
"decode.js",
],
"dependencies": {
"yargs": "~8.0.2"
},
}

28
js/test_data.js Normal file

File diff suppressed because one or more lines are too long

BIN
js/test_data.tar Normal file

Binary file not shown.

View File

@ -7,7 +7,6 @@
import _brotli
# The library version.
__version__ = _brotli.__version__

View File

@ -7,10 +7,8 @@ licenses(["notice"]) # MIT
cc_library(
name = "dm",
srcs = ["deorummolae.cc"],
hdrs = [
"deorummolae.h",
"esaxx/sais.hxx",
],
deps = ["@esaxx//:sais"],
hdrs = ["deorummolae.h"],
)
cc_library(

View File

@ -19,6 +19,7 @@ cc_library(
"-DHAVE_CONFIG_H=1",
],
includes = ["include"],
include_prefix = "third_party/libdivsufsort",
)
commom_awk_replaces = (

View File

@ -1,17 +0,0 @@
CC = g++
CFLAGS += -O2
CPPFLAGS += -std=c++11
SOURCES = $(wildcard *.cc)
EXECUTABLES = $(SOURCES:.cc=)
BINDIR = bin
all: $(EXECUTABLES)
$(BINDIR):
mkdir -p $@
$(EXECUTABLES): $(BINDIR)
$(CC) $(CFLAGS) $(CPPFLAGS) $(addsuffix .cc, $@) -o $(BINDIR)/$@ -lgflags_nothreads
clean:
rm -rf $(BINDIR)

View File

@ -1,12 +1,30 @@
workspace(name = "org_brotli_research")
load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository")
local_repository(
name = "org_brotli",
path = "..",
)
new_local_repository(
new_git_repository(
name = "divsufsort",
build_file = "BUILD.libdivsufsort",
path = "libdivsufsort",
build_file = "@//:BUILD.libdivsufsort",
commit = "5f60d6f026c30fb4ac296f696b3c8b0eb71bd428",
remote = "https://github.com/y-256/libdivsufsort",
)
new_git_repository(
name = "esaxx",
build_file_content = """
package(default_visibility = ["//visibility:public"])
cc_library(
name = "sais",
hdrs = ["sais.hxx"],
includes = ["."],
include_prefix = "third_party/esaxx",
)
""",
commit = "ca7cb332011ec37a8436487f210f396b84bd8273",
remote = "https://github.com/hillbig/esaxx",
)

View File

@ -3,7 +3,7 @@
#include <array>
#include <cstdio>
#include "./esaxx/sais.hxx"
#include "third_party/esaxx/sais.hxx"
/* Used for quick SA-entry to file mapping. Each file is padded to size that
is a multiple of chunk size. */

View File

@ -18,6 +18,7 @@
#include <gflags/gflags.h>
using gflags::ParseCommandLineFlags;
#include "third_party/absl/flags/flag.h"
#include "./read_dist.h"
DEFINE_int32(height, 1000, "Height of the resulting histogam.");
@ -32,7 +33,7 @@ DEFINE_bool(linear, false, "True if using linear distance mapping.");
DEFINE_uint64(skip, 0, "Number of bytes to skip.");
inline double DistanceTransform(double x) {
static bool linear = FLAGS_linear;
static bool linear = absl::GetFlag(FLAGS_linear);
if (linear) {
return x;
} else {
@ -48,14 +49,12 @@ inline double DensityTransform(double x) {
return sqrt(255 * 255 - z * z);
}
inline int GetMaxDistance() {
return FLAGS_max_distance;
}
inline int GetMaxDistance() { return absl::GetFlag(FLAGS_max_distance); }
void AdjustPosition(int* pos) {
static uint32_t offset = 0;
static int last = 0;
static uint32_t window_size = (1 << FLAGS_brotli_window);
static uint32_t window_size = (1 << absl::GetFlag(FLAGS_brotli_window));
assert(*pos >= 0 && *pos < window_size);
if (*pos < last) {
offset += window_size;
@ -65,10 +64,10 @@ void AdjustPosition(int* pos) {
}
void BuildHistogram(FILE* fin, int** histo) {
int height = FLAGS_height;
int width = FLAGS_width;
int skip = FLAGS_skip;
size_t min_distance = FLAGS_min_distance;
int height = absl::GetFlag(FLAGS_height);
int width = absl::GetFlag(FLAGS_width);
int skip = absl::GetFlag(FLAGS_skip);
size_t min_distance = absl::GetFlag(FLAGS_min_distance);
printf("height = %d, width = %d\n", height, width);
@ -78,7 +77,7 @@ void BuildHistogram(FILE* fin, int** histo) {
}
}
int max_pos = FLAGS_size - skip;
int max_pos = absl::GetFlag(FLAGS_size) - skip;
double min_dist = min_distance > 0 ? DistanceTransform(min_distance) : 0;
double max_dist = DistanceTransform(GetMaxDistance()) - min_dist;
int copy, pos, distance, x, y;
@ -86,7 +85,7 @@ void BuildHistogram(FILE* fin, int** histo) {
while (ReadBackwardReference(fin, &copy, &pos, &distance)) {
if (pos == -1) continue; // In case when only insert is present.
if (distance < min_distance || distance >= GetMaxDistance()) continue;
if (FLAGS_brotli_window != -1) {
if (absl::GetFlag(FLAGS_brotli_window) != -1) {
AdjustPosition(&pos);
}
if (pos >= skip && distance <= pos) {
@ -102,7 +101,7 @@ void BuildHistogram(FILE* fin, int** histo) {
assert(y >= 0 && y < width);
}
if (FLAGS_with_copies) {
if (absl::GetFlag(FLAGS_with_copies)) {
int right = 1ul * (pos + copy - 1) * width / max_pos;
if (right < 0) {
printf("pos = %d, distance = %d, copy = %d, y = %d, right = %d\n",
@ -131,8 +130,8 @@ void BuildHistogram(FILE* fin, int** histo) {
}
void ConvertToPixels(int** histo, uint8_t** pixel) {
int height = FLAGS_height;
int width = FLAGS_width;
int height = absl::GetFlag(FLAGS_height);
int width = absl::GetFlag(FLAGS_width);
int maxs = 0;
for (int i = 0; i < height; i++) {
@ -141,7 +140,7 @@ void ConvertToPixels(int** histo, uint8_t** pixel) {
}
}
bool simple = FLAGS_simple;
bool simple = absl::GetFlag(FLAGS_simple);
double max_histo = static_cast<double>(maxs);
for (int i = 0; i < height; i++) {
for (int j = 0; j < width; j++) {
@ -156,8 +155,8 @@ void ConvertToPixels(int** histo, uint8_t** pixel) {
}
void DrawPixels(uint8_t** pixel, FILE* fout) {
int height = FLAGS_height;
int width = FLAGS_width;
int height = absl::GetFlag(FLAGS_height);
int width = absl::GetFlag(FLAGS_width);
fprintf(fout, "P5\n%d %d\n255\n", width, height);
for (int i = height - 1; i >= 0; i--) {
@ -166,14 +165,14 @@ void DrawPixels(uint8_t** pixel, FILE* fout) {
}
int main(int argc, char* argv[]) {
ParseCommandLineFlags(&argc, &argv, true);
base::ParseCommandLine(&argc, &argv);
if (argc != 3) {
printf("usage: draw_histogram.cc data output_file\n");
return 1;
}
int height = FLAGS_height;
int width = FLAGS_width;
int height = absl::GetFlag(FLAGS_height);
int width = absl::GetFlag(FLAGS_width);
FILE* fin = fopen(argv[1], "r");
FILE* fout = fopen(argv[2], "wb");

View File

@ -3,7 +3,7 @@
#include <algorithm>
#include <exception> /* terminate */
#include "divsufsort.h"
#include "third_party/libdivsufsort/include/divsufsort.h"
/* Pointer to position in text. */
typedef DurchschlagTextIdx TextIdx;

@ -1 +0,0 @@
Subproject commit ca7cb332011ec37a8436487f210f396b84bd8273

View File

@ -19,7 +19,8 @@
#include <gflags/gflags.h>
using gflags::ParseCommandLineFlags;
#include "./esaxx/sais.hxx"
#include "third_party/absl/flags/flag.h"
#include "third_party/esaxx/sais.hxx"
DEFINE_bool(advanced, false, "Advanced searching mode: finds all longest "
"matches at positions that are not covered by matches of length at least "
@ -91,7 +92,7 @@ inline void PrintReference(sarray_type* sarray, lcp_type* lcp, size_t size,
inline void GoLeft(sarray_type* sarray, lcp_type* lcp, int idx, int left_ix,
int left_lcp, entry_type* entry) {
entry->first = left_lcp;
if (left_lcp > FLAGS_long_length) return;
if (left_lcp > absl::GetFlag(FLAGS_long_length)) return;
for (; left_ix >= 0; --left_ix) {
if (lcp[left_ix] < left_lcp) break;
if (sarray[left_ix] < idx) {
@ -103,7 +104,7 @@ inline void GoLeft(sarray_type* sarray, lcp_type* lcp, int idx, int left_ix,
inline void GoRight(sarray_type* sarray, lcp_type* lcp, int idx, size_t size,
int right_ix, int right_lcp, entry_type* entry) {
entry->first = right_lcp;
if (right_lcp > FLAGS_long_length) return;
if (right_lcp > absl::GetFlag(FLAGS_long_length)) return;
for (; right_ix < size - 1; ++right_ix) {
if (lcp[right_ix] < right_lcp) break;
if (sarray[right_ix] < idx) {
@ -129,8 +130,8 @@ inline void StoreReference(sarray_type* sarray, lcp_type* lcp, size_t size,
void ProcessReferences(sarray_type* sarray, lcp_type* lcp, size_t size,
uint32_t* pos, const Fn& process_output) {
int min_length = FLAGS_min_length;
for (int idx = FLAGS_skip; idx < size; ++idx) {
int min_length = absl::GetFlag(FLAGS_min_length);
for (int idx = absl::GetFlag(FLAGS_skip); idx < size; ++idx) {
int left_lcp = -1;
int left_ix;
for (left_ix = pos[idx] - 1; left_ix >= 0; --left_ix) {
@ -162,7 +163,7 @@ void ProcessReferences(sarray_type* sarray, lcp_type* lcp, size_t size,
}
void ProcessEntries(entry_type* entries, size_t size, FILE* fout) {
int long_length = FLAGS_long_length;
int long_length = absl::GetFlag(FLAGS_long_length);
std::vector<std::pair<int, int> > segments;
size_t idx;
for (idx = 0; idx < size;) {
@ -195,7 +196,7 @@ void ProcessEntries(entry_type* entries, size_t size, FILE* fout) {
}
int main(int argc, char* argv[]) {
ParseCommandLineFlags(&argc, &argv, true);
base::ParseCommandLine(&argc, &argv);
if (argc != 3) {
printf("usage: %s input_file output_file\n", argv[0]);
return 1;
@ -235,7 +236,7 @@ int main(int argc, char* argv[]) {
using std::placeholders::_7;
using std::placeholders::_8;
entry_type* entries;
if (FLAGS_advanced) {
if (absl::GetFlag(FLAGS_advanced)) {
entries = new entry_type[input_size];
for (size_t i = 0; i < input_size; ++i) entries[i].first = -1;
}
@ -243,10 +244,10 @@ int main(int argc, char* argv[]) {
Fn store = std::bind(StoreReference, _1, _2, _3, _4, _5, _6, _7, _8, entries);
ProcessReferences(sarray, lcp, input_size, pos,
FLAGS_advanced ? store : print);
absl::GetFlag(FLAGS_advanced) ? store : print);
printf("References processed.\n");
if (FLAGS_advanced) {
if (absl::GetFlag(FLAGS_advanced)) {
int good_cnt = 0;
uint64_t avg_cnt = 0;
for (size_t i = 0; i < input_size; ++i) {

@ -1 +0,0 @@
Subproject commit 5f60d6f026c30fb4ac296f696b3c8b0eb71bd428