AuroraMiddleware/brotli
1
0
Fork 0
mirror of https://github.com/google/brotli.git synced 2024-11-08 13:20:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update (#908)

Browse Source 
* re-enable Js build/test
  * improve decoder performance
  * rewrite dictionary data in Java/Js to a shorter uncompressed form
  * improve dictionary generation tool
This commit is contained in:
Eugene Kliuchnikov 2021-06-23 09:40:57 +02:00 committed by GitHub
parent bbe5d72ba3
commit f8c6717745
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 446 additions and 466 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
c/dec/bit_reader.h
View File

@ -108,45 +108,55 @@ static BROTLI_INLINE void BrotliFillBitWindow(
BrotliBitReader* const br, uint32_t n_bits) {
#if (BROTLI_64_BITS)
if (!BROTLI_ALIGNED_READ && BROTLI_IS_CONSTANT(n_bits) && (n_bits <= 8)) {
if (br->bit_pos_ >= 56) {
br->val_ >>= 56;
br->bit_pos_ ^= 56; /* here same as -= 56 because of the if condition */
br->val_ |= BROTLI_UNALIGNED_LOAD64LE(br->next_in) << 8;
uint32_t bit_pos = br->bit_pos_;
if (bit_pos >= 56) {
br->val_ =
(br->val_ >> 56) | (BROTLI_UNALIGNED_LOAD64LE(br->next_in) << 8);
br->bit_pos_ =
bit_pos ^ 56; /* here same as -= 56 because of the if condition */
br->avail_in -= 7;
br->next_in += 7;
}
} else if (
!BROTLI_ALIGNED_READ && BROTLI_IS_CONSTANT(n_bits) && (n_bits <= 16)) {
if (br->bit_pos_ >= 48) {
br->val_ >>= 48;
br->bit_pos_ ^= 48; /* here same as -= 48 because of the if condition */
br->val_ |= BROTLI_UNALIGNED_LOAD64LE(br->next_in) << 16;
uint32_t bit_pos = br->bit_pos_;
if (bit_pos >= 48) {
br->val_ =
(br->val_ >> 48) | (BROTLI_UNALIGNED_LOAD64LE(br->next_in) << 16);
br->bit_pos_ =
bit_pos ^ 48; /* here same as -= 48 because of the if condition */
br->avail_in -= 6;
br->next_in += 6;
}
} else {
if (br->bit_pos_ >= 32) {
br->val_ >>= 32;
br->bit_pos_ ^= 32; /* here same as -= 32 because of the if condition */
br->val_ |= ((uint64_t)BROTLI_UNALIGNED_LOAD32LE(br->next_in)) << 32;
uint32_t bit_pos = br->bit_pos_;
if (bit_pos >= 32) {
br->val_ = (br->val_ >> 32) |
(((uint64_t)BROTLI_UNALIGNED_LOAD32LE(br->next_in)) << 32);
br->bit_pos_ =
bit_pos ^ 32; /* here same as -= 32 because of the if condition */
br->avail_in -= BROTLI_SHORT_FILL_BIT_WINDOW_READ;
br->next_in += BROTLI_SHORT_FILL_BIT_WINDOW_READ;
}
}
#else
if (!BROTLI_ALIGNED_READ && BROTLI_IS_CONSTANT(n_bits) && (n_bits <= 8)) {
if (br->bit_pos_ >= 24) {
br->val_ >>= 24;
br->bit_pos_ ^= 24; /* here same as -= 24 because of the if condition */
br->val_ |= BROTLI_UNALIGNED_LOAD32LE(br->next_in) << 8;
uint32_t bit_pos = br->bit_pos_;
if (bit_pos >= 24) {
br->val_ =
(br->val_ >> 24) | (BROTLI_UNALIGNED_LOAD32LE(br->next_in) << 8);
br->bit_pos_ =
bit_pos ^ 24; /* here same as -= 24 because of the if condition */
br->avail_in -= 3;
br->next_in += 3;
}
} else {
if (br->bit_pos_ >= 16) {
br->val_ >>= 16;
br->bit_pos_ ^= 16; /* here same as -= 16 because of the if condition */
br->val_ |= ((uint32_t)BROTLI_UNALIGNED_LOAD16LE(br->next_in)) << 16;
uint32_t bit_pos = br->bit_pos_;
if (bit_pos >= 16) {
br->val_ = (br->val_ >> 16) |
(((uint32_t)BROTLI_UNALIGNED_LOAD16LE(br->next_in)) << 16);
br->bit_pos_ =
bit_pos ^ 16; /* here same as -= 16 because of the if condition */
br->avail_in -= BROTLI_SHORT_FILL_BIT_WINDOW_READ;
br->next_in += BROTLI_SHORT_FILL_BIT_WINDOW_READ;
}

2
java/org/brotli/dec/CompoundDictionaryTest.java
View File

@ -1,4 +1,4 @@
/* Copyright 2020 Google Inc. All Rights Reserved.
/* Copyright 2016 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT

8
java/org/brotli/dec/Decode.java
View File

@ -1127,13 +1127,13 @@ final class Decode {
s.distanceCode = CMD_LOOKUP[cmdCode + 3];
BitReader.fillBitWindow(s);
{
int extraBits = insertAndCopyExtraBits & 0xFF;
s.insertLength = insertLengthOffset + BitReader.readBits(s, extraBits);
int insertLengthExtraBits = insertAndCopyExtraBits & 0xFF;
s.insertLength = insertLengthOffset + BitReader.readBits(s, insertLengthExtraBits);
}
BitReader.fillBitWindow(s);
{
int extraBits = insertAndCopyExtraBits >> 8;
s.copyLength = copyLengthOffset + BitReader.readBits(s, extraBits);
int copyLengthExtraBits = insertAndCopyExtraBits >> 8;
s.copyLength = copyLengthOffset + BitReader.readBits(s, copyLengthExtraBits);
}
s.j = 0;

11
java/org/brotli/dec/DictionaryData.java
View File

File diff suppressed because one or more lines are too long

2
java/org/brotli/dec/SynthTest.java
View File

@ -2915,7 +2915,7 @@ public class SynthTest {
(byte) 0x12, (byte) 0x00, (byte) 0x00, (byte) 0x77, (byte) 0xda, (byte) 0xcc, (byte) 0xe1,
(byte) 0x7b, (byte) 0xfa, (byte) 0x0f
};
/* This lines is added manually. */
/* This line is added manually. */
char[] expected = new char[16777216]; Arrays.fill(expected, '*');
checkSynth(
/*

8
js/BUILD
View File

@ -23,11 +23,7 @@ closure_js_library(
closure_js_library(
name = "decode",
srcs = ["decode.js"],
suppress = [
"JSC_DUP_VAR_DECLARATION",
"JSC_INVALID_OCTAL_LITERAL",
"JSC_USELESS_BLOCK",
],
suppress = ["JSC_USELESS_BLOCK"],
deps = [":polyfill"],
)
@ -36,6 +32,8 @@ load("@io_bazel_rules_closure//closure:defs.bzl", "closure_js_test")
closure_js_test(
name = "all_tests",
srcs = ["decode_test.js"],
entry_points = ["decode_test.js"],
suppress = ["moduleLoad"],
deps = [
":decode",
":polyfill",

649
js/decode.js
View File

File diff suppressed because one or more lines are too long

3
js/decode.min.js vendored
View File

File diff suppressed because one or more lines are too long

96
js/decode_test.js
View File

@ -1,5 +1,11 @@
/* Copyright 2017 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
import {BrotliDecode} from "./decode.js";
const testSuite = goog.require('goog.testing.testSuite');
goog.require('goog.testing.asserts');
goog.require('goog.testing.jsunit');
/**
* @param {!Int8Array} bytes
@ -19,79 +25,17 @@ function stringToBytes(str) {
return out;
}
function testMetadata() {
assertEquals("", bytesToString(BrotliDecode(Int8Array.from([1, 11, 0, 42, 3]))));
}
testSuite({
testMetadata() {
assertEquals("", bytesToString(BrotliDecode(Int8Array.from([1, 11, 0, 42, 3]))));
},
function testEmpty() {
assertEquals("", bytesToString(BrotliDecode(Int8Array.from([6]))));
assertEquals("", bytesToString(BrotliDecode(Int8Array.from([0x81, 1]))));
}
function testBaseDictWord() {
var input = Int8Array.from([
0x1b, 0x03, 0x00, 0x00, 0x00, 0x00, 0x80, 0xe3, 0xb4, 0x0d, 0x00, 0x00,
0x07, 0x5b, 0x26, 0x31, 0x40, 0x02, 0x00, 0xe0, 0x4e, 0x1b, 0x41, 0x02
]);
/** @type {!Int8Array} */
var output = BrotliDecode(input);
assertEquals("time", bytesToString(output));
}
function testBlockCountMessage() {
var input = Int8Array.from([
0x1b, 0x0b, 0x00, 0x11, 0x01, 0x8c, 0xc1, 0xc5, 0x0d, 0x08, 0x00, 0x22,
0x65, 0xe1, 0xfc, 0xfd, 0x22, 0x2c, 0xc4, 0x00, 0x00, 0x38, 0xd8, 0x32,
0x89, 0x01, 0x12, 0x00, 0x00, 0x77, 0xda, 0x04, 0x10, 0x42, 0x00, 0x00, 0x00
]);
/** @type {!Int8Array} */
var output = BrotliDecode(input);
assertEquals("aabbaaaaabab", bytesToString(output));
}
function testCompressedUncompressedShortCompressedSmallWindow() {
var input = Int8Array.from([
0x21, 0xf4, 0x0f, 0x00, 0x00, 0x00, 0x00, 0x1c, 0xa7, 0x6d, 0x00, 0x00,
0x38, 0xd8, 0x32, 0x89, 0x01, 0x12, 0x00, 0x00, 0x77, 0xda, 0x34, 0x7b,
0xdb, 0x50, 0x80, 0x02, 0x80, 0x62, 0x62, 0x62, 0x62, 0x62, 0x62, 0x31,
0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x4e, 0xdb, 0x00, 0x00, 0x70, 0xb0,
0x65, 0x12, 0x03, 0x24, 0x00, 0x00, 0xee, 0xb4, 0x11, 0x24, 0x00
]);
/** @type {!Int8Array} */
var output = BrotliDecode(input);
assertEquals(
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +
"aaaaaaaaaaaaaabbbbbbbbbb", bytesToString(output));
}
function testIntactDistanceRingBuffer0() {
var input = Int8Array.from([
0x1b, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x80, 0xe3, 0xb4, 0x0d, 0x00, 0x00,
0x07, 0x5b, 0x26, 0x31, 0x40, 0x02, 0x00, 0xe0, 0x4e, 0x1b, 0xa1, 0x80,
0x20, 0x00
]);
/** @type {!Int8Array} */
var output = BrotliDecode(input);
assertEquals("himselfself", bytesToString(output));
}
function testCompoundDictionary() {
var txt = "kot lomom kolol slona\n";
var dictionary = stringToBytes(txt);
var compressed = [0xa1, 0xa8, 0x00, 0xc0, 0x2f, 0x01, 0x10, 0xc4, 0x44, 0x09, 0x00];
assertEquals(txt.length, compressed.length * 2);
assertEquals(txt, bytesToString(BrotliDecode(Int8Array.from(compressed), {customDictionary: dictionary})));
}
testCompoundDictionary() {
var txt = "kot lomom kolol slona\n";
var dictionary = stringToBytes(txt);
var compressed = [0xa1, 0xa8, 0x00, 0xc0, 0x2f, 0x01, 0x10, 0xc4, 0x44, 0x09, 0x00];
assertEquals(txt.length, compressed.length * 2);
var options = {"customDictionary": dictionary};
assertEquals(txt, bytesToString(BrotliDecode(Int8Array.from(compressed), options)));
}
});

9
js/polyfill.js
View File

@ -63,3 +63,12 @@ if (!Int8Array.prototype.fill) {
if (!Int32Array.prototype.fill) {
Int32Array.prototype.fill = Array.prototype.fill;
}
if (!Int8Array.prototype.slice) {
Object.defineProperty(Int8Array.prototype, 'slice', {
value: function (begin, end)
{
return new Int8Array(Array.prototype.slice.call(this, begin, end));
}
});
}

71
research/dictionary_generator.cc
View File

@ -3,6 +3,7 @@
#include <cstdio>
#include <cstring>
#include <fstream>
#include <glob.h>
#include <vector>
#include "./deorummolae.h"
@ -71,14 +72,13 @@ static void writeFile(const char* file, const std::string& content) {
outfile.close();
}
static void writeSamples(char const* argv[], const std::vector<int>& pathArgs,
static void writeSamples(const std::vector<std::string>& paths,
const std::vector<size_t>& sizes, const uint8_t* data) {
size_t offset = 0;
for (size_t i = 0; i < pathArgs.size(); ++i) {
int j = pathArgs[i];
const char* file = argv[j];
for (size_t i = 0; i < paths.size(); ++i) {
const char* path = paths[i].c_str();
size_t sampleSize = sizes[i];
std::ofstream outfile(file, std::ofstream::binary);
std::ofstream outfile(path, std::ofstream::binary);
outfile.write(reinterpret_cast<const char*>(data + offset),
static_cast<std::streamsize>(sampleSize));
outfile.close();
@ -135,7 +135,7 @@ int main(int argc, char const* argv[]) {
std::vector<uint8_t> data;
std::vector<size_t> sizes;
std::vector<int> pathArgs;
std::vector<std::string> paths;
size_t total = 0;
for (int i = 1; i < argc; ++i) {
if (argv[i] == nullptr) {
@ -217,6 +217,9 @@ int main(int argc, char const* argv[]) {
}
} else if (arg1 == 's') {
sliceLen = readInt(arg2);
// TODO: investigate why sliceLen == 4..5 greatly slows down
// durschlag engine, but only from command line;
// durschlag_runner seems to work fine with those.
if (sliceLen < 4 || sliceLen > 256) {
printHelp(fileName(argv[0]));
fprintf(stderr, "Invalid option '%s'\n", argv[i]);
@ -265,28 +268,42 @@ int main(int argc, char const* argv[]) {
}
}
std::string content = readFile(argv[i]);
if (chunkLen == 0) {
pathArgs.push_back(i);
data.insert(data.end(), content.begin(), content.end());
total += content.size();
sizes.push_back(content.size());
continue;
} else if (chunkLen <= overlapLen) {
printHelp(fileName(argv[0]));
fprintf(stderr, "Invalid chunkLen - overlapLen combination\n");
exit(1);
}
for (size_t chunkStart = 0;
chunkStart < content.size();
chunkStart += chunkLen - overlapLen) {
std::string chunk = content.substr(chunkStart, chunkLen);
data.insert(data.end(), chunk.begin(), chunk.end());
total += chunk.size();
sizes.push_back(chunk.size());
glob_t resolved_paths;
memset(&resolved_paths, 0, sizeof(resolved_paths));
bool ok = true;
if (glob(argv[i], GLOB_TILDE, NULL, &resolved_paths) == 0) {
for(size_t j = 0; j < resolved_paths.gl_pathc; ++j) {
std::string content = readFile(resolved_paths.gl_pathv[j]);
if (chunkLen == 0) {
paths.emplace_back(resolved_paths.gl_pathv[j]);
data.insert(data.end(), content.begin(), content.end());
total += content.size();
sizes.push_back(content.size());
continue;
} else if (chunkLen <= overlapLen) {
printHelp(fileName(argv[0]));
fprintf(stderr, "Invalid chunkLen - overlapLen combination\n");
exit(1);
}
for (size_t chunkStart = 0;
chunkStart < content.size();
chunkStart += chunkLen - overlapLen) {
std::string chunk = content.substr(chunkStart, chunkLen);
data.insert(data.end(), chunk.begin(), chunk.end());
total += chunk.size();
sizes.push_back(chunk.size());
}
}
} else {
ok = false;
}
globfree(&resolved_paths);
if (!ok) exit(1);
}
fprintf(stderr, "Number of chunks: %zu; total size: %zu\n", sizes.size(),
total);
bool wantDictionary = (dictionaryArg == -1);
if (method == METHOD_DISTILL || method == METHOD_PURIFY) {
wantDictionary = false;
@ -313,10 +330,10 @@ int main(int argc, char const* argv[]) {
targetSize, sliceLen, blockSize, sizes, data.data()));
} else if (method == METHOD_DISTILL) {
durchschlag_distill(sliceLen, minimumPopulation, &sizes, data.data());
writeSamples(argv, pathArgs, sizes, data.data());
writeSamples(paths, sizes, data.data());
} else if (method == METHOD_PURIFY) {
durchschlag_purify(sliceLen, minimumPopulation, sizes, data.data());
writeSamples(argv, pathArgs, sizes, data.data());
writeSamples(paths, sizes, data.data());
} else {
printHelp(fileName(argv[0]));
fprintf(stderr, "Unknown generator\n");

3
scripts/.travis.sh
View File

@ -62,8 +62,7 @@ case "$1" in
bazel build -c opt ...:all &&
cd go && bazel test -c opt ...:all && cd .. &&
cd java && bazel test -c opt ...:all && cd .. &&
echo "bazelbuild/rules_closure seems to be dead; js build is skipped" &&
echo "cd js && bazel test -c opt ...:all && cd .." &&
cd js && bazel test -c opt ...:all && cd .. &&
cd research && bazel build -c opt ...:all && cd ..
;;
esac