AuroraMiddleware/brotli
1
0
Fork 0
mirror of https://github.com/google/brotli.git synced 2024-11-08 13:20:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update (#852)

Browse Source 
* Update

 * comments and clarifications in block_splitter
 * power-of-2 aligned allocations for Hasher
 * refresh decode.js from Java sources
 * disable JS build
This commit is contained in:
Eugene Kliuchnikov 2021-01-18 10:56:39 +01:00 committed by GitHub
parent f16845614d
commit 5692e422da
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 790 additions and 411 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
c/enc/block_splitter.c
View File

@ -30,6 +30,7 @@ static const double kCommandBlockSwitchCost = 13.5;
static const double kDistanceBlockSwitchCost = 14.6;
static const size_t kLiteralStrideLength = 70;
static const size_t kCommandStrideLength = 40;
static const size_t kDistanceStrideLength = 40;
static const size_t kSymbolsPerLiteralHistogram = 544;
static const size_t kSymbolsPerCommandHistogram = 530;
static const size_t kSymbolsPerDistanceHistogram = 544;
@ -119,6 +120,8 @@ void BrotliDestroyBlockSplit(MemoryManager* m, BlockSplit* self) {
BROTLI_FREE(m, self->lengths);
}
/* Extracts literals, command distance and prefix codes, then applies
* SplitByteVector to create partitioning. */
void BrotliSplitBlock(MemoryManager* m,
const Command* cmds,
const size_t num_commands,
@ -136,7 +139,9 @@ void BrotliSplitBlock(MemoryManager* m,
/* Create a continuous array of literals. */
CopyLiteralsToByteArray(cmds, num_commands, data, pos, mask, literals);
/* Create the block split on the array of literals.
Literal histograms have alphabet size 256. */
* Literal histograms can have alphabet size up to 256.
* Though, to accomodate context modeling, less than half of maximum size
* is allowed. */
SplitByteVectorLiteral(
m, literals, literals_count,
kSymbolsPerLiteralHistogram, kMaxLiteralHistograms,
@ -144,6 +149,10 @@ void BrotliSplitBlock(MemoryManager* m,
literal_split);
if (BROTLI_IS_OOM(m)) return;
BROTLI_FREE(m, literals);
/* NB: this might be a good place for injecting extra splitting without
* increasing encoder complexity; however, output parition would be less
* optimal than one produced with forced splitting inside
* SplitByteVector (FindBlocks / ClusterBlocks). */
}
{
@ -181,7 +190,7 @@ void BrotliSplitBlock(MemoryManager* m,
SplitByteVectorDistance(
m, distance_prefixes, j,
kSymbolsPerDistanceHistogram, kMaxCommandHistograms,
kCommandStrideLength, kDistanceBlockSwitchCost, params,
kDistanceStrideLength, kDistanceBlockSwitchCost, params,
dist_split);
if (BROTLI_IS_OOM(m)) return;
BROTLI_FREE(m, distance_prefixes);

76
c/enc/block_splitter_inc.h
View File

@ -71,46 +71,56 @@ static size_t FN(FindBlocks)(const DataType* data, const size_t length,
double* cost,
uint8_t* switch_signal,
uint8_t* block_id) {
const size_t data_size = FN(HistogramDataSize)();
const size_t bitmaplen = (num_histograms + 7) >> 3;
const size_t alphabet_size = FN(HistogramDataSize)();
const size_t bitmap_len = (num_histograms + 7) >> 3;
size_t num_blocks = 1;
size_t byte_ix;
size_t i;
size_t j;
BROTLI_DCHECK(num_histograms <= 256);
/* Trivial case: single historgram -> single block type. */
if (num_histograms <= 1) {
for (i = 0; i < length; ++i) {
block_id[i] = 0;
}
return 1;
}
memset(insert_cost, 0, sizeof(insert_cost[0]) * data_size * num_histograms);
/* Fill bitcost for each symbol of all histograms.
* Non-existing symbol cost: 2 + log2(total_count).
* Regular symbol cost: -log2(symbol_count / total_count). */
memset(insert_cost, 0,
sizeof(insert_cost[0]) * alphabet_size * num_histograms);
for (i = 0; i < num_histograms; ++i) {
insert_cost[i] = FastLog2((uint32_t)histograms[i].total_count_);
}
for (i = data_size; i != 0;) {
for (i = alphabet_size; i != 0;) {
/* Reverse order to use the 0-th row as a temporary storage. */
--i;
for (j = 0; j < num_histograms; ++j) {
insert_cost[i * num_histograms + j] =
insert_cost[j] - BitCost(histograms[j].data_[i]);
}
}
memset(cost, 0, sizeof(cost[0]) * num_histograms);
memset(switch_signal, 0, sizeof(switch_signal[0]) * length * bitmaplen);
/* After each iteration of this loop, cost[k] will contain the difference
between the minimum cost of arriving at the current byte position using
entropy code k, and the minimum cost of arriving at the current byte
position. This difference is capped at the block switch cost, and if it
reaches block switch cost, it means that when we trace back from the last
position, we need to switch here. */
for (i = 0; i < length; ++i) {
const size_t byte_ix = i;
size_t ix = byte_ix * bitmaplen;
size_t insert_cost_ix = data[byte_ix] * num_histograms;
memset(cost, 0, sizeof(cost[0]) * num_histograms);
memset(switch_signal, 0, sizeof(switch_signal[0]) * length * bitmap_len);
for (byte_ix = 0; byte_ix < length; ++byte_ix) {
size_t ix = byte_ix * bitmap_len;
size_t symbol = data[byte_ix];
size_t insert_cost_ix = symbol * num_histograms;
double min_cost = 1e99;
double block_switch_cost = block_switch_bitcost;
size_t k;
for (k = 0; k < num_histograms; ++k) {
/* We are coding the symbol in data[byte_ix] with entropy code k. */
/* We are coding the symbol with entropy code k. */
cost[k] += insert_cost[insert_cost_ix + k];
if (cost[k] < min_cost) {
min_cost = cost[k];
@ -126,20 +136,21 @@ static size_t FN(FindBlocks)(const DataType* data, const size_t length,
if (cost[k] >= block_switch_cost) {
const uint8_t mask = (uint8_t)(1u << (k & 7));
cost[k] = block_switch_cost;
BROTLI_DCHECK((k >> 3) < bitmaplen);
BROTLI_DCHECK((k >> 3) < bitmap_len);
switch_signal[ix + (k >> 3)] |= mask;
}
}
}
byte_ix = length - 1;
{ /* Trace back from the last position and switch at the marked places. */
size_t byte_ix = length - 1;
size_t ix = byte_ix * bitmaplen;
size_t ix = byte_ix * bitmap_len;
uint8_t cur_id = block_id[byte_ix];
while (byte_ix > 0) {
const uint8_t mask = (uint8_t)(1u << (cur_id & 7));
BROTLI_DCHECK(((size_t)cur_id >> 3) < bitmaplen);
BROTLI_DCHECK(((size_t)cur_id >> 3) < bitmap_len);
--byte_ix;
ix -= bitmaplen;
ix -= bitmap_len;
if (switch_signal[ix + (cur_id >> 3)] & mask) {
if (cur_id != block_id[byte_ix]) {
cur_id = block_id[byte_ix];
@ -185,6 +196,8 @@ static void FN(BuildBlockHistograms)(const DataType* data, const size_t length,
}
}
/* Given the initial partitioning build partitioning with limited number
* of histograms (and block types). */
static void FN(ClusterBlocks)(MemoryManager* m,
const DataType* data, const size_t length,
const size_t num_blocks,
@ -228,6 +241,7 @@ static void FN(ClusterBlocks)(MemoryManager* m,
memset(block_lengths, 0, num_blocks * sizeof(uint32_t));
/* Calculate block lengths (convert repeating values -> series length). */
{
size_t block_idx = 0;
for (i = 0; i < length; ++i) {
@ -240,6 +254,7 @@ static void FN(ClusterBlocks)(MemoryManager* m,
BROTLI_DCHECK(block_idx == num_blocks);
}
/* Pre-cluster blocks (cluster batches). */
for (i = 0; i < num_blocks; i += HISTOGRAMS_PER_BATCH) {
const size_t num_to_combine =
BROTLI_MIN(size_t, num_blocks - i, HISTOGRAMS_PER_BATCH);
@ -247,8 +262,9 @@ static void FN(ClusterBlocks)(MemoryManager* m,
size_t j;
for (j = 0; j < num_to_combine; ++j) {
size_t k;
size_t block_length = block_lengths[i + j];
FN(HistogramClear)(&histograms[j]);
for (k = 0; k < block_lengths[i + j]; ++k) {
for (k = 0; k < block_length; ++k) {
FN(HistogramAdd)(&histograms[j], data[pos++]);
}
histograms[j].bit_cost_ = FN(BrotliPopulationCost)(&histograms[j]);
@ -278,6 +294,7 @@ static void FN(ClusterBlocks)(MemoryManager* m,
}
BROTLI_FREE(m, histograms);
/* Final clustering. */
max_num_pairs =
BROTLI_MIN(size_t, 64 * num_clusters, (num_clusters / 2) * num_clusters);
if (pairs_capacity < max_num_pairs + 1) {
@ -285,7 +302,6 @@ static void FN(ClusterBlocks)(MemoryManager* m,
pairs = BROTLI_ALLOC(m, HistogramPair, max_num_pairs + 1);
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(pairs)) return;
}
clusters = BROTLI_ALLOC(m, uint32_t, num_clusters);
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(clusters)) return;
for (i = 0; i < num_clusters; ++i) {
@ -298,6 +314,7 @@ static void FN(ClusterBlocks)(MemoryManager* m,
BROTLI_FREE(m, pairs);
BROTLI_FREE(m, cluster_size);
/* Assign blocks to final histograms. */
new_index = BROTLI_ALLOC(m, uint32_t, num_clusters);
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(new_index)) return;
for (i = 0; i < num_clusters; ++i) new_index[i] = kInvalidIndex;
@ -313,6 +330,8 @@ static void FN(ClusterBlocks)(MemoryManager* m,
for (j = 0; j < block_lengths[i]; ++j) {
FN(HistogramAdd)(&histo, data[pos++]);
}
/* Among equally good histograms prefer last used. */
/* TODO: should we give a block-switch discount here? */
best_out = (i == 0) ? histogram_symbols[0] : histogram_symbols[i - 1];
best_bits =
FN(BrotliHistogramBitCostDistance)(&histo, &all_histograms[best_out]);
@ -337,6 +356,9 @@ static void FN(ClusterBlocks)(MemoryManager* m,
BROTLI_ENSURE_CAPACITY(
m, uint32_t, split->lengths, split->lengths_alloc_size, num_blocks);
if (BROTLI_IS_OOM(m)) return;
/* Rewrite final assignment to block-split. There might be less blocks
* than |num_blocks| due to clustering. */
{
uint32_t cur_length = 0;
size_t block_idx = 0;
@ -361,24 +383,36 @@ static void FN(ClusterBlocks)(MemoryManager* m,
BROTLI_FREE(m, histogram_symbols);
}
/* Create BlockSplit (partitioning) given the limits, estimates and "effort"
* parameters.
*
* NB: max_histograms is often less than number of histograms allowed by format;
* this is done intentionally, to save some "space" for context-aware
* clustering (here entropy is estimated for context-free symbols). */
static void FN(SplitByteVector)(MemoryManager* m,
const DataType* data, const size_t length,
const size_t literals_per_histogram,
const size_t symbols_per_histogram,
const size_t max_histograms,
const size_t sampling_stride_length,
const double block_switch_cost,
const BrotliEncoderParams* params,
BlockSplit* split) {
const size_t data_size = FN(HistogramDataSize)();
size_t num_histograms = length / literals_per_histogram + 1;
HistogramType* histograms;
/* Calculate number of histograms; initial estimate is one histogram per
* specified amount of symbols; however, this value is capped. */
size_t num_histograms = length / symbols_per_histogram + 1;
if (num_histograms > max_histograms) {
num_histograms = max_histograms;
}
/* Corner case: no input. */
if (length == 0) {
split->num_types = 1;
return;
} else if (length < kMinLengthForBlockSplitting) {
}
if (length < kMinLengthForBlockSplitting) {
BROTLI_ENSURE_CAPACITY(m, uint8_t,
split->types, split->types_alloc_size, split->num_blocks + 1);
BROTLI_ENSURE_CAPACITY(m, uint32_t,

59
c/enc/hash.h
View File

@ -10,6 +10,7 @@
#ifndef BROTLI_ENC_HASH_H_
#define BROTLI_ENC_HASH_H_
#include <stdlib.h> /* exit */
#include <string.h> /* memcmp, memset */
#include "../common/constants.h"
@ -28,15 +29,28 @@ extern "C" {
#endif
typedef struct {
/* Dynamically allocated area; first member for quickest access. */
void* extra;
/**
* Dynamically allocated areas; regular hasher uses one or two allocations;
* "composite" hasher uses up to 4 allocations.
*/
void* extra[4];
/**
* False before the fisrt invocation of HasherSetup (where "extra" memory)
* is allocated.
*/
BROTLI_BOOL is_setup_;
size_t dict_num_lookups;
size_t dict_num_matches;
BrotliHasherParams params;
/* False if hasher needs to be "prepared" before use. */
/**
* False if hasher needs to be "prepared" before use (before the first
* invocation of HasherSetup or after HasherReset). "preparation" is hasher
* data initialization (using input ringbuffer).
*/
BROTLI_BOOL is_prepared_;
} HasherCommon;
@ -391,42 +405,52 @@ typedef struct {
/* MUST be invoked before any other method. */
static BROTLI_INLINE void HasherInit(Hasher* hasher) {
hasher->common.extra = NULL;
hasher->common.is_setup_ = BROTLI_FALSE;
hasher->common.extra[0] = NULL;
hasher->common.extra[1] = NULL;
hasher->common.extra[2] = NULL;
hasher->common.extra[3] = NULL;
}
static BROTLI_INLINE void DestroyHasher(MemoryManager* m, Hasher* hasher) {
if (hasher->common.extra == NULL) return;
BROTLI_FREE(m, hasher->common.extra);
if (hasher->common.extra[0] != NULL) BROTLI_FREE(m, hasher->common.extra[0]);
if (hasher->common.extra[1] != NULL) BROTLI_FREE(m, hasher->common.extra[1]);
if (hasher->common.extra[2] != NULL) BROTLI_FREE(m, hasher->common.extra[2]);
if (hasher->common.extra[3] != NULL) BROTLI_FREE(m, hasher->common.extra[3]);
}
static BROTLI_INLINE void HasherReset(Hasher* hasher) {
hasher->common.is_prepared_ = BROTLI_FALSE;
}
static BROTLI_INLINE size_t HasherSize(const BrotliEncoderParams* params,
BROTLI_BOOL one_shot, const size_t input_size) {
static BROTLI_INLINE void HasherSize(const BrotliEncoderParams* params,
BROTLI_BOOL one_shot, const size_t input_size, size_t* alloc_size) {
switch (params->hasher.type) {
#define SIZE_(N) \
case N: \
return HashMemAllocInBytesH ## N(params, one_shot, input_size);
#define SIZE_(N) \
case N: \
HashMemAllocInBytesH ## N(params, one_shot, input_size, alloc_size); \
break;
FOR_ALL_HASHERS(SIZE_)
#undef SIZE_
default:
break;
}
return 0; /* Default case. */
}
static BROTLI_INLINE void HasherSetup(MemoryManager* m, Hasher* hasher,
BrotliEncoderParams* params, const uint8_t* data, size_t position,
size_t input_size, BROTLI_BOOL is_last) {
BROTLI_BOOL one_shot = (position == 0 && is_last);
if (hasher->common.extra == NULL) {
size_t alloc_size;
if (!hasher->common.is_setup_) {
size_t alloc_size[4] = {0};
size_t i;
ChooseHasher(params, &params->hasher);
alloc_size = HasherSize(params, one_shot, input_size);
hasher->common.extra = BROTLI_ALLOC(m, uint8_t, alloc_size);
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(hasher->common.extra)) return;
HasherSize(params, one_shot, input_size, alloc_size);
for (i = 0; i < 4; ++i) {
if (alloc_size[i] == 0) continue;
hasher->common.extra[i] = BROTLI_ALLOC(m, uint8_t, alloc_size[i]);
if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(hasher->common.extra[i])) return;
}
hasher->common.params = params->hasher;
switch (hasher->common.params.type) {
#define INITIALIZE_(N) \
@ -440,6 +464,7 @@ static BROTLI_INLINE void HasherSetup(MemoryManager* m, Hasher* hasher,
break;
}
HasherReset(hasher);
hasher->common.is_setup_ = BROTLI_TRUE;
}
if (!hasher->common.is_prepared_) {

33
c/enc/hash_composite_inc.h
View File

@ -30,10 +30,10 @@ static BROTLI_INLINE size_t FN(StoreLookahead)(void) {
typedef struct HashComposite {
HASHER_A ha;
HASHER_B hb;
HasherCommon ha_common;
HasherCommon hb_common;
/* Shortcuts. */
void* extra;
HasherCommon* common;
BROTLI_BOOL fresh;
@ -43,8 +43,8 @@ typedef struct HashComposite {
static void FN(Initialize)(HasherCommon* common,
HashComposite* BROTLI_RESTRICT self, const BrotliEncoderParams* params) {
self->common = common;
self->extra = common->extra;
self->ha_common = *self->common;
self->hb_common = *self->common;
self->fresh = BROTLI_TRUE;
self->params = params;
@ -59,21 +59,36 @@ static void FN(Prepare)(
size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
if (self->fresh) {
self->fresh = BROTLI_FALSE;
self->hb_common.extra = (uint8_t*)self->extra +
FN_A(HashMemAllocInBytes)(self->params, one_shot, input_size);
self->ha_common.extra[0] = self->common->extra[0];
self->ha_common.extra[1] = self->common->extra[1];
self->ha_common.extra[2] = NULL;
self->ha_common.extra[3] = NULL;
self->hb_common.extra[0] = self->common->extra[2];
self->hb_common.extra[1] = self->common->extra[3];
self->hb_common.extra[2] = NULL;
self->hb_common.extra[3] = NULL;
FN_A(Initialize)(self->common, &self->ha, self->params);
FN_A(Initialize)(&self->ha_common, &self->ha, self->params);
FN_B(Initialize)(&self->hb_common, &self->hb, self->params);
}
FN_A(Prepare)(&self->ha, one_shot, input_size, data);
FN_B(Prepare)(&self->hb, one_shot, input_size, data);
}
static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
static BROTLI_INLINE void FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
size_t input_size) {
return FN_A(HashMemAllocInBytes)(params, one_shot, input_size) +
FN_B(HashMemAllocInBytes)(params, one_shot, input_size);
size_t input_size, size_t* alloc_size) {
size_t alloc_size_a[4] = {0};
size_t alloc_size_b[4] = {0};
FN_A(HashMemAllocInBytes)(params, one_shot, input_size, alloc_size_a);
FN_B(HashMemAllocInBytes)(params, one_shot, input_size, alloc_size_b);
// Should never happen.
if (alloc_size_a[2] != 0 || alloc_size_a[3] != 0) exit(EXIT_FAILURE);
if (alloc_size_b[2] != 0 || alloc_size_b[3] != 0) exit(EXIT_FAILURE);
alloc_size[0] = alloc_size_a[0];
alloc_size[1] = alloc_size_a[1];
alloc_size[2] = alloc_size_b[0];
alloc_size[3] = alloc_size_b[1];
}
static BROTLI_INLINE void FN(Store)(HashComposite* BROTLI_RESTRICT self,

38
c/enc/hash_forgetful_chain_inc.h
View File

@ -49,7 +49,7 @@ typedef struct HashForgetfulChain {
size_t max_hops;
/* Shortcuts. */
void* extra;
void* extra[2];
HasherCommon* common;
/* --- Dynamic size members --- */
@ -77,14 +77,15 @@ static uint8_t* FN(TinyHash)(void* extra) {
}
static FN(Bank)* FN(Banks)(void* extra) {
return (FN(Bank)*)(&FN(TinyHash)(extra)[65536]);
return (FN(Bank)*)(extra);
}
static void FN(Initialize)(
HasherCommon* common, HashForgetfulChain* BROTLI_RESTRICT self,
const BrotliEncoderParams* params) {
self->common = common;
self->extra = common->extra;
self->extra[0] = common->extra[0];
self->extra[1] = common->extra[1];
self->max_hops = (params->quality > 6 ? 7u : 8u) << (params->quality - 4);
}
@ -92,9 +93,9 @@ static void FN(Initialize)(
static void FN(Prepare)(
HashForgetfulChain* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
size_t input_size, const uint8_t* BROTLI_RESTRICT data) {
uint32_t* BROTLI_RESTRICT addr = FN(Addr)(self->extra);
uint16_t* BROTLI_RESTRICT head = FN(Head)(self->extra);
uint8_t* BROTLI_RESTRICT tiny_hash = FN(TinyHash)(self->extra);
uint32_t* BROTLI_RESTRICT addr = FN(Addr)(self->extra[0]);
uint16_t* BROTLI_RESTRICT head = FN(Head)(self->extra[0]);
uint8_t* BROTLI_RESTRICT tiny_hash = FN(TinyHash)(self->extra[0]);
/* Partial preparation is 100 times slower (per socket). */
size_t partial_prepare_threshold = BUCKET_SIZE >> 6;
if (one_shot && input_size <= partial_prepare_threshold) {
@ -116,24 +117,25 @@ static void FN(Prepare)(
memset(self->free_slot_idx, 0, sizeof(self->free_slot_idx));
}
static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
static BROTLI_INLINE void FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
size_t input_size) {
size_t input_size, size_t* alloc_size) {
BROTLI_UNUSED(params);
BROTLI_UNUSED(one_shot);
BROTLI_UNUSED(input_size);
return sizeof(uint32_t) * BUCKET_SIZE + sizeof(uint16_t) * BUCKET_SIZE +
sizeof(uint8_t) * 65536 + sizeof(FN(Bank)) * NUM_BANKS;
alloc_size[0] = sizeof(uint32_t) * BUCKET_SIZE +
sizeof(uint16_t) * BUCKET_SIZE + sizeof(uint8_t) * 65536;
alloc_size[1] = sizeof(FN(Bank)) * NUM_BANKS;
}
/* Look at 4 bytes at &data[ix & mask]. Compute a hash from these, and prepend
node to corresponding chain; also update tiny_hash for current position. */
static BROTLI_INLINE void FN(Store)(HashForgetfulChain* BROTLI_RESTRICT self,
const uint8_t* BROTLI_RESTRICT data, const size_t mask, const size_t ix) {
uint32_t* BROTLI_RESTRICT addr = FN(Addr)(self->extra);
uint16_t* BROTLI_RESTRICT head = FN(Head)(self->extra);
uint8_t* BROTLI_RESTRICT tiny_hash = FN(TinyHash)(self->extra);
FN(Bank)* BROTLI_RESTRICT banks = FN(Banks)(self->extra);
uint32_t* BROTLI_RESTRICT addr = FN(Addr)(self->extra[0]);
uint16_t* BROTLI_RESTRICT head = FN(Head)(self->extra[0]);
uint8_t* BROTLI_RESTRICT tiny_hash = FN(TinyHash)(self->extra[0]);
FN(Bank)* BROTLI_RESTRICT banks = FN(Banks)(self->extra[1]);
const size_t key = FN(HashBytes)(&data[ix & mask]);
const size_t bank = key & (NUM_BANKS - 1);
const size_t idx = self->free_slot_idx[bank]++ & (BANK_SIZE - 1);
@ -196,10 +198,10 @@ static BROTLI_INLINE void FN(FindLongestMatch)(
const size_t cur_ix, const size_t max_length, const size_t max_backward,
const size_t dictionary_distance, const size_t max_distance,
HasherSearchResult* BROTLI_RESTRICT out) {
uint32_t* BROTLI_RESTRICT addr = FN(Addr)(self->extra);
uint16_t* BROTLI_RESTRICT head = FN(Head)(self->extra);
uint8_t* BROTLI_RESTRICT tiny_hashes = FN(TinyHash)(self->extra);
FN(Bank)* BROTLI_RESTRICT banks = FN(Banks)(self->extra);
uint32_t* BROTLI_RESTRICT addr = FN(Addr)(self->extra[0]);
uint16_t* BROTLI_RESTRICT head = FN(Head)(self->extra[0]);
uint8_t* BROTLI_RESTRICT tiny_hashes = FN(TinyHash)(self->extra[0]);
FN(Bank)* BROTLI_RESTRICT banks = FN(Banks)(self->extra[1]);
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
/* Don't accept a short copy from far away. */
score_t min_score = out->score;

12
c/enc/hash_longest_match64_inc.h
View File

@ -71,8 +71,8 @@ static void FN(Initialize)(
self->block_mask_ = (uint32_t)(self->block_size_ - 1);
self->num_last_distances_to_check_ =
common->params.num_last_distances_to_check;
self->num_ = (uint16_t*)common->extra;
self->buckets_ = (uint32_t*)&self->num_[self->bucket_size_];
self->num_ = (uint16_t*)common->extra[0];
self->buckets_ = (uint32_t*)common->extra[1];
}
static void FN(Prepare)(
@ -93,15 +93,15 @@ static void FN(Prepare)(
}
}
static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
static BROTLI_INLINE void FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
size_t input_size) {
size_t input_size, size_t* alloc_size) {
size_t bucket_size = (size_t)1 << params->hasher.bucket_bits;
size_t block_size = (size_t)1 << params->hasher.block_bits;
BROTLI_UNUSED(one_shot);
BROTLI_UNUSED(input_size);
return sizeof(uint16_t) * bucket_size +
sizeof(uint32_t) * bucket_size * block_size;
alloc_size[0] = sizeof(uint16_t) * bucket_size;
alloc_size[1] = sizeof(uint32_t) * bucket_size * block_size;
}
/* Look at 4 bytes at &data[ix & mask].

16
c/enc/hash_longest_match_inc.h
View File

@ -54,10 +54,6 @@ typedef struct HashLongestMatch {
uint32_t* buckets_; /* uint32_t[bucket_size * block_size]; */
} HashLongestMatch;
static BROTLI_INLINE uint16_t* FN(Num)(void* extra) {
return (uint16_t*)extra;
}
static void FN(Initialize)(
HasherCommon* common, HashLongestMatch* BROTLI_RESTRICT self,
const BrotliEncoderParams* params) {
@ -68,8 +64,8 @@ static void FN(Initialize)(
self->bucket_size_ = (size_t)1 << common->params.bucket_bits;
self->block_size_ = (size_t)1 << common->params.block_bits;
self->block_mask_ = (uint32_t)(self->block_size_ - 1);
self->num_ = (uint16_t*)common->extra;
self->buckets_ = (uint32_t*)(&self->num_[self->bucket_size_]);
self->num_ = (uint16_t*)common->extra[0];
self->buckets_ = (uint32_t*)common->extra[1];
self->block_bits_ = common->params.block_bits;
self->num_last_distances_to_check_ =
common->params.num_last_distances_to_check;
@ -92,15 +88,15 @@ static void FN(Prepare)(
}
}
static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
static BROTLI_INLINE void FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
size_t input_size) {
size_t input_size, size_t* alloc_size) {
size_t bucket_size = (size_t)1 << params->hasher.bucket_bits;
size_t block_size = (size_t)1 << params->hasher.block_bits;
BROTLI_UNUSED(one_shot);
BROTLI_UNUSED(input_size);
return sizeof(uint16_t) * bucket_size +
sizeof(uint32_t) * bucket_size * block_size;
alloc_size[0] = sizeof(uint16_t) * bucket_size;
alloc_size[1] = sizeof(uint32_t) * bucket_size * block_size;
}
/* Look at 4 bytes at &data[ix & mask].

8
c/enc/hash_longest_match_quickly_inc.h
View File

@ -49,7 +49,7 @@ static void FN(Initialize)(
self->common = common;
BROTLI_UNUSED(params);
self->buckets_ = (uint32_t*)common->extra;
self->buckets_ = (uint32_t*)common->extra[0];
}
static void FN(Prepare)(
@ -80,13 +80,13 @@ static void FN(Prepare)(
}
}
static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
static BROTLI_INLINE void FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
size_t input_size) {
size_t input_size, size_t* alloc_size) {
BROTLI_UNUSED(params);
BROTLI_UNUSED(one_shot);
BROTLI_UNUSED(input_size);
return sizeof(uint32_t) * BUCKET_SIZE;
alloc_size[0] = sizeof(uint32_t) * BUCKET_SIZE;
}
/* Look at 5 bytes at &data[ix & mask].

8
c/enc/hash_rolling_inc.h
View File

@ -67,7 +67,7 @@ static void FN(Initialize)(
self->factor_remove *= self->factor;
}
self->table = (uint32_t*)common->extra;
self->table = (uint32_t*)common->extra[0];
for (i = 0; i < NUMBUCKETS; i++) {
self->table[i] = FN(kInvalidPos);
}
@ -88,13 +88,13 @@ static void FN(Prepare)(HashRolling* BROTLI_RESTRICT self, BROTLI_BOOL one_shot,
BROTLI_UNUSED(one_shot);
}
static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
static BROTLI_INLINE void FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
size_t input_size) {
return NUMBUCKETS * sizeof(uint32_t);
size_t input_size, size_t* alloc_size) {
BROTLI_UNUSED(params);
BROTLI_UNUSED(one_shot);
BROTLI_UNUSED(input_size);
alloc_size[0] = NUMBUCKETS * sizeof(uint32_t);
}
static BROTLI_INLINE void FN(Store)(HashRolling* BROTLI_RESTRICT self,

11
c/enc/hash_to_binary_tree_inc.h
View File

@ -57,8 +57,8 @@ typedef struct HashToBinaryTree {
static void FN(Initialize)(
HasherCommon* common, HashToBinaryTree* BROTLI_RESTRICT self,
const BrotliEncoderParams* params) {
self->buckets_ = (uint32_t*)common->extra;
self->forest_ = &self->buckets_[BUCKET_SIZE];
self->buckets_ = (uint32_t*)common->extra[0];
self->forest_ = (uint32_t*)common->extra[1];
self->window_mask_ = (1u << params->lgwin) - 1u;
self->invalid_pos_ = (uint32_t)(0 - self->window_mask_);
@ -78,14 +78,15 @@ static void FN(Prepare)
}
}
static BROTLI_INLINE size_t FN(HashMemAllocInBytes)(
static BROTLI_INLINE void FN(HashMemAllocInBytes)(
const BrotliEncoderParams* params, BROTLI_BOOL one_shot,
size_t input_size) {
size_t input_size, size_t* alloc_size) {
size_t num_nodes = (size_t)1 << params->lgwin;
if (one_shot && input_size < num_nodes) {
num_nodes = input_size;
}
return sizeof(uint32_t) * BUCKET_SIZE + 2 * sizeof(uint32_t) * num_nodes;
alloc_size[0] = sizeof(uint32_t) * BUCKET_SIZE;
alloc_size[1] = 2 * sizeof(uint32_t) * num_nodes;
}
static BROTLI_INLINE size_t FN(LeftChildIndex)(

2
java/org/brotli/dec/BitReader.java
View File

@ -211,7 +211,7 @@ final class BitReader {
return limit - s.halfOffset;
}
static void copyBytes(State s, byte[] data, int offset, int length) {
static void copyRawBytes(State s, byte[] data, int offset, int length) {
if ((s.bitOffset & 7) != 0) {
throw new BrotliRuntimeException("Unaligned copyBytes");
}

2
java/org/brotli/dec/Decode.java
View File

@ -903,7 +903,7 @@ final class Decode {
}
int chunkLength = Math.min(s.ringBufferSize - s.pos, s.metaBlockLength);
BitReader.copyBytes(s, ringBuffer, s.pos, chunkLength);
BitReader.copyRawBytes(s, ringBuffer, s.pos, chunkLength);
s.metaBlockLength -= chunkLength;
s.pos += chunkLength;
if (s.pos == s.ringBufferSize) {

28
java/org/brotli/dec/Dictionary.java
View File

@ -21,7 +21,7 @@ public final class Dictionary {
static final int MIN_DICTIONARY_WORD_LENGTH = 4;
static final int MAX_DICTIONARY_WORD_LENGTH = 31;
private static ByteBuffer data;
private static ByteBuffer data = null;
static final int[] offsets = new int[32];
static final int[] sizeBits = new int[32];
@ -39,45 +39,45 @@ public final class Dictionary {
}
}
public static void setData(ByteBuffer data, int[] sizeBits) {
if (!data.isDirect() || !data.isReadOnly()) {
throw new BrotliRuntimeException("data must be a direct read-only byte buffer");
public static void setData(ByteBuffer newData, int[] newSizeBits) {
if ((Utils.isDirect(newData) == 0) || (Utils.isReadOnly(newData) == 0)) {
throw new BrotliRuntimeException("newData must be a direct read-only byte buffer");
}
// TODO: is that so?
if (sizeBits.length > MAX_DICTIONARY_WORD_LENGTH) {
if (newSizeBits.length > MAX_DICTIONARY_WORD_LENGTH) {
throw new BrotliRuntimeException(
"sizeBits length must be at most " + MAX_DICTIONARY_WORD_LENGTH);
}
for (int i = 0; i < MIN_DICTIONARY_WORD_LENGTH; ++i) {
if (sizeBits[i] != 0) {
if (newSizeBits[i] != 0) {
throw new BrotliRuntimeException("first " + MIN_DICTIONARY_WORD_LENGTH + " must be 0");
}
}
int[] dictionaryOffsets = Dictionary.offsets;
int[] dictionarySizeBits = Dictionary.sizeBits;
System.arraycopy(sizeBits, 0, dictionarySizeBits, 0, sizeBits.length);
System.arraycopy(newSizeBits, 0, dictionarySizeBits, 0, newSizeBits.length);
int pos = 0;
int limit = data.capacity();
for (int i = 0; i < sizeBits.length; ++i) {
int limit = newData.capacity();
for (int i = 0; i < newSizeBits.length; ++i) {
dictionaryOffsets[i] = pos;
int bits = dictionarySizeBits[i];
if (bits != 0) {
if (bits >= 31) {
throw new BrotliRuntimeException("sizeBits values must be less than 31");
throw new BrotliRuntimeException("newSizeBits values must be less than 31");
}
pos += i << bits;
if (pos <= 0 || pos > limit) {
throw new BrotliRuntimeException("sizeBits is inconsistent: overflow");
throw new BrotliRuntimeException("newSizeBits is inconsistent: overflow");
}
}
}
for (int i = sizeBits.length; i < 32; ++i) {
for (int i = newSizeBits.length; i < 32; ++i) {
dictionaryOffsets[i] = pos;
}
if (pos != limit) {
throw new BrotliRuntimeException("sizeBits is inconsistent: underflow");
throw new BrotliRuntimeException("newSizeBits is inconsistent: underflow");
}
Dictionary.data = data;
Dictionary.data = newData;
}
public static ByteBuffer getData() {

11
java/org/brotli/dec/DictionaryData.java
View File

@ -62,10 +62,11 @@ final class DictionaryData {
}
static {
ByteBuffer dictionary = ByteBuffer.allocateDirect(122784);
int[] sizeBits = new int[25];
unpackDictionaryData(dictionary, DATA0, DATA1, SKIP_FLIP, sizeBits, SIZE_BITS_DATA);
Utils.flipBuffer(dictionary);
Dictionary.setData(dictionary.asReadOnlyBuffer(), sizeBits);
ByteBuffer dictionaryData = ByteBuffer.allocateDirect(122784);
int[] dictionarySizeBits = new int[25];
unpackDictionaryData(
dictionaryData, DATA0, DATA1, SKIP_FLIP, dictionarySizeBits, SIZE_BITS_DATA);
Utils.flipBuffer(dictionaryData);
Dictionary.setData(Utils.asReadOnlyBuffer(dictionaryData), dictionarySizeBits);
}
}

13
java/org/brotli/dec/Utils.java
View File

@ -10,6 +10,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.nio.Buffer;
import java.nio.ByteBuffer;
/**
* A set of utility methods.
@ -88,6 +89,18 @@ final class Utils {
}
}
static ByteBuffer asReadOnlyBuffer(ByteBuffer src) {
return src.asReadOnlyBuffer();
}
static int isReadOnly(ByteBuffer src) {
return src.isReadOnly() ? 1 : 0;
}
static int isDirect(ByteBuffer src) {
return src.isDirect() ? 1 : 0;
}
// Crazy pills factory: code compiled for JDK8 does not work on JRE9.
static void flipBuffer(Buffer buffer) {
buffer.flip();

1
js/BUILD
View File

@ -25,6 +25,7 @@ closure_js_library(
srcs = ["decode.js"],
suppress = [
"JSC_DUP_VAR_DECLARATION",
"JSC_INVALID_OCTAL_LITERAL",
"JSC_USELESS_BLOCK",
],
deps = [":polyfill"],

17
js/WORKSPACE
View File

@ -1,12 +1,17 @@
workspace(name = "org_brotli_js")
load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
git_repository(
http_archive(
name = "io_bazel_rules_closure",
commit = "29ec97e7c85d607ba9e41cab3993fbb13f812c4b",
remote = "https://github.com/bazelbuild/rules_closure.git",
sha256 = "d66deed38a0bb20581c15664f0ab62270af5940786855c7adc3087b27168b529",
strip_prefix = "rules_closure-0.11.0",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/0.11.0.tar.gz",
"https://github.com/bazelbuild/rules_closure/archive/0.11.0.tar.gz",
],
)
load("@io_bazel_rules_closure//closure:defs.bzl", "closure_repositories")
closure_repositories()
load("@io_bazel_rules_closure//closure:repositories.bzl", "rules_closure_dependencies", "rules_closure_toolchains")
rules_closure_dependencies()
rules_closure_toolchains()

828
js/decode.js
View File

File diff suppressed because one or more lines are too long

4
js/decode.min.js vendored
View File

File diff suppressed because one or more lines are too long

18
js/decode_test.js
View File

@ -9,6 +9,16 @@ function bytesToString(bytes) {
return String.fromCharCode.apply(null, new Uint16Array(bytes));
}
/**
* @param {string} str
* @return {!Int8Array}
*/
function stringToBytes(str) {
var out = new Int8Array(str.length);
for (var i = 0; i < str.length; ++i) out[i] = str.charCodeAt(i);
return out;
}
function testMetadata() {
assertEquals("", bytesToString(BrotliDecode(Int8Array.from([1, 11, 0, 42, 3]))));
}
@ -77,3 +87,11 @@ function testIntactDistanceRingBuffer0() {
var output = BrotliDecode(input);
assertEquals("himselfself", bytesToString(output));
}
function testCompoundDictionary() {
var txt = "kot lomom kolol slona\n";
var dictionary = stringToBytes(txt);
var compressed = [0xa1, 0xa8, 0x00, 0xc0, 0x2f, 0x01, 0x10, 0xc4, 0x44, 0x09, 0x00];
assertEquals(txt.length, compressed.length * 2);
assertEquals(txt, bytesToString(BrotliDecode(Int8Array.from(compressed), {customDictionary: dictionary})));
}

3
scripts/.travis.sh
View File

@ -62,7 +62,8 @@ case "$1" in
bazel build -c opt ...:all &&
cd go && bazel test -c opt ...:all && cd .. &&
cd java && bazel test -c opt ...:all && cd .. &&
cd js && bazel test -c opt ...:all && cd .. &&
echo "bazelbuild/rules_closure seems to be dead; js build is skipped" &&
echo "cd js && bazel test -c opt ...:all && cd .." &&
cd research && bazel build -c opt ...:all && cd ..
;;
esac