AuroraMiddleware/brotli
1
0
Fork 0
mirror of https://github.com/google/brotli.git synced 2025-01-01 04:40:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #338 from eustas/master

Browse Source 
Update decoder. Add encoder interface wrapper.
This commit is contained in:
eustas 2016-04-21 17:50:53 +02:00
commit 510131d1db
11 changed files with 149 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
dec/bit_reader.h
View File

@ -14,10 +14,6 @@
#include "./port.h"
#include "./types.h"
#ifdef BROTLI_DECODE_DEBUG
#include <stdio.h>
#endif
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
@ -66,13 +62,13 @@ typedef struct {
} BrotliBitReaderState;
/* Initializes the bitreader fields. */
void BrotliInitBitReader(BrotliBitReader* const br);
BROTLI_INTERNAL void BrotliInitBitReader(BrotliBitReader* const br);
/* Ensures that accumulator is not empty. May consume one byte of input.
Returns 0 if data is required but there is no input available.
For BROTLI_ALIGNED_READ this function also prepares bit reader for aligned
reading. */
int BrotliWarmupBitReader(BrotliBitReader* const br);
BROTLI_INTERNAL int BrotliWarmupBitReader(BrotliBitReader* const br);
static BROTLI_INLINE void BrotliBitReaderSaveState(
BrotliBitReader* const from, BrotliBitReaderState* to) {
@ -298,10 +294,8 @@ static BROTLI_INLINE void BrotliBitReaderUnload(BrotliBitReader* br) {
static BROTLI_INLINE void BrotliTakeBits(
BrotliBitReader* const br, uint32_t n_bits, uint32_t* val) {
*val = (uint32_t)BrotliGetBitsUnmasked(br) & BitMask(n_bits);
#ifdef BROTLI_DECODE_DEBUG
printf("[BrotliReadBits] %d %d %d val: %6x\n",
(int)br->avail_in, (int)br->bit_pos_, n_bits, (int)*val);
#endif
BROTLI_LOG(("[BrotliReadBits] %d %d %d val: %6x\n",
(int)br->avail_in, (int)br->bit_pos_, n_bits, (int)*val));
BrotliDropBits(br, n_bits);
}

55
dec/decode.c
View File

@ -10,7 +10,6 @@
#include <arm_neon.h>
#endif
#include <stdio.h> /* printf (debug output) */
#include <stdlib.h> /* free, malloc */
#include <string.h> /* memcpy, memset */
@ -20,38 +19,20 @@
#include "./huffman.h"
#include "./port.h"
#include "./prefix.h"
#include "./state.h"
#include "./transform.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
/* BROTLI_FAILURE macro unwraps to BROTLI_RESULT_ERROR in non-debug build. */
/* In debug build it dumps file name, line and pretty function name. */
#if defined(_MSC_VER) || \
(!defined(BROTLI_DEBUG) && !defined(BROTLI_DECODE_DEBUG))
#define BROTLI_FAILURE() BROTLI_RESULT_ERROR
#else
#define BROTLI_FAILURE() BrotliFailure(__FILE__, __LINE__, __PRETTY_FUNCTION__)
static inline BrotliResult BrotliFailure(const char* f, int l, const char* fn) {
fprintf(stderr, "ERROR at %s:%d (%s)\n", f, l, fn);
fflush(stderr);
return BROTLI_RESULT_ERROR;
}
#endif
#define BROTLI_FAILURE() (BROTLI_DUMP(), BROTLI_RESULT_ERROR)
#ifdef BROTLI_DECODE_DEBUG
#define BROTLI_LOG_UINT(name) \
printf("[%s] %s = %lu\n", __func__, #name, (unsigned long)(name))
BROTLI_LOG(("[%s] %s = %lu\n", __func__, #name, (unsigned long)(name)))
#define BROTLI_LOG_ARRAY_INDEX(array_name, idx) \
printf("[%s] %s[%lu] = %lu\n", __func__, #array_name, \
(unsigned long)(idx), (unsigned long)array_name[idx])
#define BROTLI_LOG(x) printf x
#else
#define BROTLI_LOG_UINT(name)
#define BROTLI_LOG_ARRAY_INDEX(array_name, idx)
#define BROTLI_LOG(x)
#endif
BROTLI_LOG(("[%s] %s[%lu] = %lu\n", __func__, #array_name, \
(unsigned long)(idx), (unsigned long)array_name[idx]))
static const uint32_t kDefaultCodeLength = 8;
static const uint32_t kCodeLengthRepeatCode = 16;
@ -89,7 +70,7 @@ BrotliState* BrotliCreateState(
state = (BrotliState*)alloc_func(opaque, sizeof(BrotliState));
}
if (state == 0) {
(void)BROTLI_FAILURE();
BROTLI_DUMP();
return 0;
}
BrotliStateInitWithCustomAllocators(state, alloc_func, free_func, opaque);
@ -376,7 +357,6 @@ static BROTLI_INLINE int SafeReadSymbol(const HuffmanCode* table,
return SafeDecodeSymbol(table, br, result);
}
/* Makes a look-up in first level Huffman table. Peeks 8 bits. */
static BROTLI_INLINE void PreloadSymbol(int safe,
const HuffmanCode* table,
@ -514,7 +494,7 @@ static BROTLI_INLINE void ProcessRepeatedCodeLength(uint32_t code_len,
*repeat += repeat_delta + 3U;
repeat_delta = *repeat - old_repeat;
if (*symbol + repeat_delta > alphabet_size) {
(void)BROTLI_FAILURE();
BROTLI_DUMP();
*symbol = alphabet_size;
*space = 0xFFFFF;
return;
@ -886,7 +866,6 @@ static BROTLI_NOINLINE void InverseMoveToFrontTransform(uint8_t* v,
state->mtf_upper_bound = upper_bound;
}
/* Decodes a series of Huffman table using ReadHuffmanCode function. */
static BrotliResult HuffmanTreeGroupDecode(HuffmanTreeGroup* group,
BrotliState* s) {
@ -1248,7 +1227,7 @@ static BrotliResult BROTLI_NOINLINE CopyUncompressedBlockToOutput(
}
}
}
return BROTLI_FAILURE();
BROTLI_DCHECK(0); /* Unreachable */
}
int BrotliDecompressedSize(size_t encoded_size,
@ -1287,7 +1266,8 @@ int BrotliDecompressedSize(size_t encoded_size,
static void BROTLI_NOINLINE BrotliCalculateRingBufferSize(BrotliState* s,
BrotliBitReader* br) {
int is_last = s->is_last_metablock;
s->ringbuffer_size = 1 << s->window_bits;
int window_size = 1 << s->window_bits;
s->ringbuffer_size = window_size;
if (s->is_uncompressed) {
int next_block_header =
@ -1299,20 +1279,21 @@ static void BROTLI_NOINLINE BrotliCalculateRingBufferSize(BrotliState* s,
}
}
/* Limit custom dictionary size to stream window size. */
if (s->custom_dict_size >= window_size) {
s->custom_dict += s->custom_dict_size - window_size;
s->custom_dict_size = window_size;
}
/* We need at least 2 bytes of ring buffer size to get the last two
bytes for context from there */
if (is_last) {
while (s->ringbuffer_size >= s->meta_block_remaining_len * 2 &&
s->ringbuffer_size > 32) {
int min_size_x2 = (s->meta_block_remaining_len + s->custom_dict_size) * 2;
while (s->ringbuffer_size >= min_size_x2 && s->ringbuffer_size > 32) {
s->ringbuffer_size >>= 1;
}
}
/* But make it fit the custom dictionary if there is one. */
while (s->ringbuffer_size < s->custom_dict_size) {
s->ringbuffer_size <<= 1;
}
s->ringbuffer_mask = s->ringbuffer_size - 1;
}

18
dec/decode.h
View File

@ -9,13 +9,14 @@
#ifndef BROTLI_DEC_DECODE_H_
#define BROTLI_DEC_DECODE_H_
#include "./state.h"
#include "./types.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
typedef struct BrotliStateStruct BrotliState;
typedef enum {
/* Decoding error, e.g. corrupt input or memory allocation problem */
BROTLI_RESULT_ERROR = 0,
@ -80,14 +81,21 @@ BrotliResult BrotliDecompressStream(size_t* available_in,
Not to be confused with the built-in transformable dictionary of Brotli.
The dictionary must exist in memory until decoding is done and is owned by
the caller. To use:
1) initialize state with BrotliStateInit
2) use BrotliSetCustomDictionary
3) use BrotliDecompressStream
4) clean up with BrotliStateCleanup
1) Allocate and initialize state with BrotliCreateState
2) Use BrotliSetCustomDictionary
3) Use BrotliDecompressStream
4) Clean up and free state with BrotliDestroyState
*/
void BrotliSetCustomDictionary(
size_t size, const uint8_t* dict, BrotliState* s);
/* Returns 1, if s is in a state where we have not read any input bytes yet,
and 0 otherwise */
int BrotliStateIsStreamStart(const BrotliState* s);
/* Returns 1, if s is in a state where we reached the end of the input and
produced all of the output, and 0 otherwise. */
int BrotliStateIsStreamEnd(const BrotliState* s);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */

1
dec/huffman.c
View File

@ -99,7 +99,6 @@ static BROTLI_INLINE int NextTableBitSize(const uint16_t* const count,
return len - root_bits;
}
void BrotliBuildCodeLengthsHuffmanTable(HuffmanCode* table,
const uint8_t* const code_lengths,
uint16_t* count) {

19
dec/huffman.h
View File

@ -10,6 +10,7 @@
#define BROTLI_DEC_HUFFMAN_H_
#include "./types.h"
#include "./port.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@ -36,27 +37,21 @@ typedef struct {
uint16_t value; /* symbol value or table offset */
} HuffmanCode;
/* Builds Huffman lookup table assuming code lengths are in symbol order. */
void BrotliBuildCodeLengthsHuffmanTable(HuffmanCode* root_table,
const uint8_t* const code_lengths,
uint16_t* count);
BROTLI_INTERNAL void BrotliBuildCodeLengthsHuffmanTable(HuffmanCode* root_table,
const uint8_t* const code_lengths, uint16_t* count);
/* Builds Huffman lookup table assuming code lengths are in symbol order. */
/* Returns size of resulting table. */
uint32_t BrotliBuildHuffmanTable(HuffmanCode* root_table,
int root_bits,
const uint16_t* const symbol_lists,
uint16_t* count_arg);
BROTLI_INTERNAL uint32_t BrotliBuildHuffmanTable(HuffmanCode* root_table,
int root_bits, const uint16_t* const symbol_lists, uint16_t* count_arg);
/* Builds a simple Huffman table. The num_symbols parameter is to be */
/* interpreted as follows: 0 means 1 symbol, 1 means 2 symbols, 2 means 3 */
/* symbols, 3 means 4 symbols with lengths 2,2,2,2, 4 means 4 symbols with */
/* lengths 1,2,3,3. */
uint32_t BrotliBuildSimpleHuffmanTable(HuffmanCode* table,
int root_bits,
uint16_t* symbols,
uint32_t num_symbols);
BROTLI_INTERNAL uint32_t BrotliBuildSimpleHuffmanTable(HuffmanCode* table,
int root_bits, uint16_t* symbols, uint32_t num_symbols);
/* Contains a collection of Huffman trees with the same alphabet size. */
typedef struct {

36
dec/port.h
View File

@ -18,13 +18,16 @@
read and overlapping memcpy; this reduces decompression speed by 5%
* BROTLI_DEBUG dumps file name and line number when decoder detects stream
or memory error
* BROTLI_DECODE_DEBUG enables asserts and dumps various state information
* BROTLI_ENABLE_LOG enables asserts and dumps various state information
*/
#ifndef BROTLI_DEC_PORT_H_
#define BROTLI_DEC_PORT_H_
#if defined(BROTLI_ENABLE_LOG) || defined(BROTLI_DEBUG)
#include <assert.h>
#include <stdio.h>
#endif
/* Compatibility with non-clang compilers. */
#ifndef __has_builtin
@ -84,12 +87,13 @@
#endif
#ifdef BROTLI_BUILD_PORTABLE
#define BROTLI_ALIGNED_READ 1
#define BROTLI_ALIGNED_READ (!!1)
#elif defined(BROTLI_TARGET_X86) || defined(BROTLI_TARGET_X64) || \
defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8)
#define BROTLI_ALIGNED_READ 0 /* Allow unaligned access on whitelisted CPUs. */
/* Allow unaligned read only for whitelisted CPUs. */
#define BROTLI_ALIGNED_READ (!!0)
#else
#define BROTLI_ALIGNED_READ 1
#define BROTLI_ALIGNED_READ (!!1)
#endif
/* Define "PREDICT_TRUE" and "PREDICT_FALSE" macros for capable compilers.
@ -130,6 +134,16 @@ OR:
#define ATTRIBUTE_ALWAYS_INLINE
#endif
#if BROTLI_MODERN_COMPILER || __has_attribute(visibility)
#define ATTRIBUTE_VISIBILITY_HIDDEN __attribute__ ((visibility ("hidden")))
#else
#define ATTRIBUTE_VISIBILITY_HIDDEN
#endif
#ifndef BROTLI_INTERNAL
#define BROTLI_INTERNAL ATTRIBUTE_VISIBILITY_HIDDEN
#endif
#ifndef _MSC_VER
#if defined(__cplusplus) || !defined(__STRICT_ANSI__) || \
__STDC_VERSION__ >= 199901L
@ -141,10 +155,22 @@ OR:
#define BROTLI_INLINE __forceinline
#endif /* _MSC_VER */
#ifdef BROTLI_DECODE_DEBUG
#ifdef BROTLI_ENABLE_LOG
#define BROTLI_DCHECK(x) assert(x)
#define BROTLI_LOG(x) printf x
#else
#define BROTLI_DCHECK(x)
#define BROTLI_LOG(x)
#endif
#if defined(BROTLI_DEBUG) || defined(BROTLI_ENABLE_LOG)
static inline void BrotliDump(const char* f, int l, const char* fn) {
fprintf(stderr, "%s:%d (%s)\n", f, l, fn);
fflush(stderr);
}
#define BROTLI_DUMP() BrotliDump(__FILE__, __LINE__, __FUNCTION__)
#else
#define BROTLI_DUMP() (void)(0)
#endif
#if defined(BROTLI_BUILD_64_BIT)

5
dec/state.c
View File

@ -15,6 +15,10 @@
extern "C" {
#endif
/* Declared in decode.h */
int BrotliStateIsStreamStart(const BrotliState* s);
int BrotliStateIsStreamEnd(const BrotliState* s);
static void* DefaultAllocFunc(void* opaque, size_t size) {
BROTLI_UNUSED(opaque);
return malloc(size);
@ -76,7 +80,6 @@ void BrotliStateInitWithCustomAllocators(BrotliState* s,
s->distance_hgroup.codes = NULL;
s->distance_hgroup.htrees = NULL;
s->custom_dict = NULL;
s->custom_dict_size = 0;

34
dec/state.h
View File

@ -12,6 +12,7 @@
#include "./bit_reader.h"
#include "./huffman.h"
#include "./types.h"
#include "./port.h"
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
@ -156,7 +157,6 @@ struct BrotliStateStruct {
uint32_t repeat_code_len;
uint32_t prev_code_len;
int copy_length;
int distance_code;
@ -220,27 +220,19 @@ struct BrotliStateStruct {
uint8_t* context_modes;
};
typedef struct BrotliStateStruct BrotliState;
typedef struct BrotliStateStruct BrotliStateInternal;
#define BrotliState BrotliStateInternal
void BrotliStateInit(BrotliState* s);
void BrotliStateInitWithCustomAllocators(BrotliState* s,
brotli_alloc_func alloc_func,
brotli_free_func free_func,
void* opaque);
void BrotliStateCleanup(BrotliState* s);
void BrotliStateMetablockBegin(BrotliState* s);
void BrotliStateCleanupAfterMetablock(BrotliState* s);
void BrotliHuffmanTreeGroupInit(BrotliState* s, HuffmanTreeGroup* group,
uint32_t alphabet_size, uint32_t ntrees);
void BrotliHuffmanTreeGroupRelease(BrotliState* s, HuffmanTreeGroup* group);
/* Returns 1, if s is in a state where we have not read any input bytes yet,
and 0 otherwise */
int BrotliStateIsStreamStart(const BrotliState* s);
/* Returns 1, if s is in a state where we reached the end of the input and
produced all of the output, and 0 otherwise. */
int BrotliStateIsStreamEnd(const BrotliState* s);
BROTLI_INTERNAL void BrotliStateInit(BrotliState* s);
BROTLI_INTERNAL void BrotliStateInitWithCustomAllocators(BrotliState* s,
brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
BROTLI_INTERNAL void BrotliStateCleanup(BrotliState* s);
BROTLI_INTERNAL void BrotliStateMetablockBegin(BrotliState* s);
BROTLI_INTERNAL void BrotliStateCleanupAfterMetablock(BrotliState* s);
BROTLI_INTERNAL void BrotliHuffmanTreeGroupInit(BrotliState* s,
HuffmanTreeGroup* group, uint32_t alphabet_size, uint32_t ntrees);
BROTLI_INTERNAL void BrotliHuffmanTreeGroupRelease(BrotliState* s,
HuffmanTreeGroup* group);
#if defined(__cplusplus) || defined(c_plusplus)
} /* extern "C" */

15
enc/compressor.h Normal file
View File

@ -0,0 +1,15 @@
/* Copyright 2016 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* C++ API for Brotli compression. */
#ifndef BROTLI_ENC_COMPRESSOR_H_
#define BROTLI_ENC_COMPRESSOR_H_
#include "./encode.h"
#include "./streams.h"
#endif /* BROTLI_ENC_COMPRESSOR_H_ */

9
python/brotlimodule.cc
View File

@ -220,10 +220,9 @@ static PyObject* brotli_decompress(PyObject *self, PyObject *args, PyObject *key
std::vector<uint8_t> output;
const size_t kBufferSize = 65536;
uint8_t* buffer = new uint8_t[kBufferSize];
BrotliState state;
BrotliStateInit(&state);
BrotliState* state = BrotliCreateState(0, 0, 0);
if (custom_dictionary_length != 0) {
BrotliSetCustomDictionary(custom_dictionary_length, custom_dictionary, &state);
BrotliSetCustomDictionary(custom_dictionary_length, custom_dictionary, state);
}
BrotliResult result = BROTLI_RESULT_NEEDS_MORE_OUTPUT;
@ -233,7 +232,7 @@ static PyObject* brotli_decompress(PyObject *self, PyObject *args, PyObject *key
size_t total_out = 0;
result = BrotliDecompressStream(&length, &input,
&available_out, &next_out,
&total_out, &state);
&total_out, state);
size_t used_out = kBufferSize - available_out;
if (used_out != 0)
output.insert(output.end(), buffer, buffer + used_out);
@ -245,7 +244,7 @@ static PyObject* brotli_decompress(PyObject *self, PyObject *args, PyObject *key
PyErr_SetString(BrotliError, "BrotliDecompress failed");
}
BrotliStateCleanup(&state);
BrotliDestroyState(state);
delete[] buffer;
return ret;

53
tools/bro.cc
View File

@ -10,13 +10,42 @@
#include <stdio.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <string>
#include "../dec/decode.h"
#include "../enc/encode.h"
#include "../enc/compressor.h"
#if !defined(_WIN32)
#include <unistd.h>
#else
#include <io.h>
#define STDIN_FILENO _fileno(stdin)
#define STDOUT_FILENO _fileno(stdout)
#define S_IRUSR S_IREAD
#define S_IWUSR S_IWRITE
#define fdopen _fdopen
#define unlink _unlink
#define fopen ms_fopen
#define open ms_open
static inline FILE* ms_fopen(const char *filename, const char *mode) {
FILE* result = 0;
fopen_s(&result, filename, mode);
return result;
}
static inline int ms_open(const char *filename, int oflag, int pmode) {
int result = -1;
_sopen_s(&result, filename, oflag | O_BINARY, _SH_DENYNO, pmode);
return result;
}
#endif /* MSC_VER */
static bool ParseQuality(const char* s, int* quality) {
@ -146,13 +175,8 @@ static FILE *OpenOutputFile(const char *output_path, const int force) {
return fdopen(STDOUT_FILENO, "wb");
}
int excl = force ? 0 : O_EXCL;
#if defined(_WIN32)
int fd = open(output_path, O_CREAT | excl | O_WRONLY | O_TRUNC | O_BINARY,
S_IREAD | S_IWRITE);
#else
int fd = open(output_path, O_CREAT | excl | O_WRONLY | O_TRUNC,
S_IRUSR | S_IWUSR);
#endif
if (fd < 0) {
if (!force) {
struct stat statbuf;
@ -186,6 +210,11 @@ static int64_t FileSize(char *path) {
static const size_t kFileBufferSize = 65536;
static void Decompresss(FILE* fin, FILE* fout) {
BrotliState* s = BrotliCreateState(NULL, NULL, NULL);
if (!s) {
fprintf(stderr, "out of memory\n");
exit(1);
}
uint8_t* input = new uint8_t[kFileBufferSize];
uint8_t* output = new uint8_t[kFileBufferSize];
size_t total_out;
@ -194,8 +223,6 @@ static void Decompresss(FILE* fin, FILE* fout) {
size_t available_out = kFileBufferSize;
uint8_t* next_out = output;
BrotliResult result = BROTLI_RESULT_NEEDS_MORE_INPUT;
BrotliState s;
BrotliStateInit(&s);
while (1) {
if (result == BROTLI_RESULT_NEEDS_MORE_INPUT) {
if (feof(fin)) {
@ -217,14 +244,14 @@ static void Decompresss(FILE* fin, FILE* fout) {
break; /* Error or success. */
}
result = BrotliDecompressStream(&available_in, &next_in,
&available_out, &next_out, &total_out, &s);
&available_out, &next_out, &total_out, s);
}
if (next_out != output) {
fwrite(output, 1, next_out - output, fout);
fwrite(output, 1, static_cast<size_t>(next_out - output), fout);
}
BrotliStateCleanup(&s);
delete[] input;
delete[] output;
BrotliDestroyState(s);
if ((result == BROTLI_RESULT_NEEDS_MORE_OUTPUT) || ferror(fout)) {
fprintf(stderr, "failed to write output\n");
exit(1);
@ -291,7 +318,7 @@ int main(int argc, char** argv) {
exit(1);
}
double uncompressed_bytes_in_MB =
(repeat * uncompressed_size) / (1024.0 * 1024.0);
static_cast<double>(repeat * uncompressed_size) / (1024.0 * 1024.0);
if (decompress) {
printf("Brotli decompression speed: ");
} else {