AuroraMiddleware/brotli
1
0
Fork 0
mirror of https://github.com/google/brotli.git synced 2024-11-09 13:40:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into dependabot/github_actions/actions/setup-python-4.7.1

Browse Source
This commit is contained in:
Eugene Kliuchnikov 2023-12-08 15:35:35 +01:00 committed by GitHub
commit bf867c126b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 107 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
.github/workflows/build_test.yml vendored
View File

@ -43,28 +43,28 @@ jobs:
# TODO: consider running this combination in docker
#- name: cmake:clang3.5
- name: cmake:clang12
- name: cmake:clang15
build_system: cmake
c_compiler: clang-12
cxx_compiler: clang++12
c_compiler: clang-15
cxx_compiler: clang++15
- name: cmake:clang12:asan
- name: cmake:clang15:asan
build_system: cmake
sanitizer: address
c_compiler: clang-12
cxx_compiler: clang++12
c_compiler: clang-15
cxx_compiler: clang++15
- name: cmake:clang12:tsan
- name: cmake:clang15:tsan
build_system: cmake
sanitizer: thread
c_compiler: clang-12
cxx_compiler: clang++12
c_compiler: clang-15
cxx_compiler: clang++15
- name: cmake:clang12:ubsan
- name: cmake:clang15:ubsan
build_system: cmake
sanitizer: undefined
c_compiler: clang-12
cxx_compiler: clang++-12
c_compiler: clang-15
cxx_compiler: clang++-15
c_flags: -fno-sanitize-recover=undefined,integer
- name: cmake:qemu-arm-neon-gcc
@ -98,10 +98,10 @@ jobs:
cmake_config: Debug
os: windows-2019
- name: fuzz:clang12
- name: fuzz:clang15
build_system: fuzz
c_compiler: clang-12
cxx_compiler: clang++12
c_compiler: clang-15
cxx_compiler: clang++15
# TODO: consider running this combination in docker
#- name: python2.7:gcc5
@ -131,15 +131,15 @@ jobs:
# TODO: consider running this combination in docker
#- name: python3.8:gcc5
- name: python39:clang12
- name: python3.10:clang15
build_system: python
python_version: 3.9
c_compiler: clang-12
cxx_compiler: clang++-12
python_version: "3.10"
c_compiler: clang-15
cxx_compiler: clang++-15
- name: python39-win
- name: python3.10-win
build_system: python
python_version: 3.9
python_version: "3.10"
# TODO: investigate why win-builds can't run tests
py_setuptools_cmd: build_ext
os: windows-2019
@ -195,7 +195,7 @@ jobs:
- name: bazel-win:root
build_system: bazel
bazel_project: .
os: windows-2019
os: windows-latest
# TODO: use single dll on windows, otherwise it fails to link
#- name: bazel-win:go
@ -206,7 +206,7 @@ jobs:
- name: bazel-win:java
build_system: bazel
bazel_project: java
os: windows-2019
os: windows-latest
# TODO: blocked by Bazel Closure rules issue
#- name: bazel-win:js
@ -217,7 +217,7 @@ jobs:
- name: bazel-win:research
build_system: bazel
bazel_project: research
os: windows-2019
os: windows-latest
env:
CC: ${{ matrix.c_compiler || 'gcc' }}
@ -226,7 +226,7 @@ jobs:
steps:
- name: Install extra deps @ Ubuntu
if: ${{ runner.os == 'Linux' }}
# Already installed: bazel, clang{10-12}, cmake, gcc{9,10}, java{8,11}, maven, python{3.5-3.9}
# Already installed: bazel, clang{13-15}, cmake, gcc{9.5-13.1}, java{8,11,17,21}, maven, python{3.10}
run: |
EXTRA_PACKAGES="${{ matrix.extra_apt_pkgs || '' }}"
sudo apt update

2
.github/workflows/release.yaml vendored
View File

@ -23,7 +23,7 @@ concurrency:
jobs:
windows_build:
name: Windows Build (vcpkg / ${{ matrix.triplet }})
runs-on: [windows-2022]
runs-on: [windows-latest]
strategy:
fail-fast: false
matrix:

33
CMakeLists.txt
View File

@ -11,10 +11,11 @@ cmake_policy(SET CMP0048 NEW)
project(brotli C)
option(BUILD_SHARED_LIBS "Build shared libraries" ON)
set(BROTLI_BUILD_TOOLS ON CACHE BOOL "Build/install CLI tools")
if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
message(STATUS "Setting build type to Release as none was specified.")
set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build." FORCE)
set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build" FORCE)
else()
message(STATUS "Build type is '${CMAKE_BUILD_TYPE}'")
endif()
@ -81,7 +82,7 @@ include(CheckFunctionExists)
set(LIBM_LIBRARY)
CHECK_FUNCTION_EXISTS(log2 LOG2_RES)
if(NOT LOG2_RES)
set(orig_req_libs "${CMAKE_REQUIRED_LIBRARIES}")
set(_ORIG_REQ_LIBS "${CMAKE_REQUIRED_LIBRARIES}")
set(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES};m")
CHECK_FUNCTION_EXISTS(log2 LOG2_LIBM_RES)
if(LOG2_LIBM_RES)
@ -91,9 +92,9 @@ if(NOT LOG2_RES)
add_definitions(-DBROTLI_HAVE_LOG2=0)
endif()
set(CMAKE_REQUIRED_LIBRARIES "${orig_req_libs}")
set(CMAKE_REQUIRED_LIBRARIES "${_ORIG_REQ_LIBS}")
unset(LOG2_LIBM_RES)
unset(orig_req_libs)
unset(_ORIG_REQ_LIBS)
else()
add_definitions(-DBROTLI_HAVE_LOG2=1)
endif()
@ -169,15 +170,19 @@ if(BROTLI_PARENT_DIRECTORY)
endif()
# Build the brotli executable
if(BROTLI_BUILD_TOOLS)
add_executable(brotli c/tools/brotli.c)
target_link_libraries(brotli ${BROTLI_LIBRARIES})
endif()
# Installation
if(NOT BROTLI_BUNDLED_MODE)
if (BROTLI_BUILD_TOOLS)
install(
TARGETS brotli
RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
)
endif()
install(
TARGETS ${BROTLI_LIBRARIES_CORE}
@ -194,9 +199,10 @@ endif() # BROTLI_BUNDLED_MODE
# Tests
# Integration tests, those depend on `brotli` binary
if(NOT BROTLI_DISABLE_TESTS AND BROTLI_BUILD_TOOLS)
# If we're targeting Windows but not running on Windows, we need Wine
# to run the tests...
if(NOT BROTLI_DISABLE_TESTS)
if(WIN32 AND NOT CMAKE_HOST_WIN32)
find_program(BROTLI_WRAPPER NAMES wine)
@ -205,11 +211,8 @@ if(NOT BROTLI_DISABLE_TESTS)
set(BROTLI_DISABLE_TESTS TRUE)
endif()
endif()
endif()
# If our compiler is a cross-compiler that we know about (arm/aarch64),
# then we need to use qemu to execute the tests.
if(NOT BROTLI_DISABLE_TESTS)
if ("${CMAKE_C_COMPILER}" MATCHES "^.*/arm-linux-gnueabihf-.*$")
message(STATUS "Detected arm-linux-gnueabihf cross-compilation")
set(BROTLI_WRAPPER "qemu-arm")
@ -227,9 +230,7 @@ if(NOT BROTLI_DISABLE_TESTS)
set(BROTLI_WRAPPER "qemu-aarch64")
set(BROTLI_WRAPPER_LD_PREFIX "/usr/aarch64-linux-gnu")
endif()
endif()
if(NOT BROTLI_DISABLE_TESTS)
include(CTest)
enable_testing()
@ -279,7 +280,7 @@ if(NOT BROTLI_DISABLE_TESTS)
-DINPUT=${CMAKE_CURRENT_SOURCE_DIR}/${INPUT}
-P ${CMAKE_CURRENT_SOURCE_DIR}/tests/run-compatibility-test.cmake)
endforeach()
endif()
endif() # BROTLI_DISABLE_TESTS
# Generate a pkg-config files
@ -359,10 +360,14 @@ if(NOT BROTLI_BUNDLED_MODE)
DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
endif() # BROTLI_BUNDLED_MODE
INSTALL(FILES "docs/brotli.1" DESTINATION "${SHARE_INSTALL_PREFIX}/man/man1")
INSTALL(FILES docs/constants.h.3 docs/decode.h.3 docs/encode.h.3 docs/types.h.3
if (BROTLI_BUILD_TOOLS)
install(FILES "docs/brotli.1"
DESTINATION "${SHARE_INSTALL_PREFIX}/man/man1")
endif()
install(FILES docs/constants.h.3 docs/decode.h.3 docs/encode.h.3 docs/types.h.3
DESTINATION "${SHARE_INSTALL_PREFIX}/man/man3")
if (ENABLE_COVERAGE STREQUAL "yes")
SETUP_TARGET_FOR_COVERAGE(coverage test coverage)
setup_target_for_coverage(coverage test coverage)
endif()

18
c/enc/encode.c
View File

@ -271,18 +271,18 @@ static void ChooseContextMap(int quality,
uint32_t two_prefix_histo[6] = { 0 };
size_t total;
size_t i;
size_t dummy;
size_t sink;
double entropy[4];
for (i = 0; i < 9; ++i) {
monogram_histo[i % 3] += bigram_histo[i];
two_prefix_histo[i % 6] += bigram_histo[i];
}
entropy[1] = ShannonEntropy(monogram_histo, 3, &dummy);
entropy[2] = (ShannonEntropy(two_prefix_histo, 3, &dummy) +
ShannonEntropy(two_prefix_histo + 3, 3, &dummy));
entropy[1] = ShannonEntropy(monogram_histo, 3, &sink);
entropy[2] = (ShannonEntropy(two_prefix_histo, 3, &sink) +
ShannonEntropy(two_prefix_histo + 3, 3, &sink));
entropy[3] = 0;
for (i = 0; i < 3; ++i) {
entropy[3] += ShannonEntropy(bigram_histo + 3 * i, 3, &dummy);
entropy[3] += ShannonEntropy(bigram_histo + 3 * i, 3, &sink);
}
total = monogram_histo[0] + monogram_histo[1] + monogram_histo[2];
@ -348,7 +348,7 @@ static BROTLI_BOOL ShouldUseComplexStaticContextMap(const uint8_t* input,
uint32_t* BROTLI_RESTRICT const context_histo = arena + 32;
uint32_t total = 0;
double entropy[3];
size_t dummy;
size_t sink;
size_t i;
ContextLut utf8_lut = BROTLI_CONTEXT_LUT(CONTEXT_UTF8);
memset(arena, 0, sizeof(arena[0]) * 32 * 14);
@ -370,10 +370,10 @@ static BROTLI_BOOL ShouldUseComplexStaticContextMap(const uint8_t* input,
prev1 = literal;
}
}
entropy[1] = ShannonEntropy(combined_histo, 32, &dummy);
entropy[1] = ShannonEntropy(combined_histo, 32, &sink);
entropy[2] = 0;
for (i = 0; i < 13; ++i) {
entropy[2] += ShannonEntropy(context_histo + (i << 5), 32, &dummy);
entropy[2] += ShannonEntropy(context_histo + (i << 5), 32, &sink);
}
entropy[0] = 1.0 / (double)total;
entropy[1] *= entropy[0];
@ -817,7 +817,7 @@ static void CopyInputToRingBuffer(BrotliEncoderState* s,
reading new bytes from the input. However, at the last few indexes of
the ring buffer, there are not enough bytes to build full-length
substrings from. Since the hash table always contains full-length
substrings, we erase with dummy zeros here to make sure that those
substrings, we overwrite with zeros here to make sure that those
substrings will contain zeros at the end instead of uninitialized
data.

4
c/enc/encoder_dict.c
View File

@ -118,8 +118,8 @@ static uint32_t BrotliTrieAlloc(MemoryManager* m, size_t num, BrotliTrie* trie,
keep_index = (uint32_t)(*keep - trie->pool);
}
if (trie->pool_size == 0) {
/* Have a dummy node in the front. We do not want the result to be 0, it
must be at least 1, 0 represents "null pointer" */
/* Have a placeholder node in the front. We do not want the result to be 0,
it must be at least 1, 0 represents "null pointer" */
trie->pool_size = 1;
}
BROTLI_ENSURE_CAPACITY(m, BrotliTrieNode, trie->pool, trie->pool_capacity,

4
java/org/brotli/dec/BitReader.java
View File

@ -68,7 +68,7 @@ final class BitReader {
s.halfOffset = 0;
while (bytesInBuffer < CAPACITY) {
final int spaceLeft = CAPACITY - bytesInBuffer;
final int len = Utils.readInput(s.input, s.byteBuffer, bytesInBuffer, spaceLeft);
final int len = Utils.readInput(s, s.byteBuffer, bytesInBuffer, spaceLeft);
// EOF is -1 in Java, but 0 in C#.
if (len <= 0) {
s.endOfStreamReached = 1;
@ -255,7 +255,7 @@ final class BitReader {
// Now it is possible to copy bytes directly.
while (length > 0) {
final int len = Utils.readInput(s.input, data, offset, length);
final int len = Utils.readInput(s, data, offset, length);
if (len == -1) {
throw new BrotliRuntimeException("Unexpected end of input");
}

6
java/org/brotli/dec/BitReaderTest.java
View File

@ -22,7 +22,8 @@ public class BitReaderTest {
@Test
public void testReadAfterEos() {
State reader = new State();
Decode.initState(reader, new ByteArrayInputStream(new byte[1]));
reader.input = new ByteArrayInputStream(new byte[1]);
Decode.initState(reader);
BitReader.readBits(reader, 9);
try {
BitReader.checkHealth(reader, 0);
@ -36,7 +37,8 @@ public class BitReaderTest {
@Test
public void testAccumulatorUnderflowDetected() {
State reader = new State();
Decode.initState(reader, new ByteArrayInputStream(new byte[8]));
reader.input = new ByteArrayInputStream(new byte[8]);
Decode.initState(reader);
// 65 bits is enough for both 32 and 64 bit systems.
BitReader.readBits(reader, 13);
BitReader.readBits(reader, 13);

3
java/org/brotli/dec/BrotliInputStream.java
View File

@ -86,7 +86,8 @@ public class BrotliInputStream extends InputStream {
this.remainingBufferBytes = 0;
this.bufferOffset = 0;
try {
Decode.initState(state, source);
state.input = source;
Decode.initState(state);
} catch (BrotliRuntimeException ex) {
throw new IOException("Brotli decoder initialization failed", ex);
}

9
java/org/brotli/dec/Decode.java
View File

@ -7,7 +7,6 @@
package org.brotli.dec;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
/**
@ -290,7 +289,7 @@ final class Decode {
* @param s uninitialized state without associated input
* @param input compressed data source
*/
static void initState(State s, InputStream input) {
static void initState(State s) {
if (s.runningState != UNINITIALIZED) {
throw new IllegalStateException("State MUST be uninitialized");
}
@ -302,7 +301,6 @@ final class Decode {
calculateDistanceAlphabetLimit(MAX_ALLOWED_DISTANCE, 3, 15 << 3);
s.distExtraBits = new byte[maxDistanceAlphabetLimit];
s.distOffset = new int[maxDistanceAlphabetLimit];
s.input = input;
BitReader.initBitReader(s);
s.runningState = INITIALIZED;
}
@ -315,10 +313,7 @@ final class Decode {
return;
}
s.runningState = CLOSED;
if (s.input != null) {
Utils.closeInput(s.input);
s.input = null;
}
Utils.closeInput(s);
}
/**

12
java/org/brotli/dec/Utils.java
View File

@ -7,7 +7,6 @@
package org.brotli.dec;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.nio.Buffer;
import java.nio.ByteBuffer;
@ -67,16 +66,19 @@ final class Utils {
System.arraycopy(bytes, start, bytes, target, end - start);
}
static int readInput(InputStream src, byte[] dst, int offset, int length) {
static int readInput(State s, byte[] dst, int offset, int length) {
try {
return src.read(dst, offset, length);
return s.input.read(dst, offset, length);
} catch (IOException e) {
throw new BrotliRuntimeException("Failed to read input", e);
}
}
static void closeInput(InputStream src) throws IOException {
src.close();
static void closeInput(State s) throws IOException {
if (s.input != null) {
s.input.close();
s.input = null;
}
}
static byte[] toUsAsciiBytes(String src) {

20
js/decode.js
View File

@ -193,10 +193,9 @@ let makeBrotliDecode = () => {
}
/**
* @param {!State} s
* @param {!InputStream} input
* @return {void}
*/
function initState(s, input) {
function initState(s) {
if (s.runningState !== 0) {
throw new Error("State MUST be uninitialized");
}
@ -206,7 +205,6 @@ let makeBrotliDecode = () => {
const /** @type {number} */ maxDistanceAlphabetLimit = calculateDistanceAlphabetLimit(0x7FFFFFFC, 3, 120);
s.distExtraBits = new Int8Array(maxDistanceAlphabetLimit);
s.distOffset = new Int32Array(maxDistanceAlphabetLimit);
s.input = input;
initBitReader(s);
s.runningState = 1;
}
@ -222,10 +220,8 @@ let makeBrotliDecode = () => {
return;
}
s.runningState = 11;
if (s.input !== null) {
s.input = null;
}
}
/**
* @param {!State} s
* @return {number}
@ -1666,7 +1662,7 @@ let makeBrotliDecode = () => {
s.halfOffset = 0;
while (bytesInBuffer < 4096) {
const /** @type {number} */ spaceLeft = 4096 - bytesInBuffer;
const /** @type {number} */ len = readInput(s.input, s.byteBuffer, bytesInBuffer, spaceLeft);
const /** @type {number} */ len = readInput(s, s.byteBuffer, bytesInBuffer, spaceLeft);
if (len <= 0) {
s.endOfStreamReached = 1;
s.tailBytes = bytesInBuffer;
@ -1820,7 +1816,7 @@ let makeBrotliDecode = () => {
return;
}
while (length > 0) {
const /** @type {number} */ len = readInput(s.input, data, offset, length);
const /** @type {number} */ len = readInput(s, data, offset, length);
if (len === -1) {
throw new Error("Unexpected end of input");
}
@ -2151,16 +2147,17 @@ let makeBrotliDecode = () => {
}
/**
* @param {?InputStream} src
* @param {!State} s
* @param {!Int8Array} dst
* @param {number} offset
* @param {number} length
* @return {number}
*/
function readInput(src, dst, offset, length) {
if (src === null) {
function readInput(s, dst, offset, length) {
if (s.input === null) {
return -1;
}
const /** @type {!InputStream} */ src = s.input;
let /** @type {number} */ end = Math.min(src.offset + length, src.data.length);
let /** @type {number} */ bytesRead = end - src.offset;
dst.set(src.data.subarray(src.offset, end), offset);
@ -2192,7 +2189,8 @@ let makeBrotliDecode = () => {
*/
function decode(bytes, options) {
let /** @type {!State} */ s = new State();
initState(s, new InputStream(bytes));
s.input = new InputStream(bytes);
initState(s);
if (options) {
let customDictionary =
/** @type {?Int8Array} */ (options["customDictionary"]);

2
js/decode.min.js vendored
View File

File diff suppressed because one or more lines are too long

17
js/decode.ts
View File

@ -136,7 +136,7 @@ function attachDictionaryChunk(s: State, data: Int8Array): void {
s.cdTotalSize += data.length;
s.cdChunkOffsets[s.cdNumChunks] = s.cdTotalSize;
}
function initState(s: State, input: InputStream): void {
function initState(s: State): void {
if (s.runningState !== 0) {
throw new Error("State MUST be uninitialized");
}
@ -146,7 +146,6 @@ function initState(s: State, input: InputStream): void {
const maxDistanceAlphabetLimit: number = calculateDistanceAlphabetLimit(0x7FFFFFFC, 3, 120);
s.distExtraBits = new Int8Array(maxDistanceAlphabetLimit);
s.distOffset = new Int32Array(maxDistanceAlphabetLimit);
s.input = input;
initBitReader(s);
s.runningState = 1;
}
@ -158,10 +157,8 @@ function close(s: State): void {
return;
}
s.runningState = 11;
if (s.input !== null) {
s.input = null;
}
}
function decodeVarLenUnsignedByte(s: State): number {
if (s.bitOffset >= 16) {
s.accumulator32 = (s.shortBuffer[s.halfOffset++] << 16) | (s.accumulator32 >>> 16);
@ -1398,7 +1395,7 @@ function doReadMoreInput(s: State): void {
s.halfOffset = 0;
while (bytesInBuffer < 4096) {
const spaceLeft: number = 4096 - bytesInBuffer;
const len: number = readInput(s.input, s.byteBuffer, bytesInBuffer, spaceLeft);
const len: number = readInput(s, s.byteBuffer, bytesInBuffer, spaceLeft);
if (len <= 0) {
s.endOfStreamReached = 1;
s.tailBytes = bytesInBuffer;
@ -1510,7 +1507,7 @@ function copyRawBytes(s: State, data: Int8Array, offset: number, length: number)
return;
}
while (length > 0) {
const len: number = readInput(s.input, data, offset, length);
const len: number = readInput(s, data, offset, length);
if (len === -1) {
throw new Error("Unexpected end of input");
}
@ -1728,10 +1725,11 @@ function valueOf(x: number): string {
return x.toString();
}
function readInput(src: InputStream|null, dst: Int8Array, offset: number, length: number): number {
if (src === null) {
function readInput(s: State, dst: Int8Array, offset: number, length: number): number {
if (s.input === null) {
return -1;
}
const src: InputStream = s.input;
const end: number = Math.min(src.offset + length, src.data.length);
const bytesRead: number = end - src.offset;
dst.set(src.data.subarray(src.offset, end), offset);
@ -1757,7 +1755,8 @@ type ByteBuffer = Int8Array;
export function brotliDecode(
bytes: Int8Array, options?: BrotliDecodeOptions): Int8Array {
const s = new State();
initState(s, new InputStream(bytes));
s.input = new InputStream(bytes);
initState(s);
if (options) {
const customDictionary: Int8Array|null = options.customDictionary;
if (customDictionary) attachDictionaryChunk(s, customDictionary);

2
research/brotlidump.py
View File

@ -2240,7 +2240,7 @@ __test__ = {
'file': """
>>> try: Layout(BitStream(
... open("H:/Downloads/brotli-master/tests/testdata/10x10y.compressed",'rb')
... open("./tests/testdata/10x10y.compressed",'rb')
... .read())).processStream()
... except NotImplementedError: pass
addr hex binary context explanation