Merge branch 'master' into macos-relocatable

.github/workflows/build_test.yml

@@ -43,28 +43,28 @@ jobs:
           # TODO: consider running this combination in docker
           #- name: cmake:clang3.5
 
-          - name: cmake:clang12
+          - name: cmake:clang15
             build_system: cmake
-            c_compiler: clang-12
-            cxx_compiler: clang++12
+            c_compiler: clang-15
+            cxx_compiler: clang++15
 
-          - name: cmake:clang12:asan
+          - name: cmake:clang15:asan
             build_system: cmake
             sanitizer: address
-            c_compiler: clang-12
-            cxx_compiler: clang++12
+            c_compiler: clang-15
+            cxx_compiler: clang++15
 
-          - name: cmake:clang12:tsan
+          - name: cmake:clang15:tsan
             build_system: cmake
             sanitizer: thread
-            c_compiler: clang-12
-            cxx_compiler: clang++12
+            c_compiler: clang-15
+            cxx_compiler: clang++15
 
-          - name: cmake:clang12:ubsan
+          - name: cmake:clang15:ubsan
             build_system: cmake
             sanitizer: undefined
-            c_compiler: clang-12
-            cxx_compiler: clang++-12
+            c_compiler: clang-15
+            cxx_compiler: clang++-15
             c_flags: -fno-sanitize-recover=undefined,integer
 
           - name: cmake:qemu-arm-neon-gcc
@@ -98,10 +98,10 @@ jobs:
             cmake_config: Debug
             os: windows-2019
 
-          - name: fuzz:clang12
+          - name: fuzz:clang15
             build_system: fuzz
-            c_compiler: clang-12
-            cxx_compiler: clang++12
+            c_compiler: clang-15
+            cxx_compiler: clang++15
 
           # TODO: consider running this combination in docker
           #- name: python2.7:gcc5
@@ -131,15 +131,15 @@ jobs:
           # TODO: consider running this combination in docker
           #- name: python3.8:gcc5
 
-          - name: python39:clang12
+          - name: python3.10:clang15
             build_system: python
-            python_version: 3.9
-            c_compiler: clang-12
-            cxx_compiler: clang++-12
+            python_version: "3.10"
+            c_compiler: clang-15
+            cxx_compiler: clang++-15
 
-          - name: python39-win
+          - name: python3.10-win
             build_system: python
-            python_version: 3.9
+            python_version: "3.10"
             # TODO: investigate why win-builds can't run tests
             py_setuptools_cmd: build_ext
             os: windows-2019
@@ -195,7 +195,7 @@ jobs:
           - name: bazel-win:root
             build_system: bazel
             bazel_project: .
-            os: windows-2019
+            os: windows-latest
 
           # TODO: use single dll on windows, otherwise it fails to link
           #- name: bazel-win:go
@@ -206,7 +206,7 @@ jobs:
           - name: bazel-win:java
             build_system: bazel
             bazel_project: java
-            os: windows-2019
+            os: windows-latest
 
           # TODO: blocked by Bazel Closure rules issue
           #- name: bazel-win:js
@@ -217,7 +217,7 @@ jobs:
           - name: bazel-win:research
             build_system: bazel
             bazel_project: research
-            os: windows-2019
+            os: windows-latest
 
     env:
       CC: ${{ matrix.c_compiler || 'gcc' }}
@@ -226,7 +226,7 @@ jobs:
     steps:
     - name: Install extra deps @ Ubuntu
       if: ${{ runner.os == 'Linux' }}
-      # Already installed: bazel, clang{10-12}, cmake, gcc{9,10}, java{8,11}, maven, python{3.5-3.9}
+      # Already installed: bazel, clang{13-15}, cmake, gcc{9.5-13.1}, java{8,11,17,21}, maven, python{3.10}
       run: |
         EXTRA_PACKAGES="${{ matrix.extra_apt_pkgs || '' }}"
         sudo apt update
@@ -338,7 +338,7 @@ jobs:
         cd integration
         mvn -B verify
 
-    - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
+    - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
       if: ${{ matrix.build_system == 'python' }}
       with:
         python-version: ${{ matrix.python_version }}

.github/workflows/release.yaml

@@ -23,7 +23,7 @@ concurrency:
 jobs:
   windows_build:
     name: Windows Build (vcpkg / ${{ matrix.triplet }})
-    runs-on: [windows-2022]
+    runs-on: [windows-latest]
     strategy:
       fail-fast: false
       matrix:

.github/workflows/scorecard.yml

@@ -41,7 +41,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@08b4669551908b1024bb425080c797723083c031 # v2.2.0
+        uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1
         with:
           results_file: results.sarif
           results_format: sarif

CMakeLists.txt

@@ -11,10 +11,11 @@ cmake_policy(SET CMP0048 NEW)
 project(brotli C)
 
 option(BUILD_SHARED_LIBS "Build shared libraries" ON)
+set(BROTLI_BUILD_TOOLS ON CACHE BOOL "Build/install CLI tools")
 
 if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
   message(STATUS "Setting build type to Release as none was specified.")
-  set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build." FORCE)
+  set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build" FORCE)
 else()
   message(STATUS "Build type is '${CMAKE_BUILD_TYPE}'")
 endif()
@@ -81,7 +82,7 @@ include(CheckFunctionExists)
 set(LIBM_LIBRARY)
 CHECK_FUNCTION_EXISTS(log2 LOG2_RES)
 if(NOT LOG2_RES)
-  set(orig_req_libs "${CMAKE_REQUIRED_LIBRARIES}")
+  set(_ORIG_REQ_LIBS "${CMAKE_REQUIRED_LIBRARIES}")
   set(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES};m")
   CHECK_FUNCTION_EXISTS(log2 LOG2_LIBM_RES)
   if(LOG2_LIBM_RES)
@@ -91,9 +92,9 @@ if(NOT LOG2_RES)
     add_definitions(-DBROTLI_HAVE_LOG2=0)
   endif()
 
-  set(CMAKE_REQUIRED_LIBRARIES "${orig_req_libs}")
+  set(CMAKE_REQUIRED_LIBRARIES "${_ORIG_REQ_LIBS}")
   unset(LOG2_LIBM_RES)
-  unset(orig_req_libs)
+  unset(_ORIG_REQ_LIBS)
 else()
   add_definitions(-DBROTLI_HAVE_LOG2=1)
 endif()
@@ -151,8 +152,8 @@ foreach(lib ${BROTLI_LIBRARIES_CORE})
 endforeach()
 
 if(NOT BROTLI_EMSCRIPTEN)
-target_link_libraries(brotlidec brotlicommon)
-target_link_libraries(brotlienc brotlicommon)
+  target_link_libraries(brotlidec brotlicommon)
+  target_link_libraries(brotlienc brotlicommon)
 endif()
 
 # For projects stuck on older versions of CMake, this will set the
@@ -167,15 +168,19 @@ if(BROTLI_PARENT_DIRECTORY)
 endif()
 
 # Build the brotli executable
-add_executable(brotli c/tools/brotli.c)
-target_link_libraries(brotli ${BROTLI_LIBRARIES})
+if(BROTLI_BUILD_TOOLS)
+  add_executable(brotli c/tools/brotli.c)
+  target_link_libraries(brotli ${BROTLI_LIBRARIES})
+endif()
 
 # Installation
 if(NOT BROTLI_BUNDLED_MODE)
-  install(
-    TARGETS brotli
-    RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
-  )
+  if (BROTLI_BUILD_TOOLS)
+    install(
+      TARGETS brotli
+      RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
+    )
+  endif()
 
   install(
     TARGETS ${BROTLI_LIBRARIES_CORE}
@@ -192,9 +197,10 @@ endif()  # BROTLI_BUNDLED_MODE
 
 # Tests
 
-# If we're targeting Windows but not running on Windows, we need Wine
-# to run the tests...
-if(NOT BROTLI_DISABLE_TESTS)
+# Integration tests, those depend on `brotli` binary
+if(NOT BROTLI_DISABLE_TESTS AND BROTLI_BUILD_TOOLS)
+  # If we're targeting Windows but not running on Windows, we need Wine
+  # to run the tests...
   if(WIN32 AND NOT CMAKE_HOST_WIN32)
     find_program(BROTLI_WRAPPER NAMES wine)
 
@@ -203,11 +209,8 @@ if(NOT BROTLI_DISABLE_TESTS)
       set(BROTLI_DISABLE_TESTS TRUE)
     endif()
   endif()
-endif()
-
-# If our compiler is a cross-compiler that we know about (arm/aarch64),
-# then we need to use qemu to execute the tests.
-if(NOT BROTLI_DISABLE_TESTS)
+  # If our compiler is a cross-compiler that we know about (arm/aarch64),
+  # then we need to use qemu to execute the tests.
   if ("${CMAKE_C_COMPILER}" MATCHES "^.*/arm-linux-gnueabihf-.*$")
     message(STATUS "Detected arm-linux-gnueabihf cross-compilation")
     set(BROTLI_WRAPPER "qemu-arm")
@@ -225,9 +228,7 @@ if(NOT BROTLI_DISABLE_TESTS)
     set(BROTLI_WRAPPER "qemu-aarch64")
     set(BROTLI_WRAPPER_LD_PREFIX "/usr/aarch64-linux-gnu")
   endif()
-endif()
 
-if(NOT BROTLI_DISABLE_TESTS)
   include(CTest)
   enable_testing()
 
@@ -277,7 +278,7 @@ if(NOT BROTLI_DISABLE_TESTS)
         -DINPUT=${CMAKE_CURRENT_SOURCE_DIR}/${INPUT}
         -P ${CMAKE_CURRENT_SOURCE_DIR}/tests/run-compatibility-test.cmake)
   endforeach()
-endif()
+endif()  # BROTLI_DISABLE_TESTS
 
 # Generate a pkg-config files
 
@@ -357,10 +358,14 @@ if(NOT BROTLI_BUNDLED_MODE)
     DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
 endif()  # BROTLI_BUNDLED_MODE
 
-INSTALL(FILES "docs/brotli.1" DESTINATION "${SHARE_INSTALL_PREFIX}/man/man1")
-INSTALL(FILES docs/constants.h.3 docs/decode.h.3 docs/encode.h.3 docs/types.h.3
-  DESTINATION "${SHARE_INSTALL_PREFIX}/man/man3")
+if (BROTLI_BUILD_TOOLS)
+  install(FILES "docs/brotli.1"
+    DESTINATION "${CMAKE_INSTALL_FULL_MANDIR}/man1")
+endif()
+
+install(FILES docs/constants.h.3 docs/decode.h.3 docs/encode.h.3 docs/types.h.3
+  DESTINATION "${CMAKE_INSTALL_FULL_MANDIR}/man3")
 
 if (ENABLE_COVERAGE STREQUAL "yes")
-  SETUP_TARGET_FOR_COVERAGE(coverage test coverage)
-endif ()
+  setup_target_for_coverage(coverage test coverage)
+endif()

java/org/brotli/dec/BitReader.java

@@ -68,7 +68,7 @@ final class BitReader {
     s.halfOffset = 0;
     while (bytesInBuffer < CAPACITY) {
       final int spaceLeft = CAPACITY - bytesInBuffer;
-      final int len = Utils.readInput(s.input, s.byteBuffer, bytesInBuffer, spaceLeft);
+      final int len = Utils.readInput(s, s.byteBuffer, bytesInBuffer, spaceLeft);
       // EOF is -1 in Java, but 0 in C#.
       if (len <= 0) {
         s.endOfStreamReached = 1;
@@ -255,7 +255,7 @@ final class BitReader {
 
     // Now it is possible to copy bytes directly.
     while (length > 0) {
-      final int len = Utils.readInput(s.input, data, offset, length);
+      final int len = Utils.readInput(s, data, offset, length);
       if (len == -1) {
         throw new BrotliRuntimeException("Unexpected end of input");
       }

java/org/brotli/dec/BitReaderTest.java

@@ -22,7 +22,8 @@ public class BitReaderTest {
   @Test
   public void testReadAfterEos() {
     State reader = new State();
-    Decode.initState(reader, new ByteArrayInputStream(new byte[1]));
+    reader.input = new ByteArrayInputStream(new byte[1]);
+    Decode.initState(reader);
     BitReader.readBits(reader, 9);
     try {
       BitReader.checkHealth(reader, 0);
@@ -36,7 +37,8 @@ public class BitReaderTest {
   @Test
   public void testAccumulatorUnderflowDetected() {
     State reader = new State();
-    Decode.initState(reader, new ByteArrayInputStream(new byte[8]));
+    reader.input = new ByteArrayInputStream(new byte[8]);
+    Decode.initState(reader);
     // 65 bits is enough for both 32 and 64 bit systems.
     BitReader.readBits(reader, 13);
     BitReader.readBits(reader, 13);

java/org/brotli/dec/BrotliInputStream.java

@@ -86,7 +86,8 @@ public class BrotliInputStream extends InputStream {
     this.remainingBufferBytes = 0;
     this.bufferOffset = 0;
     try {
-      Decode.initState(state, source);
+      state.input = source;
+      Decode.initState(state);
     } catch (BrotliRuntimeException ex) {
       throw new IOException("Brotli decoder initialization failed", ex);
     }

java/org/brotli/dec/Decode.java

@@ -7,7 +7,6 @@
 package org.brotli.dec;
 
 import java.io.IOException;
-import java.io.InputStream;
 import java.nio.ByteBuffer;
 
 /**
@@ -290,7 +289,7 @@ final class Decode {
    * @param s uninitialized state without associated input
    * @param input compressed data source
    */
-  static void initState(State s, InputStream input) {
+  static void initState(State s) {
     if (s.runningState != UNINITIALIZED) {
       throw new IllegalStateException("State MUST be uninitialized");
     }
@@ -302,7 +301,6 @@ final class Decode {
         calculateDistanceAlphabetLimit(MAX_ALLOWED_DISTANCE, 3, 15 << 3);
     s.distExtraBits = new byte[maxDistanceAlphabetLimit];
     s.distOffset = new int[maxDistanceAlphabetLimit];
-    s.input = input;
     BitReader.initBitReader(s);
     s.runningState = INITIALIZED;
   }
@@ -315,10 +313,7 @@ final class Decode {
       return;
     }
     s.runningState = CLOSED;
-    if (s.input != null) {
-      Utils.closeInput(s.input);
-      s.input = null;
-    }
+    Utils.closeInput(s);
   }
 
   /**

java/org/brotli/dec/Utils.java

@@ -7,7 +7,6 @@
 package org.brotli.dec;
 
 import java.io.IOException;
-import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
 import java.nio.Buffer;
 import java.nio.ByteBuffer;
@@ -67,16 +66,19 @@ final class Utils {
     System.arraycopy(bytes, start, bytes, target, end - start);
   }
 
-  static int readInput(InputStream src, byte[] dst, int offset, int length) {
+  static int readInput(State s, byte[] dst, int offset, int length) {
     try {
-      return src.read(dst, offset, length);
+      return s.input.read(dst, offset, length);
     } catch (IOException e) {
       throw new BrotliRuntimeException("Failed to read input", e);
     }
   }
 
-  static void closeInput(InputStream src) throws IOException {
-    src.close();
+  static void closeInput(State s) throws IOException {
+    if (s.input != null) {
+      s.input.close();
+      s.input = null;
+    }
   }
 
   static byte[] toUsAsciiBytes(String src) {

js/decode.js

@@ -193,10 +193,9 @@ let makeBrotliDecode = () => {
   }
   /**
    * @param {!State} s
-   * @param {!InputStream} input
    * @return {void}
    */
-  function initState(s, input) {
+  function initState(s) {
     if (s.runningState !== 0) {
       throw new Error("State MUST be uninitialized");
     }
@@ -206,7 +205,6 @@ let makeBrotliDecode = () => {
     const /** @type {number} */ maxDistanceAlphabetLimit = calculateDistanceAlphabetLimit(0x7FFFFFFC, 3, 120);
     s.distExtraBits = new Int8Array(maxDistanceAlphabetLimit);
     s.distOffset = new Int32Array(maxDistanceAlphabetLimit);
-    s.input = input;
     initBitReader(s);
     s.runningState = 1;
   }
@@ -222,9 +220,7 @@ let makeBrotliDecode = () => {
       return;
     }
     s.runningState = 11;
-    if (s.input !== null) {
-      s.input = null;
-    }
+    s.input = null;
   }
   /**
    * @param {!State} s
@@ -1666,7 +1662,7 @@ let makeBrotliDecode = () => {
     s.halfOffset = 0;
     while (bytesInBuffer < 4096) {
       const /** @type {number} */ spaceLeft = 4096 - bytesInBuffer;
-      const /** @type {number} */ len = readInput(s.input, s.byteBuffer, bytesInBuffer, spaceLeft);
+      const /** @type {number} */ len = readInput(s, s.byteBuffer, bytesInBuffer, spaceLeft);
       if (len <= 0) {
         s.endOfStreamReached = 1;
         s.tailBytes = bytesInBuffer;
@@ -1820,7 +1816,7 @@ let makeBrotliDecode = () => {
       return;
     }
     while (length > 0) {
-      const /** @type {number} */ len = readInput(s.input, data, offset, length);
+      const /** @type {number} */ len = readInput(s, data, offset, length);
       if (len === -1) {
         throw new Error("Unexpected end of input");
       }
@@ -2151,16 +2147,17 @@ let makeBrotliDecode = () => {
   }
 
   /**
-   * @param {?InputStream} src
+   * @param {!State} s
    * @param {!Int8Array} dst
    * @param {number} offset
    * @param {number} length
    * @return {number}
    */
-  function readInput(src, dst, offset, length) {
-    if (src === null) {
+  function readInput(s, dst, offset, length) {
+    if (s.input === null) {
       return -1;
     }
+    const /** @type {!InputStream} */ src = s.input;
     let /** @type {number} */ end = Math.min(src.offset + length, src.data.length);
     let /** @type {number} */ bytesRead = end - src.offset;
     dst.set(src.data.subarray(src.offset, end), offset);
@@ -2192,7 +2189,8 @@ let makeBrotliDecode = () => {
    */
   function decode(bytes, options) {
     let /** @type {!State} */ s = new State();
-    initState(s, new InputStream(bytes));
+    s.input = new InputStream(bytes);
+    initState(s);
     if (options) {
       let customDictionary =
           /** @type {?Int8Array} */ (options["customDictionary"]);

js/decode.ts

@@ -136,7 +136,7 @@ function attachDictionaryChunk(s: State, data: Int8Array): void {
   s.cdTotalSize += data.length;
   s.cdChunkOffsets[s.cdNumChunks] = s.cdTotalSize;
 }
-function initState(s: State, input: InputStream): void {
+function initState(s: State): void {
   if (s.runningState !== 0) {
     throw new Error("State MUST be uninitialized");
   }
@@ -146,7 +146,6 @@ function initState(s: State, input: InputStream): void {
   const maxDistanceAlphabetLimit: number = calculateDistanceAlphabetLimit(0x7FFFFFFC, 3, 120);
   s.distExtraBits = new Int8Array(maxDistanceAlphabetLimit);
   s.distOffset = new Int32Array(maxDistanceAlphabetLimit);
-  s.input = input;
   initBitReader(s);
   s.runningState = 1;
 }
@@ -158,9 +157,7 @@ function close(s: State): void {
     return;
   }
   s.runningState = 11;
-  if (s.input !== null) {
-    s.input = null;
-  }
+  s.input = null;
 }
 function decodeVarLenUnsignedByte(s: State): number {
   if (s.bitOffset >= 16) {
@@ -1398,7 +1395,7 @@ function doReadMoreInput(s: State): void {
   s.halfOffset = 0;
   while (bytesInBuffer < 4096) {
     const spaceLeft: number = 4096 - bytesInBuffer;
-    const len: number = readInput(s.input, s.byteBuffer, bytesInBuffer, spaceLeft);
+    const len: number = readInput(s, s.byteBuffer, bytesInBuffer, spaceLeft);
     if (len <= 0) {
       s.endOfStreamReached = 1;
       s.tailBytes = bytesInBuffer;
@@ -1510,7 +1507,7 @@ function copyRawBytes(s: State, data: Int8Array, offset: number, length: number)
     return;
   }
   while (length > 0) {
-    const len: number = readInput(s.input, data, offset, length);
+    const len: number = readInput(s, data, offset, length);
     if (len === -1) {
       throw new Error("Unexpected end of input");
     }
@@ -1728,10 +1725,11 @@ function valueOf(x: number): string {
   return x.toString();
 }
 
-function readInput(src: InputStream|null, dst: Int8Array, offset: number, length: number): number {
-  if (src === null) {
+function readInput(s: State, dst: Int8Array, offset: number, length: number): number {
+  if (s.input === null) {
     return -1;
   }
+  const src: InputStream = s.input;
   const end: number = Math.min(src.offset + length, src.data.length);
   const bytesRead: number = end - src.offset;
   dst.set(src.data.subarray(src.offset, end), offset);
@@ -1757,7 +1755,8 @@ type ByteBuffer = Int8Array;
 export function brotliDecode(
     bytes: Int8Array, options?: BrotliDecodeOptions): Int8Array {
   const s = new State();
-  initState(s, new InputStream(bytes));
+  s.input = new InputStream(bytes);
+  initState(s);
   if (options) {
     const customDictionary: Int8Array|null = options.customDictionary;
     if (customDictionary) attachDictionaryChunk(s, customDictionary);