Merge branch 'google:master' into master

2024-11-25 04:50:05 +00:00 · 2024-11-21 12:26:45 +05:00 · 2024-11-21 12:26:45 +05:00 · 3a444a6f2d
commit 3a444a6f2d
parent 1c380db165 39904bdfe8
14 changed files with 58 additions and 39 deletions
--- a/.github/workflows/build_test.yml
+++ b/.github/workflows/build_test.yml
@ -266,7 +266,7 @@ jobs:
      if: ${{ matrix.build_system == 'bazel' }}
      run: |
        cd ${GITHUB_WORKSPACE}/${{ matrix.bazel_project }}
-        bazelisk build -c opt ...:all
+        bazelisk build -c opt ...:all --java_runtime_version=remotejdk_21

    - name: Fix symlinks for Bazel (Windows)
      if: ${{ matrix.build_system == 'bazel' && runner.os == 'Windows' && matrix.bazel_project == 'java' }}
@ -311,7 +311,7 @@ jobs:
      run: |
        cd ${GITHUB_WORKSPACE}/${{ matrix.bazel_project }}
        bazelisk query "tests(...)" --output=label > ${RUNNER_TEMP}/tests.lst
-        [ -s ${RUNNER_TEMP}/tests.lst ] && bazelisk test -c opt ...:all
+        [ -s ${RUNNER_TEMP}/tests.lst ] && bazelisk test -c opt ...:all --java_runtime_version=remotejdk_21
        bazelisk clean

    - name: Build / Test with Maven
@ -324,7 +324,7 @@ jobs:
        # cd integration
        # mvn -B verify

-    - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
+    - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
      if: ${{ matrix.build_system == 'python' }}
      with:
        python-version: ${{ matrix.python_version }}
--- a/.github/workflows/fuzz.yml
+++ b/.github/workflows/fuzz.yml
@ -28,7 +28,7 @@ jobs:
        fuzz-seconds: 600
        dry-run: false
    - name: Upload Crash
-      uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+      uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
      if: failure()
      with:
        name: artifacts
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@ -53,7 +53,7 @@ jobs:
        submodules: false
        fetch-depth: 1

-    - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
+    - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
      id: cache-vcpkg
      with:
        path: vcpkg
@ -102,7 +102,7 @@ jobs:
        cmake --build out --config Release --target install
        cp LICENSE prefix/bin/LICENSE.brotli
    - name: Upload artifacts
-      uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+      uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
      with:
        name: brotli-${{matrix.triplet}}
        path: |
--- a/.github/workflows/scorecard.yml
+++ b/.github/workflows/scorecard.yml
@ -41,7 +41,7 @@ jobs:
          persist-credentials: false

      - name: "Run analysis"
-        uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1
+        uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0
        with:
          results_file: results.sarif
          results_format: sarif
@ -63,7 +63,7 @@ jobs:
      # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
      # format to the repository Actions tab.
      - name: "Upload artifact"
-        uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
        with:
          name: SARIF file
          path: results.sarif
--- a/c/dec/decode.c
+++ b/c/dec/decode.c
@ -217,7 +217,7 @@ static BROTLI_NOINLINE BrotliDecoderErrorCode DecodeVarLenUint8(
        s->substate_decode_uint8 = BROTLI_STATE_DECODE_UINT8_LONG;
        return BROTLI_DECODER_NEEDS_MORE_INPUT;
      }
-      *value = (1U << *value) + bits;
+      *value = ((brotli_reg_t)1U << *value) + bits;
      s->substate_decode_uint8 = BROTLI_STATE_DECODE_UINT8_NONE;
      return BROTLI_DECODER_SUCCESS;

@ -1136,7 +1136,7 @@ static BrotliDecoderErrorCode DecodeContextMap(brotli_reg_t context_map_size,
            h->context_index = context_index;
            return BROTLI_DECODER_NEEDS_MORE_INPUT;
          }
-          reps += 1U << code;
+          reps += (brotli_reg_t)1U << code;
          BROTLI_LOG_UINT(reps);
          if (context_index + reps > context_map_size) {
            return
@ -1800,7 +1800,7 @@ static void CalculateDistanceLut(BrotliDecoderState* s) {
  brotli_reg_t npostfix = s->distance_postfix_bits;
  brotli_reg_t ndirect = s->num_direct_distance_codes;
  brotli_reg_t alphabet_size_limit = s->distance_hgroup.alphabet_size_limit;
-  brotli_reg_t postfix = 1u << npostfix;
+  brotli_reg_t postfix = (brotli_reg_t)1u << npostfix;
  brotli_reg_t j;
  brotli_reg_t bits = 1;
  brotli_reg_t half = 0;
--- a/c/enc/block_splitter_inc.h
+++ b/c/enc/block_splitter_inc.h
@ -118,6 +118,8 @@ static size_t FN(FindBlocks)(const DataType* data, const size_t length,
    size_t insert_cost_ix = symbol * num_histograms;
    double min_cost = 1e99;
    double block_switch_cost = block_switch_bitcost;
+    static const size_t prologue_length = 2000;
+    static const double multiplier = 0.07 / 2000;
    size_t k;
    for (k = 0; k < num_histograms; ++k) {
      /* We are coding the symbol with entropy code k. */
@ -128,8 +130,8 @@ static size_t FN(FindBlocks)(const DataType* data, const size_t length,
      }
    }
    /* More blocks for the beginning. */
-    if (byte_ix < 2000) {
-      block_switch_cost *= 0.77 + 0.07 * (double)byte_ix / 2000;
+    if (byte_ix < prologue_length) {
+      block_switch_cost *= 0.77 + multiplier * (double)byte_ix;
    }
    for (k = 0; k < num_histograms; ++k) {
      cost[k] -= min_cost;
--- a/c/enc/compound_dictionary.c
+++ b/c/enc/compound_dictionary.c
@ -191,8 +191,8 @@ BROTLI_BOOL AttachPreparedDictionary(
  compound->chunk_offsets[index + 1] = compound->total_size;
  {
    uint32_t* slot_offsets = (uint32_t*)(&dictionary[1]);
-    uint16_t* heads = (uint16_t*)(&slot_offsets[1u << dictionary->slot_bits]);
-    uint32_t* items = (uint32_t*)(&heads[1u << dictionary->bucket_bits]);
+    uint16_t* heads = (uint16_t*)(&slot_offsets[(size_t)1u << dictionary->slot_bits]);
+    uint32_t* items = (uint32_t*)(&heads[(size_t)1u << dictionary->bucket_bits]);
    const void* tail = (void*)&items[dictionary->num_items];
    if (dictionary->magic == kPreparedDictionaryMagic) {
      compound->chunk_source[index] = (const uint8_t*)tail;
--- a/c/enc/encode.c
+++ b/c/enc/encode.c
@ -437,9 +437,10 @@ static BROTLI_BOOL ShouldCompress(
    if ((double)num_literals > 0.99 * (double)bytes) {
      uint32_t literal_histo[256] = { 0 };
      static const uint32_t kSampleRate = 13;
+      static const double kInvSampleRate = 1.0 / 13.0;
      static const double kMinEntropy = 7.92;
      const double bit_cost_threshold =
-          (double)bytes * kMinEntropy / kSampleRate;
+          (double)bytes * kMinEntropy * kInvSampleRate;
      size_t t = (bytes + kSampleRate - 1) / kSampleRate;
      uint32_t pos = (uint32_t)last_flush_pos;
      size_t i;
@ -1856,7 +1857,7 @@ size_t BrotliEncoderEstimatePeakMemoryUsage(int quality, int lgwin,
  if (params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY ||
      params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY) {
    size_t state_size = sizeof(BrotliEncoderState);
-    size_t block_size = BROTLI_MIN(size_t, input_size, (1ul << params.lgwin));
+    size_t block_size = BROTLI_MIN(size_t, input_size, ((size_t)1ul << params.lgwin));
    size_t hash_table_size =
        HashTableSize(MaxHashTableSize(params.quality), block_size);
    size_t hash_size =
@ -1873,7 +1874,7 @@ size_t BrotliEncoderEstimatePeakMemoryUsage(int quality, int lgwin,
    size_t short_ringbuffer_size = (size_t)1 << params.lgblock;
    int ringbuffer_bits = ComputeRbBits(&params);
    size_t ringbuffer_size = input_size < short_ringbuffer_size ?
-        input_size : (1u << ringbuffer_bits) + short_ringbuffer_size;
+        input_size : ((size_t)1u << ringbuffer_bits) + short_ringbuffer_size;
    size_t hash_size[4] = {0};
    size_t metablock_size =
        BROTLI_MIN(size_t, input_size, MaxMetablockSize(&params));
--- a/c/enc/hash.h
+++ b/c/enc/hash.h
@ -540,8 +540,8 @@ static BROTLI_INLINE void FindCompoundDictionaryMatch(
  const uint64_t hash_mask = (~((uint64_t)0U)) >> (64 - hash_bits);

  const uint32_t* slot_offsets = (uint32_t*)(&self[1]);
-  const uint16_t* heads = (uint16_t*)(&slot_offsets[1u << slot_bits]);
-  const uint32_t* items = (uint32_t*)(&heads[1u << bucket_bits]);
+  const uint16_t* heads = (uint16_t*)(&slot_offsets[(size_t)1u << slot_bits]);
+  const uint32_t* items = (uint32_t*)(&heads[(size_t)1u << bucket_bits]);
  const uint8_t* source = NULL;

  const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
@ -652,8 +652,8 @@ static BROTLI_INLINE size_t FindAllCompoundDictionaryMatches(
  const uint64_t hash_mask = (~((uint64_t)0U)) >> (64 - hash_bits);

  const uint32_t* slot_offsets = (uint32_t*)(&self[1]);
-  const uint16_t* heads = (uint16_t*)(&slot_offsets[1u << slot_bits]);
-  const uint32_t* items = (uint32_t*)(&heads[1u << bucket_bits]);
+  const uint16_t* heads = (uint16_t*)(&slot_offsets[(size_t)1u << slot_bits]);
+  const uint32_t* items = (uint32_t*)(&heads[(size_t)1u << bucket_bits]);
  const uint8_t* source = NULL;

  const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
--- a/c/enc/literal_cost.c
+++ b/c/enc/literal_cost.c
@ -106,6 +106,8 @@ static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
      size_t utf8_pos = UTF8Position(last_c, c, max_utf8);
      size_t masked_pos = (pos + i) & mask;
      size_t histo = histogram[256 * utf8_pos + data[masked_pos]];
+      static const size_t prologue_length = 2000;
+      static const double multiplier = 0.35 / 2000;
      double lit_cost;
      if (histo == 0) {
        histo = 1;
@ -120,8 +122,8 @@ static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask,
         Perhaps because the entropy source is changing its properties
         rapidly in the beginning of the file, perhaps because the beginning
         of the data is a statistical "anomaly". */
-      if (i < 2000) {
-        lit_cost += 0.7 - ((double)(2000 - i) / 2000.0 * 0.35);
+      if (i < prologue_length) {
+        lit_cost += 0.35 + multiplier * (double)i;
      }
      cost[i] = (float)lit_cost;
    }
--- a/go/cbrotli/cgo.go
+++ b/go/cbrotli/cgo.go
@ -7,7 +7,5 @@ package cbrotli

 // Inform golang build system that it should link brotli libraries.

-// #cgo LDFLAGS: -lbrotlicommon
-// #cgo LDFLAGS: -lbrotlidec
-// #cgo LDFLAGS: -lbrotlienc
+// #cgo pkg-config: libbrotlicommon libbrotlidec libbrotlienc
 import "C"
--- a/java/WORKSPACE.bazel
+++ b/java/WORKSPACE.bazel
@ -7,15 +7,29 @@ local_repository(

 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")

+RULES_JAVA_VERSION = "8.3.2"
+RULES_JAVA_SHA = "9b9614f8a7f7b7ed93cb7975d227ece30fe7daed2c0a76f03a5ee37f69e437de"
 RULES_JVM_EXTERNAL_TAG = "4.0"
 RULES_JVM_EXTERNAL_SHA = "31701ad93dbfe544d597dbe62c9a1fdd76d81d8a9150c2bf1ecf928ecdf97169"
 RULES_KOTLIN_VERSION = "1.9.0"
 RULES_KOTLIN_SHA = "5766f1e599acf551aa56f49dab9ab9108269b03c557496c54acaf41f98e2b8d6"

+http_archive(
+    name = "rules_java",
+    sha256 = RULES_JAVA_SHA,
+    urls = [
+        "https://github.com/bazelbuild/rules_java/releases/download/%s/rules_java-%s.tar.gz" % (RULES_JAVA_VERSION, RULES_JAVA_VERSION),
+    ],
+)
+
+load("@rules_java//java:repositories.bzl", "rules_java_dependencies", "rules_java_toolchains")
+rules_java_dependencies()
+rules_java_toolchains()
+
 http_archive(
    name = "rules_jvm_external",
-    strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
    sha256 = RULES_JVM_EXTERNAL_SHA,
+    strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
    url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG,
 )

@ -23,15 +37,15 @@ load("@rules_jvm_external//:defs.bzl", "maven_install")

 http_archive(
    name = "rules_kotlin",
-    urls = ["https://github.com/bazelbuild/rules_kotlin/releases/download/v%s/rules_kotlin-v%s.tar.gz" % (RULES_KOTLIN_VERSION, RULES_KOTLIN_VERSION)],
    sha256 = RULES_KOTLIN_SHA,
+    urls = ["https://github.com/bazelbuild/rules_kotlin/releases/download/v%s/rules_kotlin-v%s.tar.gz" % (RULES_KOTLIN_VERSION, RULES_KOTLIN_VERSION)],
 )

 load("@rules_kotlin//kotlin:repositories.bzl", "kotlin_repositories")
-kotlin_repositories() # if you want the default. Otherwise see custom kotlinc distribution below

+kotlin_repositories()  # if you want the default. Otherwise see custom kotlinc distribution below
 load("@rules_kotlin//kotlin:core.bzl", "kt_register_toolchains")
-kt_register_toolchains() # to use the default toolchain, otherwise see toolchains below
+kt_register_toolchains()  # to use the default toolchain, otherwise see toolchains below

 maven_install(
    artifacts = ["junit:junit:4.12"],
@ -43,37 +57,37 @@ maven_install(

 http_archive(
    name = "platforms",
+    sha256 = "8150406605389ececb6da07cbcb509d5637a3ab9a24bc69b1101531367d89d74",
    urls = [
        "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.8/platforms-0.0.8.tar.gz",
        "https://github.com/bazelbuild/platforms/releases/download/0.0.8/platforms-0.0.8.tar.gz",
    ],
-    sha256 = "8150406605389ececb6da07cbcb509d5637a3ab9a24bc69b1101531367d89d74",
 )

 http_file(
    name = "openjdk_jni_h",
    downloaded_file_path = "jni.h",
-    urls = ["https://raw.githubusercontent.com/openjdk/jdk/jdk8-b120/jdk/src/share/javavm/export/jni.h"],
    sha256 = "ed99792df48670072b78028faf704a8dcb6868fe140ccc7eced9b01dfa62fef4",
+    urls = ["https://raw.githubusercontent.com/openjdk/jdk/jdk8-b120/jdk/src/share/javavm/export/jni.h"],
 )

 http_file(
    name = "openjdk_solaris_jni_md_h",
    downloaded_file_path = "jni_md.h",
-    urls = ["https://raw.githubusercontent.com/openjdk/jdk/jdk8-b120/jdk/src/solaris/javavm/export/jni_md.h"],
    sha256 = "b6cf7b06e5bba38d2daa2ff0789f99d396b3cb3bcc37d0367c8360fdccdef294",
+    urls = ["https://raw.githubusercontent.com/openjdk/jdk/jdk8-b120/jdk/src/solaris/javavm/export/jni_md.h"],
 )

 http_file(
    name = "openjdk_macosx_jni_md_h",
    downloaded_file_path = "jni_md.h",
-    urls = ["https://raw.githubusercontent.com/openjdk/jdk/jdk8-b120/jdk/src/macosx/javavm/export/jni_md.h"],
    sha256 = "8f718071022e7e7f2fc9a229984b7e83582db91ed83861b49ce1461436fe8dc4",
+    urls = ["https://raw.githubusercontent.com/openjdk/jdk/jdk8-b120/jdk/src/macosx/javavm/export/jni_md.h"],
 )

 http_file(
    name = "openjdk_windows_jni_md_h",
    downloaded_file_path = "jni_md.h",
-    urls = ["https://raw.githubusercontent.com/openjdk/jdk/jdk8-b120/jdk/src/windows/javavm/export/jni_md.h"],
    sha256 = "5479fb385ea1e11619f5c0cdfd9ccb3ea3a3fea0f5bc6176fb3ce62be29d759b",
+    urls = ["https://raw.githubusercontent.com/openjdk/jdk/jdk8-b120/jdk/src/windows/javavm/export/jni_md.h"],
 )
--- a/java/org/brotli/dec/build_defs.bzl
+++ b/java/org/brotli/dec/build_defs.bzl
@ -1,6 +1,6 @@
 """Utilities for Java brotli tests."""

-load("//third_party/bazel_rules/rules_java/java:java_test.bzl", "java_test")
+load("@rules_java//java:java_test.bzl", "java_test")

 _TEST_JVM_FLAGS = [
    "-DBROTLI_ENABLE_ASSERTS=true",
--- a/python/brotli.py
+++ b/python/brotli.py
@ -27,8 +27,10 @@ def compress(string, mode=MODE_GENERIC, quality=11, lgwin=22, lgblock=0):

    Args:
      string (bytes): The input data.
-      mode (int, optional): The compression mode can be MODE_GENERIC (default),
-        MODE_TEXT (for UTF-8 format text input) or MODE_FONT (for WOFF 2.0).
+      mode (int, optional): The compression mode; value 0 should be used for
+        generic input (MODE_GENERIC); value 1 might be beneficial for UTF-8 text
+        input (MODE_TEXT); value 2 tunes encoder for WOFF 2.0 data (MODE_FONT).
+        Defaults to 0.
      quality (int, optional): Controls the compression-speed vs compression-
        density tradeoff. The higher the quality, the slower the compression.
        Range is 0 to 11. Defaults to 11.