diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index ec13929..166f6bb 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -266,7 +266,7 @@ jobs: if: ${{ matrix.build_system == 'bazel' }} run: | cd ${GITHUB_WORKSPACE}/${{ matrix.bazel_project }} - bazelisk build -c opt ...:all + bazelisk build -c opt ...:all --java_runtime_version=remotejdk_21 - name: Fix symlinks for Bazel (Windows) if: ${{ matrix.build_system == 'bazel' && runner.os == 'Windows' && matrix.bazel_project == 'java' }} @@ -311,7 +311,7 @@ jobs: run: | cd ${GITHUB_WORKSPACE}/${{ matrix.bazel_project }} bazelisk query "tests(...)" --output=label > ${RUNNER_TEMP}/tests.lst - [ -s ${RUNNER_TEMP}/tests.lst ] && bazelisk test -c opt ...:all + [ -s ${RUNNER_TEMP}/tests.lst ] && bazelisk test -c opt ...:all --java_runtime_version=remotejdk_21 bazelisk clean - name: Build / Test with Maven @@ -324,7 +324,7 @@ jobs: # cd integration # mvn -B verify - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 if: ${{ matrix.build_system == 'python' }} with: python-version: ${{ matrix.python_version }} diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index a1720bf..bebc560 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -28,7 +28,7 @@ jobs: fuzz-seconds: 600 dry-run: false - name: Upload Crash - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 if: failure() with: name: artifacts diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c99cc12..08b8a0e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -53,7 +53,7 @@ jobs: submodules: false fetch-depth: 1 - - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 + - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 id: cache-vcpkg with: path: vcpkg @@ -102,7 +102,7 @@ jobs: cmake --build out --config Release --target install cp LICENSE prefix/bin/LICENSE.brotli - name: Upload artifacts - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: brotli-${{matrix.triplet}} path: | diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 0c5193f..e217e0a 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -41,7 +41,7 @@ jobs: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 + uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0 with: results_file: results.sarif results_format: sarif @@ -63,7 +63,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: SARIF file path: results.sarif diff --git a/c/dec/decode.c b/c/dec/decode.c index 921aa4a..fc7d535 100644 --- a/c/dec/decode.c +++ b/c/dec/decode.c @@ -217,7 +217,7 @@ static BROTLI_NOINLINE BrotliDecoderErrorCode DecodeVarLenUint8( s->substate_decode_uint8 = BROTLI_STATE_DECODE_UINT8_LONG; return BROTLI_DECODER_NEEDS_MORE_INPUT; } - *value = (1U << *value) + bits; + *value = ((brotli_reg_t)1U << *value) + bits; s->substate_decode_uint8 = BROTLI_STATE_DECODE_UINT8_NONE; return BROTLI_DECODER_SUCCESS; @@ -1136,7 +1136,7 @@ static BrotliDecoderErrorCode DecodeContextMap(brotli_reg_t context_map_size, h->context_index = context_index; return BROTLI_DECODER_NEEDS_MORE_INPUT; } - reps += 1U << code; + reps += (brotli_reg_t)1U << code; BROTLI_LOG_UINT(reps); if (context_index + reps > context_map_size) { return @@ -1800,7 +1800,7 @@ static void CalculateDistanceLut(BrotliDecoderState* s) { brotli_reg_t npostfix = s->distance_postfix_bits; brotli_reg_t ndirect = s->num_direct_distance_codes; brotli_reg_t alphabet_size_limit = s->distance_hgroup.alphabet_size_limit; - brotli_reg_t postfix = 1u << npostfix; + brotli_reg_t postfix = (brotli_reg_t)1u << npostfix; brotli_reg_t j; brotli_reg_t bits = 1; brotli_reg_t half = 0; diff --git a/c/enc/block_splitter_inc.h b/c/enc/block_splitter_inc.h index aa40bfd..64409ec 100644 --- a/c/enc/block_splitter_inc.h +++ b/c/enc/block_splitter_inc.h @@ -118,6 +118,8 @@ static size_t FN(FindBlocks)(const DataType* data, const size_t length, size_t insert_cost_ix = symbol * num_histograms; double min_cost = 1e99; double block_switch_cost = block_switch_bitcost; + static const size_t prologue_length = 2000; + static const double multiplier = 0.07 / 2000; size_t k; for (k = 0; k < num_histograms; ++k) { /* We are coding the symbol with entropy code k. */ @@ -128,8 +130,8 @@ static size_t FN(FindBlocks)(const DataType* data, const size_t length, } } /* More blocks for the beginning. */ - if (byte_ix < 2000) { - block_switch_cost *= 0.77 + 0.07 * (double)byte_ix / 2000; + if (byte_ix < prologue_length) { + block_switch_cost *= 0.77 + multiplier * (double)byte_ix; } for (k = 0; k < num_histograms; ++k) { cost[k] -= min_cost; diff --git a/c/enc/compound_dictionary.c b/c/enc/compound_dictionary.c index a3b5e69..59ec864 100644 --- a/c/enc/compound_dictionary.c +++ b/c/enc/compound_dictionary.c @@ -191,8 +191,8 @@ BROTLI_BOOL AttachPreparedDictionary( compound->chunk_offsets[index + 1] = compound->total_size; { uint32_t* slot_offsets = (uint32_t*)(&dictionary[1]); - uint16_t* heads = (uint16_t*)(&slot_offsets[1u << dictionary->slot_bits]); - uint32_t* items = (uint32_t*)(&heads[1u << dictionary->bucket_bits]); + uint16_t* heads = (uint16_t*)(&slot_offsets[(size_t)1u << dictionary->slot_bits]); + uint32_t* items = (uint32_t*)(&heads[(size_t)1u << dictionary->bucket_bits]); const void* tail = (void*)&items[dictionary->num_items]; if (dictionary->magic == kPreparedDictionaryMagic) { compound->chunk_source[index] = (const uint8_t*)tail; diff --git a/c/enc/encode.c b/c/enc/encode.c index 96c517b..fb2c0ce 100644 --- a/c/enc/encode.c +++ b/c/enc/encode.c @@ -437,9 +437,10 @@ static BROTLI_BOOL ShouldCompress( if ((double)num_literals > 0.99 * (double)bytes) { uint32_t literal_histo[256] = { 0 }; static const uint32_t kSampleRate = 13; + static const double kInvSampleRate = 1.0 / 13.0; static const double kMinEntropy = 7.92; const double bit_cost_threshold = - (double)bytes * kMinEntropy / kSampleRate; + (double)bytes * kMinEntropy * kInvSampleRate; size_t t = (bytes + kSampleRate - 1) / kSampleRate; uint32_t pos = (uint32_t)last_flush_pos; size_t i; @@ -1856,7 +1857,7 @@ size_t BrotliEncoderEstimatePeakMemoryUsage(int quality, int lgwin, if (params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY || params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY) { size_t state_size = sizeof(BrotliEncoderState); - size_t block_size = BROTLI_MIN(size_t, input_size, (1ul << params.lgwin)); + size_t block_size = BROTLI_MIN(size_t, input_size, ((size_t)1ul << params.lgwin)); size_t hash_table_size = HashTableSize(MaxHashTableSize(params.quality), block_size); size_t hash_size = @@ -1873,7 +1874,7 @@ size_t BrotliEncoderEstimatePeakMemoryUsage(int quality, int lgwin, size_t short_ringbuffer_size = (size_t)1 << params.lgblock; int ringbuffer_bits = ComputeRbBits(¶ms); size_t ringbuffer_size = input_size < short_ringbuffer_size ? - input_size : (1u << ringbuffer_bits) + short_ringbuffer_size; + input_size : ((size_t)1u << ringbuffer_bits) + short_ringbuffer_size; size_t hash_size[4] = {0}; size_t metablock_size = BROTLI_MIN(size_t, input_size, MaxMetablockSize(¶ms)); diff --git a/c/enc/hash.h b/c/enc/hash.h index 3ceb66c..cda27d1 100644 --- a/c/enc/hash.h +++ b/c/enc/hash.h @@ -540,8 +540,8 @@ static BROTLI_INLINE void FindCompoundDictionaryMatch( const uint64_t hash_mask = (~((uint64_t)0U)) >> (64 - hash_bits); const uint32_t* slot_offsets = (uint32_t*)(&self[1]); - const uint16_t* heads = (uint16_t*)(&slot_offsets[1u << slot_bits]); - const uint32_t* items = (uint32_t*)(&heads[1u << bucket_bits]); + const uint16_t* heads = (uint16_t*)(&slot_offsets[(size_t)1u << slot_bits]); + const uint32_t* items = (uint32_t*)(&heads[(size_t)1u << bucket_bits]); const uint8_t* source = NULL; const size_t cur_ix_masked = cur_ix & ring_buffer_mask; @@ -652,8 +652,8 @@ static BROTLI_INLINE size_t FindAllCompoundDictionaryMatches( const uint64_t hash_mask = (~((uint64_t)0U)) >> (64 - hash_bits); const uint32_t* slot_offsets = (uint32_t*)(&self[1]); - const uint16_t* heads = (uint16_t*)(&slot_offsets[1u << slot_bits]); - const uint32_t* items = (uint32_t*)(&heads[1u << bucket_bits]); + const uint16_t* heads = (uint16_t*)(&slot_offsets[(size_t)1u << slot_bits]); + const uint32_t* items = (uint32_t*)(&heads[(size_t)1u << bucket_bits]); const uint8_t* source = NULL; const size_t cur_ix_masked = cur_ix & ring_buffer_mask; diff --git a/c/enc/literal_cost.c b/c/enc/literal_cost.c index 2ac847f..a129657 100644 --- a/c/enc/literal_cost.c +++ b/c/enc/literal_cost.c @@ -106,6 +106,8 @@ static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask, size_t utf8_pos = UTF8Position(last_c, c, max_utf8); size_t masked_pos = (pos + i) & mask; size_t histo = histogram[256 * utf8_pos + data[masked_pos]]; + static const size_t prologue_length = 2000; + static const double multiplier = 0.35 / 2000; double lit_cost; if (histo == 0) { histo = 1; @@ -120,8 +122,8 @@ static void EstimateBitCostsForLiteralsUTF8(size_t pos, size_t len, size_t mask, Perhaps because the entropy source is changing its properties rapidly in the beginning of the file, perhaps because the beginning of the data is a statistical "anomaly". */ - if (i < 2000) { - lit_cost += 0.7 - ((double)(2000 - i) / 2000.0 * 0.35); + if (i < prologue_length) { + lit_cost += 0.35 + multiplier * (double)i; } cost[i] = (float)lit_cost; } diff --git a/go/cbrotli/cgo.go b/go/cbrotli/cgo.go index f953f72..50d9d71 100644 --- a/go/cbrotli/cgo.go +++ b/go/cbrotli/cgo.go @@ -7,7 +7,5 @@ package cbrotli // Inform golang build system that it should link brotli libraries. -// #cgo LDFLAGS: -lbrotlicommon -// #cgo LDFLAGS: -lbrotlidec -// #cgo LDFLAGS: -lbrotlienc +// #cgo pkg-config: libbrotlicommon libbrotlidec libbrotlienc import "C" diff --git a/java/WORKSPACE.bazel b/java/WORKSPACE.bazel index 0b23445..9e9ef28 100644 --- a/java/WORKSPACE.bazel +++ b/java/WORKSPACE.bazel @@ -7,15 +7,29 @@ local_repository( load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") +RULES_JAVA_VERSION = "8.3.2" +RULES_JAVA_SHA = "9b9614f8a7f7b7ed93cb7975d227ece30fe7daed2c0a76f03a5ee37f69e437de" RULES_JVM_EXTERNAL_TAG = "4.0" RULES_JVM_EXTERNAL_SHA = "31701ad93dbfe544d597dbe62c9a1fdd76d81d8a9150c2bf1ecf928ecdf97169" RULES_KOTLIN_VERSION = "1.9.0" RULES_KOTLIN_SHA = "5766f1e599acf551aa56f49dab9ab9108269b03c557496c54acaf41f98e2b8d6" +http_archive( + name = "rules_java", + sha256 = RULES_JAVA_SHA, + urls = [ + "https://github.com/bazelbuild/rules_java/releases/download/%s/rules_java-%s.tar.gz" % (RULES_JAVA_VERSION, RULES_JAVA_VERSION), + ], +) + +load("@rules_java//java:repositories.bzl", "rules_java_dependencies", "rules_java_toolchains") +rules_java_dependencies() +rules_java_toolchains() + http_archive( name = "rules_jvm_external", - strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG, sha256 = RULES_JVM_EXTERNAL_SHA, + strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG, url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG, ) @@ -23,15 +37,15 @@ load("@rules_jvm_external//:defs.bzl", "maven_install") http_archive( name = "rules_kotlin", - urls = ["https://github.com/bazelbuild/rules_kotlin/releases/download/v%s/rules_kotlin-v%s.tar.gz" % (RULES_KOTLIN_VERSION, RULES_KOTLIN_VERSION)], sha256 = RULES_KOTLIN_SHA, + urls = ["https://github.com/bazelbuild/rules_kotlin/releases/download/v%s/rules_kotlin-v%s.tar.gz" % (RULES_KOTLIN_VERSION, RULES_KOTLIN_VERSION)], ) load("@rules_kotlin//kotlin:repositories.bzl", "kotlin_repositories") -kotlin_repositories() # if you want the default. Otherwise see custom kotlinc distribution below +kotlin_repositories() # if you want the default. Otherwise see custom kotlinc distribution below load("@rules_kotlin//kotlin:core.bzl", "kt_register_toolchains") -kt_register_toolchains() # to use the default toolchain, otherwise see toolchains below +kt_register_toolchains() # to use the default toolchain, otherwise see toolchains below maven_install( artifacts = ["junit:junit:4.12"], @@ -43,37 +57,37 @@ maven_install( http_archive( name = "platforms", + sha256 = "8150406605389ececb6da07cbcb509d5637a3ab9a24bc69b1101531367d89d74", urls = [ "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.8/platforms-0.0.8.tar.gz", "https://github.com/bazelbuild/platforms/releases/download/0.0.8/platforms-0.0.8.tar.gz", ], - sha256 = "8150406605389ececb6da07cbcb509d5637a3ab9a24bc69b1101531367d89d74", ) http_file( name = "openjdk_jni_h", downloaded_file_path = "jni.h", - urls = ["https://raw.githubusercontent.com/openjdk/jdk/jdk8-b120/jdk/src/share/javavm/export/jni.h"], sha256 = "ed99792df48670072b78028faf704a8dcb6868fe140ccc7eced9b01dfa62fef4", + urls = ["https://raw.githubusercontent.com/openjdk/jdk/jdk8-b120/jdk/src/share/javavm/export/jni.h"], ) http_file( name = "openjdk_solaris_jni_md_h", downloaded_file_path = "jni_md.h", - urls = ["https://raw.githubusercontent.com/openjdk/jdk/jdk8-b120/jdk/src/solaris/javavm/export/jni_md.h"], sha256 = "b6cf7b06e5bba38d2daa2ff0789f99d396b3cb3bcc37d0367c8360fdccdef294", + urls = ["https://raw.githubusercontent.com/openjdk/jdk/jdk8-b120/jdk/src/solaris/javavm/export/jni_md.h"], ) http_file( name = "openjdk_macosx_jni_md_h", downloaded_file_path = "jni_md.h", - urls = ["https://raw.githubusercontent.com/openjdk/jdk/jdk8-b120/jdk/src/macosx/javavm/export/jni_md.h"], sha256 = "8f718071022e7e7f2fc9a229984b7e83582db91ed83861b49ce1461436fe8dc4", + urls = ["https://raw.githubusercontent.com/openjdk/jdk/jdk8-b120/jdk/src/macosx/javavm/export/jni_md.h"], ) http_file( name = "openjdk_windows_jni_md_h", downloaded_file_path = "jni_md.h", - urls = ["https://raw.githubusercontent.com/openjdk/jdk/jdk8-b120/jdk/src/windows/javavm/export/jni_md.h"], sha256 = "5479fb385ea1e11619f5c0cdfd9ccb3ea3a3fea0f5bc6176fb3ce62be29d759b", + urls = ["https://raw.githubusercontent.com/openjdk/jdk/jdk8-b120/jdk/src/windows/javavm/export/jni_md.h"], ) diff --git a/java/org/brotli/dec/build_defs.bzl b/java/org/brotli/dec/build_defs.bzl index 40ccbd1..e25f40f 100644 --- a/java/org/brotli/dec/build_defs.bzl +++ b/java/org/brotli/dec/build_defs.bzl @@ -1,6 +1,6 @@ """Utilities for Java brotli tests.""" -load("//third_party/bazel_rules/rules_java/java:java_test.bzl", "java_test") +load("@rules_java//java:java_test.bzl", "java_test") _TEST_JVM_FLAGS = [ "-DBROTLI_ENABLE_ASSERTS=true", diff --git a/python/brotli.py b/python/brotli.py index 9be4ed4..4787d63 100644 --- a/python/brotli.py +++ b/python/brotli.py @@ -27,8 +27,10 @@ def compress(string, mode=MODE_GENERIC, quality=11, lgwin=22, lgblock=0): Args: string (bytes): The input data. - mode (int, optional): The compression mode can be MODE_GENERIC (default), - MODE_TEXT (for UTF-8 format text input) or MODE_FONT (for WOFF 2.0). + mode (int, optional): The compression mode; value 0 should be used for + generic input (MODE_GENERIC); value 1 might be beneficial for UTF-8 text + input (MODE_TEXT); value 2 tunes encoder for WOFF 2.0 data (MODE_FONT). + Defaults to 0. quality (int, optional): Controls the compression-speed vs compression- density tradeoff. The higher the quality, the slower the compression. Range is 0 to 11. Defaults to 11.