Migrate to github actions (#920)

Not all combinations are migrated to the initial configuration; corresponding TODOs added.

Drive-by: additional combinations uncovered minor portability problems -> fixed
Drive-by: remove no-longer used "script" files.

Co-authored-by: Eugene Kliuchnikov <eustas@chromium.org>
This commit is contained in:
Eugene Kliuchnikov 2021-08-31 14:07:17 +02:00 committed by GitHub
parent 68f1b90ad0
commit 0e42caf359
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 523 additions and 543 deletions

382
.github/workflows/build_test.yml vendored Normal file
View File

@ -0,0 +1,382 @@
# Copyright 2021 Google Inc. All Rights Reserved.
#
# Distributed under MIT license.
# See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
# Workflow for building and running tests under Ubuntu
name: Build/Test
on:
push:
branches:
- master
pull_request:
types: [opened, reopened, labeled, synchronize]
jobs:
ubuntu_build:
name: Build and test ${{ matrix.name }}
runs-on: ${{ matrix.os || 'ubuntu-latest' }}
defaults:
run:
shell: bash
strategy:
matrix:
include:
# TODO: consider running this combination in docker
#- name: cmake:gcc4.4
# TODO: consider running this combination in docker
#- name: cmake:gcc7
- name: cmake:gcc9
build_system: cmake
c_compiler: gcc-9
cxx_compiler: g++-9
os: ubuntu-18.04
# TODO: consider running this combination in docker
#- name: cmake:clang3.5
- name: cmake:clang12
build_system: cmake
c_compiler: clang-12
cxx_compiler: clang++12
- name: cmake:clang12:asan
build_system: cmake
sanitizer: address
c_compiler: clang-12
cxx_compiler: clang++12
- name: cmake:clang12:tsan
build_system: cmake
sanitizer: thread
c_compiler: clang-12
cxx_compiler: clang++12
- name: cmake:clang12:ubsan
build_system: cmake
sanitizer: undefined
c_compiler: clang-12
cxx_compiler: clang++-12
c_flags: -fno-sanitize-recover=undefined,integer
- name: cmake:qemu-arm-neon-gcc
build_system: cmake
c_compiler: arm-linux-gnueabihf-gcc
cxx_compiler: arm-linux-gnueabihf-g++
c_flags: -march=armv7-a -mfloat-abi=hard -mfpu=neon
extra_apt_pkgs: gcc-arm-linux-gnueabihf libc6-dev-armhf-cross qemu-user
- name: cmake-osx:clang
build_system: cmake
c_compiler: clang
cxx_compiler: clang++
os: macos-latest
- name: cmake-osx:gcc
build_system: cmake
c_compiler: gcc
cxx_compiler: g++
os: macos-latest
- name: cmake-win64:msvc2017-rel
build_system: cmake
cmake_generator: Visual Studio 15 2017 Win64
cmake_config: Release
os: windows-2016
- name: cmake-win64:msvc2017-dbg
build_system: cmake
cmake_generator: Visual Studio 15 2017 Win64
cmake_config: Debug
os: windows-2016
# TODO: consider running this combination in docker
#- name: autotools:gcc5
- name: autotools:gcc10
build_system: autotools
c_compiler: gcc-10
cxx_compiler: g++-10
- name: autotools:clang12
build_system: autotools
c_compiler: clang-12
cxx_compiler: clang++12
- name: fuzz:clang12
build_system: fuzz
c_compiler: clang-12
cxx_compiler: clang++12
# TODO: consider running this combination in docker
#- name: python2.7:gcc5
- name: python27:clang9
build_system: python
c_compiler: clang-9
python_version: 2.7
cxx_compiler: clang++-9
os: ubuntu-18.04
# TODO: consider running this combination in docker
#- name: python27-win
# build_system: python
# python_version: 2.7
# # TODO: investigate why win-builds can't run tests
# py_setuptools_cmd: build_ext
# os: windows-2016
# TODO: consider running this combination in docker
#- name: python3.6:gcc5
# TODO: consider running this combination in docker
#- name: python3.7:gcc5
# TODO: consider running this combination in docker
#- name: python3.8:gcc5
- name: python39:clang12
build_system: python
python_version: 3.9
c_compiler: clang-12
cxx_compiler: clang++-12
- name: python39-win
build_system: python
python_version: 3.9
# TODO: investigate why win-builds can't run tests
py_setuptools_cmd: build_ext
os: windows-2019
- name: maven
build_system: maven
- name: bazel:root
build_system: bazel
bazel_project: .
- name: bazel:go
build_system: bazel
bazel_project: go
- name: bazel:java
build_system: bazel
bazel_project: java
- name: bazel:js
build_system: bazel
bazel_project: js
- name: bazel:research
build_system: bazel
bazel_project: research
- name: bazel-osx:root
build_system: bazel
bazel_project: .
os: macos-latest
- name: bazel-osx:go
build_system: bazel
bazel_project: go
os: macos-latest
- name: bazel-osx:java
build_system: bazel
bazel_project: java
os: macos-latest
- name: bazel-osx:js
build_system: bazel
bazel_project: js
os: macos-latest
- name: bazel-osx:research
build_system: bazel
bazel_project: research
os: macos-latest
- name: bazel-win:root
build_system: bazel
bazel_project: .
os: windows-latest
# TODO: use single dll on windows, otherwise it fails to link
#- name: bazel-win:go
# build_system: bazel
# bazel_project: go
# os: windows-latest
- name: bazel-win:java
build_system: bazel
bazel_project: java
os: windows-latest
# TODO: blocked by Bazel Closure rules issue
#- name: bazel-win:js
# build_system: bazel
# bazel_project: js
# os: windows-latest
- name: bazel-win:research
build_system: bazel
bazel_project: research
os: windows-latest
- name: make
build_system: make
- name: make-osx
build_system: make
os: macos-latest
# TODO: add 32/64-bit x MSYS2/mingw/Cygwin toolchain support.
- name: make-win
build_system: make
os: windows-latest
env:
CC: ${{ matrix.c_compiler || 'gcc' }}
CXX: ${{ matrix.cxx_compiler || 'gcc' }}
steps:
- name: Install extra deps @ Ubuntu
if: ${{ runner.os == 'Linux' }}
# Already installed: bazel, clang{10-12}, cmake, gcc{9,10}, java{8,11}, maven, python{2.7,3.5-3.9}
run: |
EXTRA_PACKAGES="${{ matrix.extra_apt_pkgs || '' }}"
sudo apt update
sudo apt install -y ${EXTRA_PACKAGES}
- name: Checkout the source
uses: actions/checkout@v2
with:
submodules: false
fetch-depth: 1
#- name: Checkout VC9 for Python
# if: ${{ runner.os == 'Windows' && matrix.build_system == 'python' && matrix.python_version == '2.7' }}
# uses: actions/checkout@v2
# with:
# repository: reider-roque/sulley-win-installer
# path: third_party/VCForPython27
- name: Configure / Build / Test with CMake
if: ${{ matrix.build_system == 'cmake' }}
run: |
export ASAN_OPTIONS=detect_leaks=0
declare -a CMAKE_OPTIONS=()
CMAKE_OPTIONS+=("-DCMAKE_VERBOSE_MAKEFILE=ON")
[ ! -z '${{ matrix.c_compiler || '' }}' ] && CMAKE_OPTIONS+=(-DCMAKE_C_COMPILER='${{ matrix.c_compiler }}')
[ ! -z '${{ matrix.cxx_compiler || '' }}' ] && CMAKE_OPTIONS+=(-DCMAKE_CXX_COMPILER='${{ matrix.cxx_compiler }}')
[ ! -z '${{ matrix.sanitizer || '' }}' ] && CMAKE_OPTIONS+=(-DENABLE_SANITIZER='${{ matrix.sanitizer }}')
[ ! -z '${{ matrix.cmake_generator || '' }}' ] && export CMAKE_GENERATOR='${{ matrix.cmake_generator }}'
declare -a CMAKE_BUILD_OPTIONS=()
[ ! -z '${{ matrix.cmake_config || '' }}' ] && CMAKE_BUILD_OPTIONS+=(--config '${{ matrix.cmake_config }}')
declare -a CMAKE_TEST_OPTIONS=()
[ ! -z '${{ matrix.cmake_config || '' }}' ] && CMAKE_TEST_OPTIONS+=(-C '${{ matrix.cmake_config }}')
cmake -B out . ${CMAKE_OPTIONS[*]} -DCMAKE_C_FLAGS='${{ matrix.c_flags || '' }}'
cmake --build out ${CMAKE_BUILD_OPTIONS[*]}
cd out; ctest ${CMAKE_TEST_OPTIONS[*]}; cd ..
- name: Quick Fuzz
if: ${{ matrix.build_system == 'fuzz' }}
run: |
export ASAN_OPTIONS=detect_leaks=0
./c/fuzz/test_fuzzer.sh
- name: Build with Bazel
if: ${{ matrix.build_system == 'bazel' }}
run: |
cd ${GITHUB_WORKSPACE}/${{ matrix.bazel_project }}
bazel build -c opt ...:all
- name: Fix symlinks for Bazel (Windows)
if: ${{ matrix.build_system == 'bazel' && runner.os == 'Windows' && matrix.bazel_project == 'java' }}
shell: python
run: |
import fnmatch
import os
import os.path
from shutil import copyfile
os.chdir('${{ matrix.bazel_project }}')
print('Searching for manifests in ' + os.getcwd())
matches = []
for root, dirnames, filenames in os.walk('bazel-bin\\org\\brotli'):
for filename in fnmatch.filter(filenames, '*.runfiles_manifest'):
matches.append(os.path.join(root, filename))
for match in matches:
print('Scanning manifest ' + match)
runfiles = match[:-len('_manifest')]
with open(match) as manifest:
for entry in manifest:
entry = entry.strip()
if not entry.startswith("org_brotli_java"):
continue
if entry.startswith('org_brotli_java/external'):
continue
(alias, space, link) = entry.partition(' ')
if alias.endswith('.jar') or alias.endswith('.exe'):
continue
link = link.replace('/', '\\')
alias = alias.replace('/', '\\')
dst = os.path.join(runfiles, alias)
if not os.path.exists(dst):
print(link + ' -> ' + dst)
parent = os.path.dirname(dst)
if not os.path.exists(parent):
os.makedirs(parent)
copyfile(link, dst)
print('Finished resolving symlinks')
- name: Test with Bazel
if: ${{ matrix.build_system == 'bazel' }}
run: |
cd ${GITHUB_WORKSPACE}/${{ matrix.bazel_project }}
bazel query "tests(...)" --output=label > ${RUNNER_TEMP}/tests.lst
[ -s ${RUNNER_TEMP}/tests.lst ] && bazel test -c opt ...:all
bazel clean
- name: Build / Test with Maven
if: ${{ matrix.build_system == 'maven' }}
run: |
export MAVEN_OPTS=-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
cd java/org/brotli
mvn -B install
cd integration
mvn -B verify
- name: Build / Test with Autotools
if: ${{ matrix.build_system == 'autotools' }}
run: |
./bootstrap && ./configure && make
- name: Build / Test with Make
if: ${{ matrix.build_system == 'make' }}
run: |
make brotli
make test
- uses: actions/setup-python@v2
if: ${{ matrix.build_system == 'python' }}
with:
python-version: ${{ matrix.python_version }}
# TODO: investigate, why msiexec hangs
#- name: Install VC9 for Python
# if: ${{ runner.os == 'Windows' && matrix.build_system == 'python' && matrix.python_version == '2.7' }}
# run: |
# echo "070474db76a2e625513a5835df4595df9324d820f9cc97eab2a596dcbc2f5cbf third_party/VCForPython27/VCForPython27.msi" | sha256sum --check --status
# msiexec ALLUSERS=1 /qn /norestart /i third_party/VCForPython27/VCForPython27.msi /l*v ${RUNNER_TEMP}/msiexec.log
# cat ${RUNNER_TEMP}/msiexec.log
- name: Build / Test with Python
if: ${{ matrix.build_system == 'python' }}
run: |
python -VV
python -c "import sys; sys.exit('Invalid python version') if '.'.join(map(str,sys.version_info[0:2])) != '${{ matrix.python_version }}' else True"
python setup.py ${{ matrix.py_setuptools_cmd || 'test'}}

View File

@ -1,267 +0,0 @@
language: c
sudo: false
branches:
only:
- master
cache:
directories:
- $HOME/Library/Caches/Homebrew
- /usr/local/Homebrew
before_cache:
- if [ "${TRAVIS_OS_NAME}" = "osx" ]; then brew cleanup; fi
- if [ "${TRAVIS_OS_NAME}" = "osx" ]; then find /usr/local/Homebrew \! -regex ".+\.git.+" -delete; fi
matrix:
include:
###
## Linux builds using various versions of GCC.
###
- os: linux
env: BUILD_SYSTEM=cmake C_COMPILER=gcc-7 CXX_COMPILER=g++-7
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-7
- g++-7
- os: linux
env: BUILD_SYSTEM=cmake C_COMPILER=gcc-4.4 CXX_COMPILER=g++-4.4
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-4.4
- g++-4.4
###
## Test that Autotools build works.
###
- os: linux
env: BUILD_SYSTEM=autotools C_COMPILER=gcc-5 CXX_COMPILER=g++-5
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-5
- g++-5
###
## Test that fuzzer is compiling / working.
###
- os: linux
env: BUILD_SYSTEM=fuzz C_COMPILER=clang-7 CXX_COMPILER=clang++-7 ASAN_OPTIONS=detect_leaks=0
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-xenial-7
packages:
- clang-7
###
## clang on Linux
###
- os: linux
env: BUILD_SYSTEM=cmake C_COMPILER=clang-7 CXX_COMPILER=clang++-7
addons:
apt:
sources:
- llvm-toolchain-xenial-7
- ubuntu-toolchain-r-test
packages:
- clang-7
- os: linux
env: BUILD_SYSTEM=cmake C_COMPILER=clang-3.5 CXX_COMPILER=clang++-3.5
addons:
apt:
sources:
- llvm-toolchain-trusty-3.5
- ubuntu-toolchain-r-test
packages:
- clang-3.5
###
## testing arm via qemu on Linux
###
- os: linux
env: BUILD_SYSTEM=cmake C_COMPILER=arm-linux-gnueabihf-gcc CXX_COMPILER=arm-linux-gnueabihf-g++ CFLAGS="-march=armv7-a -mfloat-abi=hard -mfpu=neon"
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- qemu
- gcc-arm-linux-gnueabihf
- libc6-dev-armhf-cross
###
## PGI Community Edition on Linux
###
# Installer is currently broken
#- os: linux
# env: BUILD_SYSTEM=cmake C_COMPILER=pgcc CXX_COMPILER=pgc++
###
## Python builds on Linux
###
- os: linux
language: python
python: 2.7
env: BUILD_SYSTEM=python C_COMPILER=gcc-5 CXX_COMPILER=g++-5
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-5
- g++-5
- os: linux
language: python
python: 3.6
env: BUILD_SYSTEM=python C_COMPILER=gcc-5 CXX_COMPILER=g++-5
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-5
- g++-5
- os: linux
language: python
python: 3.7
env: BUILD_SYSTEM=python C_COMPILER=gcc-5 CXX_COMPILER=g++-5
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-5
- g++-5
- os: linux
language: python
python: 3.8
env: BUILD_SYSTEM=python C_COMPILER=gcc-5 CXX_COMPILER=g++-5
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-5
- g++-5
###
## CMake on OS X
##
## These all work, but it seems unnecessary to actually build them
## all since we already test all these versions of GCC on Linux.
## We'll just test 4.4 and the most recent version.
###
- os: osx
osx_image: xcode12.2
env: BUILD_SYSTEM=cmake C_COMPILER=gcc CXX_COMPILER=g++
#- os: osx
# osx_image: xcode12.2
# env: BUILD_SYSTEM=cmake C_COMPILER=gcc-4.9 CXX_COMPILER=g++-4.9
- os: osx
osx_image: xcode12.2
env: BUILD_SYSTEM=cmake
###
## Python 2.7 OS X build (using the system /usr/bin/python)
###
- os: osx
osx_image: xcode12.2
env: BUILD_SYSTEM=python C_COMPILER=gcc CXX_COMPILER=g++
###
## Sanitizers
###
- os: linux
env: BUILD_SYSTEM=cmake C_COMPILER=clang-7 CXX_COMPILER=clang++-7 SANITIZER=address ASAN_OPTIONS=detect_leaks=0
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-xenial-7
packages:
- clang-7
- os: linux
env: BUILD_SYSTEM=cmake C_COMPILER=clang-7 CXX_COMPILER=clang++-7 SANITIZER=thread
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-xenial-7
packages:
- clang-7
- os: linux
env: BUILD_SYSTEM=cmake C_COMPILER=clang-7 CXX_COMPILER=clang++-7 SANITIZER=undefined CFLAGS="-fno-sanitize-recover=undefined,integer"
addons:
apt:
sources:
- ubuntu-toolchain-r-test
- llvm-toolchain-xenial-7
packages:
- clang-7
- os: linux
env: BUILD_SYSTEM=maven
jdk:
# maven + jdk11 + javadoc == trouble
- openjdk10
language: java
- os: linux
sudo: required
language: java
jdk: oraclejdk9
env: BUILD_SYSTEM=bazel
addons:
apt:
sources:
- sourceline: "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8"
key_url: "https://storage.googleapis.com/bazel-apt/doc/apt-key.pub.gpg"
- ubuntu-toolchain-r-test
packages:
- bazel
- os: osx
osx_image: xcode12.2
env: BUILD_SYSTEM=bazel
language: java
before_install:
###
## If we use the matrix to set CC/CXX Travis, overwrites the values,
## so instead we use C/CXX_COMPILER, then copy the values to CC/CXX
## here (after Travis has set CC/CXX).
###
- if [ -n "${C_COMPILER}" ]; then export CC="${C_COMPILER}"; fi
- if [ -n "${CXX_COMPILER}" ]; then export CXX="${CXX_COMPILER}"; fi
- scripts/.travis.sh before_install
install:
- scripts/.travis.sh install
script:
- scripts/.travis.sh script
after_success:
- scripts/.travis.sh after_success
before_deploy:
- scripts/.travis.sh before_deploy
deploy:
- provider: bintray
file: "scripts/.bintray.json"
user: "eustas"
key:
secure: "Kbam/lTAdz72fZivbs6riJT+Y4PbuKP7r6t5PAWxJxAAykjwnYTRe3zF472g9HCE14KYMsdB+KSYSgg6TGJnqGC9gL9xhhGU9U/WmA+vbMWS/MSnMWpK9IRpp77pM2i2NKZD4v33JuEwKFCBJP3Vj6QQ5Qd1NKdobuXJyznhgnw="
on:
condition: "${BUILD_SYSTEM} = bazel"
skip_cleanup: true

View File

@ -164,7 +164,7 @@ BROTLI_DEC_API BROTLI_BOOL BrotliDecoderSetParameter(
* Data provided to this method should be kept accessible until
* decoding is finished and decoder instance is destroyed.
*
* @note Dictionaries could NOT be attached after actual decoding is started.
* @note Dictionaries can NOT be attached after actual decoding is started.
*
* @param state decoder instance
* @param type dictionary data format

View File

@ -11,4 +11,4 @@ LT_INIT
AC_CONFIG_FILES([Makefile scripts/libbrotlicommon.pc scripts/libbrotlidec.pc scripts/libbrotlienc.pc])
AC_OUTPUT
AC_OUTPUT

View File

@ -1,4 +1,4 @@
.TH "BROTLI" "1" "February 2018" "brotli 1.0.0" "User commands"
.TH "BROTLI" "1" "August 2021" "" "User commands"
.SH "NAME"
\fBbrotli\fR \- brotli, unbrotli \- compress or decompress files
.SH SYNOPSIS
@ -108,6 +108,10 @@ Conflicting or duplicate \fIoptions\fR are not allowed\.
windows size improve density; decoder might require up to window size
memory to operate
.IP \(bu 2
\fB\-D FILE\fP, \fB\-\-dictionary=FILE\fP:
use FILE as raw (LZ77) dictionary; same dictionary MUST be used both for
compression and decompression
.IP \(bu 2
\fB\-S SUF\fP, \fB\-\-suffix=SUF\fP:
output file suffix (default: \fB\|\.br\fP)
.IP \(bu 2

View File

@ -1,4 +1,4 @@
.TH "constants.h" 3 "Wed May 13 2020" "Brotli" \" -*- nroff -*-
.TH "constants.h" 3 "August 2021" "Brotli" \" -*- nroff -*-
.ad l
.nh
.SH NAME

View File

@ -1,4 +1,4 @@
.TH "decode.h" 3 "Thu Feb 22 2018" "Brotli" \" -*- nroff -*-
.TH "decode.h" 3 "August 2021" "Brotli" \" -*- nroff -*-
.ad l
.nh
.SH NAME
@ -36,6 +36,10 @@ decode.h \- API for Brotli decompression\&.
.in +1c
.ti -1c
.RI "\fBBROTLI_BOOL\fP \fBBrotliDecoderAttachDictionary\fP (\fBBrotliDecoderState\fP *state, BrotliSharedDictionaryType type, size_t data_size, const uint8_t data[data_size])"
.br
.RI "\fIAdds LZ77 prefix dictionary, adds or replaces built-in static dictionary and transforms\&. \fP"
.ti -1c
.RI "\fBBrotliDecoderState\fP * \fBBrotliDecoderCreateInstance\fP (\fBbrotli_alloc_func\fP alloc_func, \fBbrotli_free_func\fP free_func, void *opaque)"
.br
.RI "\fICreates an instance of \fBBrotliDecoderState\fP and initializes it\&. \fP"
@ -167,6 +171,35 @@ Partially done; should be called again with more input\&.
Partially done; should be called again with more output\&.
.SH "Function Documentation"
.PP
.SS "\fBBROTLI_BOOL\fP BrotliDecoderAttachDictionary (\fBBrotliDecoderState\fP * state, BrotliSharedDictionaryType type, size_t data_size, const uint8_t data[data_size])"
.PP
Adds LZ77 prefix dictionary, adds or replaces built-in static dictionary and transforms\&. Attached dictionary ownership is not transferred\&. Data provided to this method should be kept accessible until decoding is finished and decoder instance is destroyed\&.
.PP
\fBNote:\fP
.RS 4
Dictionaries can NOT be attached after actual decoding is started\&.
.RE
.PP
\fBParameters:\fP
.RS 4
\fIstate\fP decoder instance
.br
\fItype\fP dictionary data format
.br
\fIdata_size\fP length of memory region pointed by \fCdata\fP
.br
\fIdata\fP dictionary data in format corresponding to \fCtype\fP
.RE
.PP
\fBReturns:\fP
.RS 4
\fBBROTLI_FALSE\fP if dictionary is corrupted, or dictionary count limit is reached
.PP
\fBBROTLI_TRUE\fP if dictionary is accepted / attached
.RE
.PP
.SS "\fBBrotliDecoderState\fP* BrotliDecoderCreateInstance (\fBbrotli_alloc_func\fP alloc_func, \fBbrotli_free_func\fP free_func, void * opaque)"
.PP

View File

@ -1,4 +1,4 @@
.TH "encode.h" 3 "Thu Feb 22 2018" "Brotli" \" -*- nroff -*-
.TH "encode.h" 3 "August 2021" "Brotli" \" -*- nroff -*-
.ad l
.nh
.SH NAME
@ -76,6 +76,10 @@ encode.h \- API for Brotli compression\&.
.in +1c
.ti -1c
.RI "\fBBROTLI_BOOL\fP \fBBrotliEncoderAttachPreparedDictionary\fP (\fBBrotliEncoderState\fP *state, const BrotliEncoderPreparedDictionary *dictionary)"
.br
.RI "\fIAttaches a prepared dictionary of any type to the encoder\&. \fP"
.ti -1c
.RI "\fBBROTLI_BOOL\fP \fBBrotliEncoderCompress\fP (int quality, int lgwin, \fBBrotliEncoderMode\fP mode, size_t input_size, const uint8_t input_buffer[input_size], size_t *encoded_size, uint8_t encoded_buffer[*encoded_size])"
.br
.RI "\fIPerforms one-shot memory-to-memory compression\&. \fP"
@ -104,6 +108,10 @@ encode.h \- API for Brotli compression\&.
.br
.RI "\fICalculates the output size bound for the given \fCinput_size\fP\&. \fP"
.ti -1c
.RI "BrotliEncoderPreparedDictionary * \fBBrotliEncoderPrepareDictionary\fP (BrotliSharedDictionaryType type, size_t data_size, const uint8_t data[data_size], int quality, \fBbrotli_alloc_func\fP alloc_func, \fBbrotli_free_func\fP free_func, void *opaque)"
.br
.RI "\fIPrepares a shared dictionary from the given file format for the encoder\&. \fP"
.ti -1c
.RI "\fBBROTLI_BOOL\fP \fBBrotliEncoderSetParameter\fP (\fBBrotliEncoderState\fP *state, \fBBrotliEncoderParameter\fP param, uint32_t value)"
.br
.RI "\fISets the specified parameter to the given encoder instance\&. \fP"
@ -318,6 +326,19 @@ If offset is not 0, then stream header is omitted\&. In any case output start is
Range is not artificially limited, but all the values greater or equal to maximal window size have the same effect\&. Values greater than 2**30 are not allowed\&.
.SH "Function Documentation"
.PP
.SS "\fBBROTLI_BOOL\fP BrotliEncoderAttachPreparedDictionary (\fBBrotliEncoderState\fP * state, const BrotliEncoderPreparedDictionary * dictionary)"
.PP
Attaches a prepared dictionary of any type to the encoder\&. Can be used multiple times to attach multiple dictionaries\&. The dictionary type was determined by BrotliEncoderPrepareDictionary\&. Multiple raw prefix dictionaries and/or max 1 serialized dictionary with custom words can be attached\&.
.PP
\fBReturns:\fP
.RS 4
\fBBROTLI_FALSE\fP in case of error
.PP
\fBBROTLI_TRUE\fP otherwise
.RE
.PP
.SS "\fBBROTLI_BOOL\fP BrotliEncoderCompress (int quality, int lgwin, \fBBrotliEncoderMode\fP mode, size_t input_size, const uint8_t input_buffer[input_size], size_t * encoded_size, uint8_t encoded_buffer[*encoded_size])"
.PP
@ -509,6 +530,29 @@ Result is only valid if quality is at least \fC2\fP and, in case \fBBrotliEncode
.RE
.PP
.SS "BrotliEncoderPreparedDictionary* BrotliEncoderPrepareDictionary (BrotliSharedDictionaryType type, size_t data_size, const uint8_t data[data_size], int quality, \fBbrotli_alloc_func\fP alloc_func, \fBbrotli_free_func\fP free_func, void * opaque)"
.PP
Prepares a shared dictionary from the given file format for the encoder\&. \fCalloc_func\fP and \fCfree_func\fP \fBMUST\fP be both zero or both non-zero\&. In the case they are both zero, default memory allocators are used\&. \fCopaque\fP is passed to \fCalloc_func\fP and \fCfree_func\fP when they are called\&. \fCfree_func\fP has to return without doing anything when asked to free a NULL pointer\&.
.PP
\fBParameters:\fP
.RS 4
\fItype\fP type of dictionary stored in data
.br
\fIdata_size\fP size of \fCdata\fP buffer
.br
\fIdata\fP pointer to the dictionary data
.br
\fIquality\fP the maximum Brotli quality to prepare the dictionary for, use BROTLI_MAX_QUALITY by default
.br
\fIalloc_func\fP custom memory allocation function
.br
\fIfree_func\fP custom memory free function
.br
\fIopaque\fP custom memory manager handle
.RE
.PP
.SS "\fBBROTLI_BOOL\fP BrotliEncoderSetParameter (\fBBrotliEncoderState\fP * state, \fBBrotliEncoderParameter\fP param, uint32_t value)"
.PP

View File

@ -1,4 +1,4 @@
.TH "types.h" 3 "Thu Feb 22 2018" "Brotli" \" -*- nroff -*-
.TH "types.h" 3 "August 2021" "Brotli" \" -*- nroff -*-
.ad l
.nh
.SH NAME

View File

@ -20,4 +20,4 @@ load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_depe
go_rules_dependencies()
go_register_toolchains(version = "1.16")
go_register_toolchains(version = "1.16")

View File

@ -76,6 +76,9 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.10.4</version>
<configuration>
<source>8</source>
</configuration>
<executions>
<execution>
<id>attach-javadocs</id>

View File

@ -38,7 +38,8 @@ genrule(
srcs = ["include/config.h.cmake"],
outs = ["include/config.h"],
cmd = ("awk '{ " +
"gsub(/@HAVE_IO_H 1@/, \"HAVE_IO_H 0\"); " +
"gsub(/HAVE_IO_H 1/, \"HAVE_IO_H 0\"); " +
"gsub(/HAVE_STRINGS_H 1/, \"HAVE_STRINGS_H 0\"); " +
commom_awk_replaces +
"print; }' $(<) > $(@)"),
)

View File

@ -12,6 +12,7 @@ new_git_repository(
build_file = "@//:BUILD.libdivsufsort",
commit = "5f60d6f026c30fb4ac296f696b3c8b0eb71bd428",
remote = "https://github.com/y-256/libdivsufsort",
shallow_since = "1445958113 +0900",
)
new_git_repository(
@ -27,4 +28,5 @@ cc_library(
""",
commit = "ca7cb332011ec37a8436487f210f396b84bd8273",
remote = "https://github.com/hillbig/esaxx",
shallow_since = "1391400691 +0900",
)

View File

@ -6,7 +6,17 @@
#include <stdio.h>
#include <stdlib.h>
#if !defined(_WIN32)
#include <unistd.h>
#else
#include <io.h>
#define fdopen _fdopen
#if !defined(__MINGW32__)
#define STDIN_FILENO _fileno(stdin)
#define STDOUT_FILENO _fileno(stdout)
#endif
#endif
#include <brotli/decode.h>

View File

@ -5,6 +5,10 @@
#include "third_party/esaxx/sais.hxx"
#if defined(_MSC_VER)
#include <intrin.h> /* __popcnt64 */
#endif
/* Used for quick SA-entry to file mapping. Each file is padded to size that
is a multiple of chunk size. */
#define CHUNK_SIZE 64
@ -30,7 +34,11 @@ typedef uint32_t TextIdx;
typedef int32_t TextSaIdx;
static size_t popcount(uint64_t u) {
#if defined(_MSC_VER)
return static_cast<size_t>(__popcnt64(u));
#else
return static_cast<size_t>(__builtin_popcountll(u));
#endif
}
/* Condense terminators and pad file entries. */

View File

@ -3,7 +3,9 @@
#include <cstdio>
#include <cstring>
#include <fstream>
#if !defined(_MSC_VER)
#include <glob.h>
#endif
#include <vector>
#include "./deorummolae.h"
@ -268,14 +270,19 @@ int main(int argc, char const* argv[]) {
}
}
bool ok = true;
#if defined(_MSC_VER)
const char* resolved_path = argv[i];
#else
glob_t resolved_paths;
memset(&resolved_paths, 0, sizeof(resolved_paths));
bool ok = true;
if (glob(argv[i], GLOB_TILDE, NULL, &resolved_paths) == 0) {
for(size_t j = 0; j < resolved_paths.gl_pathc; ++j) {
std::string content = readFile(resolved_paths.gl_pathv[j]);
const char* resolved_path = resolved_paths.gl_pathv[j];
#endif
std::string content = readFile(resolved_path);
if (chunkLen == 0) {
paths.emplace_back(resolved_paths.gl_pathv[j]);
paths.emplace_back(resolved_path);
data.insert(data.end(), content.begin(), content.end());
total += content.size();
sizes.push_back(content.size());
@ -293,11 +300,13 @@ int main(int argc, char const* argv[]) {
total += chunk.size();
sizes.push_back(chunk.size());
}
#if !defined(_MSC_VER)
}
} else {
ok = false;
}
globfree(&resolved_paths);
#endif
if (!ok) exit(1);
}

View File

@ -1,19 +0,0 @@
{
"package": {
"name": "bin",
"repo": "brotli",
"subject": "eustas"
},
"version": {"name": "latest"},
"files": [
{
"includePattern": "brotli.zip",
"uploadPattern": "brotli-${TRAVIS_OS_NAME}-${RELEASE_DATE}.zip",
"matrix_params": {"override": 1}
}
],
"publish": true
}

View File

@ -1 +0,0 @@
#!/usr/bin/env bash

View File

@ -1,81 +0,0 @@
#!/bin/bash
case "$1" in
"before_install")
;;
"install")
case "${TRAVIS_OS_NAME}" in
"osx")
brew update >/dev/null
brew install binutils
case "${CC}" in
"gcc-"*)
which ${CC} || brew install $(echo "${CC}" | sed 's/\-/@/') || brew link --overwrite $(echo "${CC}" | sed 's/\-/@/')
;;
esac
case "${BUILD_SYSTEM}" in
"bazel")
brew cask install homebrew/cask-versions/adoptopenjdk8
brew install bazel
;;
esac
;;
"linux")
case "${CC}" in
"pgcc")
wget 'https://raw.githubusercontent.com/nemequ/pgi-travis/de6212d94fd0e7d07a6ef730c23548c337c436a7/install-pgi.sh'
echo 'acd3ef995ad93cfb87d26f65147395dcbedd4c3c844ee6ec39616f1a347c8df5 install-pgi.sh' | sha256sum -c --strict || exit 1
/bin/sh install-pgi.sh
;;
esac
;;
esac
;;
"script")
case "${BUILD_SYSTEM}" in
"cmake")
mkdir builddir && cd builddir
CMAKE_FLAGS=
if [ "${CROSS_COMPILE}" = "yes" ]; then
CMAKE_FLAGS="-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY -DCMAKE_SYSTEM_NAME=Windows -DCMAKE_RC_COMPILER=${RC_COMPILER}"
fi
cmake ${CMAKE_FLAGS} -DCMAKE_C_COMPILER="$CC" -DCMAKE_CXX_COMPILER="$CXX" -DENABLE_SANITIZER="${SANITIZER}" -DCMAKE_C_FLAGS="${CFLAGS}" .. || exit 1
make VERBOSE=1 || exit 1
ctest -V || exit 1
;;
"python")
python setup.py test
;;
"maven")
cd java/org/brotli
mvn install && cd integration && mvn verify
;;
"autotools")
./bootstrap && ./configure && make
;;
"fuzz")
./c/fuzz/test_fuzzer.sh
;;
"bazel")
bazel build -c opt ...:all &&
cd go && bazel test -c opt ...:all && cd .. &&
cd java && bazel test -c opt ...:all && cd .. &&
cd js && bazel test -c opt ...:all && cd .. &&
cd research && bazel build -c opt ...:all && cd ..
;;
esac
;;
"after_success")
;;
"before_deploy")
case "${BUILD_SYSTEM}" in
"bazel")
export RELEASE_DATE=`date +%Y-%m-%d`
perl -p -i -e 's/\$\{([^}]+)\}/defined $ENV{$1} ? $ENV{$1} : $&/eg' scripts/.bintray.json
zip -j9 brotli.zip bazel-bin/libbrotli*.a bazel-bin/libbrotli*.so bazel-bin/brotli
;;
esac
;;
esac

View File

@ -1,111 +0,0 @@
branches:
only:
- master
environment:
matrix:
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
BUILD_SYSTEM: CMake
GENERATOR: Visual Studio 15 2017 Win64
CONFIG: Release
- APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
BUILD_SYSTEM: CMake
GENERATOR: Visual Studio 15 2017 Win64
CONFIG: Debug
- BUILD_SYSTEM: CMake
GENERATOR: Visual Studio 14 2015 Win64
CONFIG: Debug
- BUILD_SYSTEM: CMake
GENERATOR: Visual Studio 14 2015
CONFIG: Debug
- BUILD_SYSTEM: Python
PYTHON: "C:\\Python27"
PYTHON_VERSION: "2.7.x"
PYTHON_ARCH: "32"
- BUILD_SYSTEM: Python
PYTHON: "C:\\Python36-x64"
PYTHON_VERSION: "3.6.x"
PYTHON_ARCH: "64"
- BUILD_SYSTEM: make
ARCH: "i686"
- BUILD_SYSTEM: make
ARCH: "x86_64"
- BUILD_SYSTEM: bazel
install:
- IF "%BUILD_SYSTEM%"=="Python" (
SET "PATH=%PYTHON%;%PYTHON%\Scripts;%PATH%" &&
python -m pip install --upgrade pip &&
pip install --upgrade setuptools
)
- IF "%BUILD_SYSTEM%"=="make" (
IF "%ARCH%"=="i686" (
SET "TOOLCHAIN=i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32"
) ELSE (
SET "TOOLCHAIN=x86_64-7.3.0-posix-seh-rt_v5-rev0\mingw64"
)
)
- IF "%BUILD_SYSTEM%"=="bazel" (
appveyor DownloadFile https://github.com/bazelbuild/bazel/releases/download/2.2.0/bazel-2.2.0-windows-x86_64.exe -FileName bazel.exe
)
before_build:
- FOR /f %%i in ('C:\cygwin64\bin\date.exe +%%Y-%%m-%%d') DO SET "RELEASE_DATE=%%i"
- IF "%BUILD_SYSTEM%"=="CMake" ( mkdir builddir && cd builddir && cmake -G "%GENERATOR%" .. )
- IF "%BUILD_SYSTEM%"=="make" (
SET "CC=gcc" &&
SET "PATH=C:\mingw-w64\%TOOLCHAIN%\bin;%PATH%" &&
COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\%TOOLCHAIN%\bin\make.exe
)
- SET "ROOT=%APPVEYOR_BUILD_FOLDER%"
build_script:
- IF "%BUILD_SYSTEM%"=="CMake" ( cmake --build . --config %CONFIG% )
- IF "%BUILD_SYSTEM%"=="Python" ( python setup.py build_ext )
- IF "%BUILD_SYSTEM%"=="make" (
sh -c "make brotli" &&
cd bin && 7z a -tzip -mx9 brotli-win-%ARCH%-%RELEASE_DATE%.zip brotli.exe &&
appveyor PushArtifact brotli-win-%ARCH%-%RELEASE_DATE%.zip && cd ..
)
- IF "%BUILD_SYSTEM%"=="bazel" (
cd java &&
%ROOT%\bazel.exe --batch build -c opt org/brotli/wrapper/...:all &&
python %ROOT%\scripts\fix-win-bazel-build.py &&
cd bazel-bin &&
7z a -tzip -mx9 brotli-win-bazel-jni-%RELEASE_DATE%.zip brotli_jni.dll &&
appveyor PushArtifact brotli-win-bazel-jni-%RELEASE_DATE%.zip &&
cd .. &&
cd ..
)
test_script:
- IF "%BUILD_SYSTEM%"=="CMake" ( ctest --output-on-failure --interactive-debug-mode 0 -C %CONFIG% )
- IF "%BUILD_SYSTEM%"=="Python" ( python setup.py test )
- IF "%BUILD_SYSTEM%"=="make" ( sh -c "make test" )
- IF "%BUILD_SYSTEM%"=="bazel" (
cd java &&
%ROOT%\bazel.exe --batch test -c opt --test_output streamed org/brotli/wrapper/...:all &&
cd ..
)
deploy:
- provider: BinTray
on:
branch: master
username: eustas
api_key:
secure: B3rM8JYeIHIw6BfOjHggAeyTmxEf8ZDYmyF9gCwqyWaxyxECD9BuN50SiV2KE/RI
subject: eustas
repo: brotli
package: bin
version: latest
publish: true
override: true

View File

@ -5,7 +5,7 @@
import urllib2
response = urllib2.urlopen('https://tools.ietf.org/rfc/rfc7932.txt')
response = urllib2.urlopen("https://tools.ietf.org/rfc/rfc7932.txt")
text = response.read()
path = "rfc7932.txt"

View File

@ -19,7 +19,7 @@ for line in lines:
if re_data_line.match(line) is not None:
data = line.strip()
for i in range(32):
dictionary.append(int(data[2 * i : 2 * i + 2], 16))
dictionary.append(int(data[2 * i:2 * i + 2], 16))
if len(dictionary) == 122784:
break
else:

View File

@ -10,6 +10,7 @@ bin_path = "dictionary.bin"
with open(bin_path, "rb") as raw:
data = raw.read()
def check_digest(name, expected, actual):
if expected == actual:
print("[OK] " + name)
@ -22,15 +23,11 @@ check_digest(
"0x5136cb04",
hex(zlib.crc32(data)))
check_digest(
"MD5",
"96cecd2ee7a666d5aa3627d74735b32a",
hashlib.md5(data).hexdigest())
check_digest("MD5", "96cecd2ee7a666d5aa3627d74735b32a",
hashlib.md5(data).hexdigest())
check_digest(
"SHA1",
"72b41051cb61a9281ba3c4414c289da50d9a7640",
hashlib.sha1(data).hexdigest())
check_digest("SHA1", "72b41051cb61a9281ba3c4414c289da50d9a7640",
hashlib.sha1(data).hexdigest())
check_digest(
"SHA256",

View File

@ -1,6 +1,6 @@
# Step 04 - generate Java literals.
#
# Java byte-code has ridiculous restrictions. There is no such thing as
# Java byte-code has severe restrictions. There is no such thing as
# "array literal" - those are implemented as series of data[x] = y;
# as a consequence N-byte array will use 7N bytes in class, plus N bytes
# in instantiated variable. Also no literal could be longer than 64KiB.
@ -17,7 +17,7 @@
try:
unichr # Python 2
except NameError
except NameError:
unichr = chr # Python 3
bin_path = "dictionary.bin"
@ -49,8 +49,9 @@ for b in data:
cntr = skip_flip_offset + 1
hi.append(unichr(cntr))
low0 = low[0 : len(low) // 2]
low1 = low[len(low) // 2 : len(low)]
low0 = low[0:len(low) // 2]
low1 = low[len(low) // 2:len(low)]
def escape(chars):
result = []
@ -68,7 +69,7 @@ def escape(chars):
elif ord(c) < 32 or ord(c) >= 127:
result.append("\\u%04X" % ord(c))
else:
result.append(c);
result.append(c)
return result

View File

@ -1,36 +0,0 @@
import fnmatch
import os
import os.path
from shutil import copyfile
print('Searching for manifests...')
matches = []
for root, dirnames, filenames in os.walk('bazel-bin\\org\\brotli'):
for filename in fnmatch.filter(filenames, '*.runfiles_manifest'):
matches.append(os.path.join(root, filename))
for match in matches:
print('Scanning manifest ' + match)
runfiles = match[:-len('_manifest')]
with open(match) as manifest:
for entry in manifest:
entry = entry.strip()
if not entry.startswith("org_brotli_java"):
continue
if entry.startswith('org_brotli_java/external'):
continue
(alias, space, link) = entry.partition(' ')
if alias.endswith('.jar') or alias.endswith('.exe'):
continue
link = link.replace('/', '\\')
alias = alias.replace('/', '\\')
dst = os.path.join(runfiles, alias)
if not os.path.exists(dst):
print(link + ' -> ' + dst)
parent = os.path.dirname(dst)
if not os.path.exists(parent):
os.makedirs(parent)
copyfile(link, dst)
print('Finished resolving symlinks')

View File

@ -1,5 +1,6 @@
# IT WOULD BE FOOLISH TO USE COMPUTERS TO AUTOMATE REPETITIVE TASKS:
# ENLIST EVERY USED HEADER AND SOURCE FILE MANUALLY!
# neither CMake nor Automake support "glob" expressions,
# so every header and source file have to be listed manually.
BROTLI_CLI_C = \
c/tools/brotli.c