AuroraMiddleware/lz4
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'dev' into partialDecode

Browse Source
This commit is contained in:
Yann Collet 2018-09-10 15:44:40 -07:00
commit d28389b2dc
9 changed files with 156 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
.circleci/config.yml Normal file
View File

@ -0,0 +1,108 @@
# This configuration was automatically generated from a CircleCI 1.0 config.
# It should include any build commands you had along with commands that CircleCI
# inferred from your project structure. We strongly recommend you read all the
# comments in this file to understand the structure of CircleCI 2.0, as the idiom
# for configuration has changed substantially in 2.0 to allow arbitrary jobs rather
# than the prescribed lifecycle of 1.0. In general, we recommend using this generated
# configuration as a reference rather than using it in production, though in most
# cases it should duplicate the execution of your original 1.0 config.
version: 2
jobs:
build:
working_directory: ~/lz4/lz4
parallelism: 1
shell: /bin/bash --login
# CircleCI 2.0 does not support environment variables that refer to each other the same way as 1.0 did.
# If any of these refer to each other, rewrite them so that they don't or see https://circleci.com/docs/2.0/env-vars/#interpolating-environment-variables-to-set-other-environment-variables .
environment:
CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
# In CircleCI 1.0 we used a pre-configured image with a large number of languages and other packages.
# In CircleCI 2.0 you can now specify your own image, or use one of our pre-configured images.
# The following configuration line tells CircleCI to use the specified docker image as the runtime environment for you job.
# We have selected a pre-built image that mirrors the build environment we use on
# the 1.0 platform, but we recommend you choose an image more tailored to the needs
# of each job. For more information on choosing an image (or alternatively using a
# VM instead of a container) see https://circleci.com/docs/2.0/executor-types/
# To see the list of pre-built images that CircleCI provides for most common languages see
# https://circleci.com/docs/2.0/circleci-images/
docker:
- image: circleci/build-image:ubuntu-14.04-XXL-upstart-1189-5614f37
command: /sbin/init
steps:
# Machine Setup
# If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
# The following `checkout` command checks out your code to your working directory. In 1.0 we did this implicitly. In 2.0 you can choose where in the course of a job your code should be checked out.
- checkout
# Prepare for artifact and test results collection equivalent to how it was done on 1.0.
# In many cases you can simplify this from what is generated here.
# 'See docs on artifact collection here https://circleci.com/docs/2.0/artifacts/'
- run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS
# Dependencies
# This would typically go in either a build or a build-and-test job when using workflows
# Restore the dependency cache
- restore_cache:
keys:
# This branch if available
- v1-dep-{{ .Branch }}-
# Default branch if not
- v1-dep-dev-
# Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly
- v1-dep-
# This is based on your 1.0 configuration file or project settings
- run: sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update
- run: sudo apt-get -y install qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu
- run: sudo apt-get -y install qemu-system-arm gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
- run: sudo apt-get -y install libc6-dev-i386 clang gcc-5 gcc-5-multilib gcc-6 valgrind
# Save dependency cache
- save_cache:
key: v1-dep-{{ .Branch }}-{{ epoch }}
paths:
# This is a broad list of cache paths to include many possible development environments
# You can probably delete some of these entries
- vendor/bundle
- ~/virtualenvs
- ~/.m2
- ~/.ivy2
- ~/.bundle
- ~/.go_workspace
- ~/.gradle
- ~/.cache/bower
# Test
# This would typically be a build job when using workflows, possibly combined with build
# This is based on your 1.0 configuration file or project settings
- run: clang -v; make clangtest && make clean
- run: g++ -v; make gpptest && make clean
- run: gcc -v; make c_standards && make clean
- run: gcc -v; g++ -v; make ctocpptest && make clean
- run: gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -Werror" make check && make clean
- run: gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean
- run: gcc-6 -v; CC=gcc-6 make c_standards && make clean
- run: gcc-6 -v; CC=gcc-6 MOREFLAGS="-O2 -Werror" make check && make clean
- run: make cmake && make clean
- run: make -C tests test-lz4
- run: make -C tests test-lz4c
- run: make -C tests test-frametest
- run: make -C tests test-fullbench
- run: make -C tests test-fuzzer && make clean
- run: make -C lib all && make clean
- run: pyenv global 3.4.4; make versionsTest MOREFLAGS=-I/usr/include/x86_64-linux-gnu && make clean
- run: make travis-install && make clean
- run: gcc -v; CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean
- run: make usan && make clean
- run: clang -v; make staticAnalyze && make clean
- run: make -C tests test-mem && make clean
- run: make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static && make clean
- run: make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS=-m64 && make clean
- run: make platformTest CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static && make clean
- run: make platformTest CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static && make clean
# Teardown
# If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
# Save test results
- store_test_results:
path: /tmp/circleci-test-results
# Save artifacts
- store_artifacts:
path: /tmp/circleci-artifacts
- store_artifacts:
path: /tmp/circleci-test-results

2
Makefile
View File

@ -89,7 +89,7 @@ clean:
#-----------------------------------------------------------------------------
# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku MidnightBSD))
HOST_OS = POSIX
.PHONY: install uninstall

39
circle.yml
View File

@ -1,39 +0,0 @@
dependencies:
override:
- sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update
- sudo apt-get -y install qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu
- sudo apt-get -y install qemu-system-arm gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
- sudo apt-get -y install libc6-dev-i386 clang gcc-5 gcc-5-multilib gcc-6 valgrind
test:
override:
# Tests compilers and C standards
- clang -v; make clangtest && make clean
- g++ -v; make gpptest && make clean
- gcc -v; make c_standards && make clean
- gcc -v; g++ -v; make ctocpptest && make clean
- gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -Werror" make check && make clean
- gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean
- gcc-6 -v; CC=gcc-6 make c_standards && make clean
- gcc-6 -v; CC=gcc-6 MOREFLAGS="-O2 -Werror" make check && make clean
# Shorter tests
- make cmake && make clean
- make -C tests test-lz4
- make -C tests test-lz4c
- make -C tests test-frametest
- make -C tests test-fullbench
- make -C tests test-fuzzer && make clean
- make -C lib all && make clean
- pyenv global 3.4.4; make versionsTest MOREFLAGS=-I/usr/include/x86_64-linux-gnu && make clean
- make travis-install && make clean
# Longer tests
- gcc -v; CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean
- make usan && make clean
- clang -v; make staticAnalyze && make clean
# Valgrind tests
- make -C tests test-mem && make clean
# ARM, AArch64, PowerPC, PowerPC64 tests
- make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static && make clean
- make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS=-m64 && make clean
- make platformTest CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static && make clean
- make platformTest CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static && make clean

36
examples/frameCompress.c
View File

@ -31,12 +31,13 @@ static const LZ4F_preferences_t kPrefs = {
static void safe_fwrite(void* buf, size_t eltSize, size_t nbElt, FILE* f)
{
size_t const writtenSize = fwrite(buf, eltSize, nbElt, f);
size_t const expectedSize = eltSize * nbElt; /* note : should check for overflow */
size_t const expectedSize = eltSize * nbElt;
assert(expectedSize / nbElt == eltSize); /* check overflow */
if (writtenSize < expectedSize) {
if (ferror(f)) /* note : ferror() must follow fwrite */
printf("Write failed\n");
fprintf(stderr, "Write failed \n");
else
printf("Short write\n");
fprintf(stderr, "Short write \n");
exit(1);
}
}
@ -54,9 +55,9 @@ typedef struct {
static compressResult_t
compress_file_internal(FILE* f_in, FILE* f_out,
LZ4F_compressionContext_t ctx,
void* inBuff, size_t inChunkSize,
void* outBuff, size_t outCapacity)
LZ4F_compressionContext_t ctx,
void* inBuff, size_t inChunkSize,
void* outBuff, size_t outCapacity)
{
compressResult_t result = { 1, 0, 0 }; /* result for an error */
unsigned long long count_in = 0, count_out;
@ -167,9 +168,9 @@ static size_t get_block_size(const LZ4F_frameInfo_t* info) {
/* @return : 1==error, 0==success */
static int
decompress_file_internal(FILE* f_in, FILE* f_out,
LZ4F_dctx* dctx,
void* src, size_t srcCapacity, size_t filled, size_t alreadyConsumed,
void* dst, size_t dstCapacity)
LZ4F_dctx* dctx,
void* src, size_t srcCapacity, size_t filled, size_t alreadyConsumed,
void* dst, size_t dstCapacity)
{
int firstChunk = 1;
size_t ret = 1;
@ -194,7 +195,7 @@ decompress_file_internal(FILE* f_in, FILE* f_out,
* Continue while there is more input to read (srcPtr != srcEnd)
* and the frame isn't over (ret != 0)
*/
while (srcPtr != srcEnd && ret != 0) {
while (srcPtr < srcEnd && ret != 0) {
/* Any data within dst has been flushed at this stage */
size_t dstSize = dstCapacity;
size_t srcSize = srcEnd - srcPtr;
@ -208,9 +209,20 @@ decompress_file_internal(FILE* f_in, FILE* f_out,
/* Update input */
srcPtr += srcSize;
}
assert(srcPtr <= srcEnd);
/* Ensure all input data has been consumed.
* It is valid to have multiple frames in the same file,
* but this example only supports one frame.
*/
if (srcPtr < srcEnd) {
printf("Decompress: Trailing data left in file after frame\n");
return 1;
}
}
/* Check that there isn't trailing input data after the frame.
/* Check that there isn't trailing data in the file after the frame.
* It is valid to have multiple frames in the same file,
* but this example only supports one frame.
*/
@ -278,7 +290,7 @@ static int decompress_file(FILE* f_in, FILE* f_out)
if (!src) { perror("decompress_file(src)"); return 1; }
LZ4F_dctx* dctx;
{ size_t const dctxStatus = LZ4F_createDecompressionContext(&dctx, 100);
{ size_t const dctxStatus = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
if (LZ4F_isError(dctxStatus)) {
printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(dctxStatus));
} }

2
lib/Makefile
View File

@ -124,7 +124,7 @@ clean:
#-----------------------------------------------------------------------------
# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku MidnightBSD))
.PHONY: listL120
listL120: # extract lines >= 120 characters in *.{c,h}, by Takayuki Matsuoka (note : $$, for Makefile compatibility)

29
lib/lz4frame.h
View File

@ -33,9 +33,10 @@
*/
/* LZ4F is a stand-alone API to create LZ4-compressed frames
* conformant with specification v1.5.1.
* conformant with specification v1.6.1.
* It also offers streaming capabilities.
* lz4.h is not required when using lz4frame.h.
* lz4.h is not required when using lz4frame.h,
* except to get constant such as LZ4_VERSION_NUMBER.
* */
#ifndef LZ4F_H_09782039843
@ -159,8 +160,9 @@ typedef LZ4F_contentChecksum_t contentChecksum_t;
/*! LZ4F_frameInfo_t :
* makes it possible to set or read frame parameters.
* It's not required to set all fields, as long as the structure was initially memset() to zero.
* For all fields, 0 sets it to default value */
* Structure must be first init to 0, using memset() or LZ4F_INIT_FRAMEINFO,
* setting all parameters to default.
* It's then possible to update selectively some parameters */
typedef struct {
LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB; 0 == default */
LZ4F_blockMode_t blockMode; /* LZ4F_blockLinked, LZ4F_blockIndependent; 0 == default */
@ -171,24 +173,30 @@ typedef struct {
LZ4F_blockChecksum_t blockChecksumFlag; /* 1: each block followed by a checksum of block's compressed data; 0: disabled (default) */
} LZ4F_frameInfo_t;
#define LZ4F_INIT_FRAMEINFO { 0, 0, 0, 0, 0, 0, 0 } /* v1.8.3+ */
/*! LZ4F_preferences_t :
* makes it possible to supply detailed compression parameters to the stream interface.
* Structure is presumed initially memset() to zero, representing default settings.
* makes it possible to supply advanced compression instructions to streaming interface.
* Structure must be first init to 0, using memset() or LZ4F_INIT_PREFERENCES,
* setting all parameters to default.
* All reserved fields must be set to zero. */
typedef struct {
LZ4F_frameInfo_t frameInfo;
int compressionLevel; /* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */
unsigned autoFlush; /* 1: always flush, to reduce usage of internal buffers */
unsigned favorDecSpeed; /* 1: parser favors decompression speed vs compression ratio. Only works for high compression modes (>= LZ4LZ4HC_CLEVEL_OPT_MIN) */ /* >= v1.8.2 */
unsigned autoFlush; /* 1: always flush; reduces usage of internal buffers */
unsigned favorDecSpeed; /* 1: parser favors decompression speed vs compression ratio. Only works for high compression modes (>= LZ4HC_CLEVEL_OPT_MIN) */ /* v1.8.2+ */
unsigned reserved[3]; /* must be zero for forward compatibility */
} LZ4F_preferences_t;
LZ4FLIB_API int LZ4F_compressionLevel_max(void);
#define LZ4F_INIT_PREFERENCES { LZ4F_INIT_FRAMEINFO, 0, 0, 0, { 0, 0, 0 } } /* v1.8.3+ */
/*-*********************************
* Simple compression function
***********************************/
LZ4FLIB_API int LZ4F_compressionLevel_max(void);
/*! LZ4F_compressFrameBound() :
* Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences.
* `preferencesPtr` is optional. It can be replaced by NULL, in which case, the function will assume default preferences.
@ -222,8 +230,9 @@ typedef struct {
/*--- Resource Management ---*/
#define LZ4F_VERSION 100
#define LZ4F_VERSION 100 /* This number can be used to check for an incompatible API breaking change */
LZ4FLIB_API unsigned LZ4F_getVersion(void);
/*! LZ4F_createCompressionContext() :
* The first thing to do is to create a compressionContext object, which will be used in all compression operations.
* This is achieved using LZ4F_createCompressionContext(), which takes as argument a version.

2
programs/Makefile
View File

@ -109,7 +109,7 @@ clean:
#-----------------------------------------------------------------------------
# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku MidnightBSD))
unlz4: lz4
ln -s lz4$(EXT) unlz4$(EXT)

2
programs/platform.h
View File

@ -81,7 +81,7 @@ extern "C" {
#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \
|| defined(__midipix__) || defined(__VMS))
# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1–2001 (SUSv3) conformant */ \
|| defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* BSD distros */
|| defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__MidnightBSD__) /* BSD distros */
# define PLATFORM_POSIX_VERSION 200112L
# else
# if defined(__linux__) || defined(__linux)

2
tests/Makefile
View File

@ -133,7 +133,7 @@ checkTag: checkTag.c $(LZ4DIR)/lz4.h
#-----------------------------------------------------------------------------
# validated only for Linux, OSX, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku MidnightBSD))
MD5:=md5sum
ifneq (,$(filter $(shell uname), Darwin ))