AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11

Browse Source
This commit is contained in:
Przemyslaw Skibinski 2017-02-23 09:20:25 +01:00
commit 83b457b851
14 changed files with 827 additions and 763 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

140
.travis.yml
View File

@ -1,146 +1,34 @@
language: c
sudo: required
dist: trusty
matrix:
fast_finish: true
include:
# OS X Mavericks
- env: Ubu=OS_X_Mavericks Cmd="make gnu90test && make clean && make test && make clean && make travis-install"
- env: Cmd="make gnu90test && make clean && make test && make clean && make travis-install"
os: osx
# Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
- env: Ubu=12.04cont Cmd="make usan"
os: linux
sudo: false
- env: Ubu=12.04cont Cmd="make asan"
os: linux
sudo: false
- env: Ubu=14.04 Cmd='make -C tests test32 CC=clang-4.0 MOREFLAGS="-g -fsanitize=address"'
os: linux
dist: trusty
sudo: required
addons:
apt:
sources:
- llvm-toolchain-trusty-4.0
packages:
- libc6-dev-i386
- gcc-multilib
- clang-4.0
# Standard Ubuntu 12.04 LTS Server Edition 64 bit
- env: Ubu=12.04 Cmd='cd contrib/pzstd && make googletest && make tsan && make check && make clean && make asan && make check && make clean && cd ../..'
os: linux
sudo: required
# Ubuntu 14.04 LTS Server Edition 64 bit
- env: Cmd='make gpp6install uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan'
install:
- export CXX="g++-6" CC="gcc-6"
- export LDFLAGS="-fuse-ld=gold"
- export TESTFLAGS='--gtest_filter=-*ExtremelyLarge*'
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-6
- g++-6
# Ubuntu 14.04 LTS Server Edition 64 bit
- env: Ubu=14.04 Cmd="make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean
&& make -C contrib/pzstd googletest32 && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean"
os: linux
dist: trusty
sudo: required
install:
- export CXX="g++-4.8" CC="gcc-4.8"
addons:
apt:
packages:
- libc6-dev-i386
- g++-multilib
- gcc-4.8
- gcc-4.8-multilib
- g++-4.8
- g++-4.8-multilib
- env: Ubu=14.04 Cmd="make armtest"
dist: trusty
sudo: required
addons:
apt:
packages:
- qemu-system-arm
- qemu-user-static
- gcc-arm-linux-gnueabi
- libc6-dev-armel-cross
- env: Ubu=14.04 Cmd="make aarch64test"
dist: trusty
sudo: required
addons:
apt:
packages:
- qemu-system-arm
- qemu-user-static
- gcc-aarch64-linux-gnu
- libc6-dev-arm64-cross
- env: Ubu=14.04 Cmd='make ppctest'
dist: trusty
sudo: required
addons:
apt:
packages:
- qemu-system-ppc
- qemu-user-static
- gcc-powerpc-linux-gnu
- env: Ubu=14.04 Cmd='make ppc64test'
dist: trusty
sudo: required
addons:
apt:
packages:
- qemu-system-ppc
- qemu-user-static
- gcc-powerpc-linux-gnu
- env: Cmd='CC=gcc-6 make gcc6install uasan-test32 && make clean zlibwrapper && make -C tests clean test-zstd-nolegacy && make -C tests versionsTest'
- env: Cmd="make arminstall armtest && make clean && make aarch64test"
- env: Cmd='make ppcinstall ppctest && make clean && make ppc64test'
# other feature branches => short tests
- env: Ubu=14.04 Cmd='make lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest'
os: linux
dist: trusty
sudo: required
addons:
apt:
packages:
- valgrind
- env: Ubu=14.04 Cmd="make zlibwrapper && make clean && make -C tests test-zstd-nolegacy && make clean && make -C tests test32 versionsTest"
os: linux
dist: trusty
sudo: required
addons:
apt:
packages:
- libc6-dev-i386
- gcc-multilib
- env: Cmd='make valgrindinstall arminstall ppcinstall arm-ppc-compilation && make clean lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest'
- env: Cmd="make libc6install && make -C tests test32"
script:
- JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:')
# cron & master => long tests, as this is the final step towards a Release
# dev => normal tests
# other feature branches => short tests (number > 10)
# dev && pull requests => normal tests
# other feature branches => short tests (number > 5)
- if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "master" ]; then
FUZZERTEST=-T10mn sh -c "$Cmd" || travis_terminate 1;
FUZZERTEST=-T7mn sh -c "$Cmd" || travis_terminate 1;
else
if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 10 ] || [ "$TRAVIS_BRANCH" = "dev" ]; then
if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 5 ] || [ "$TRAVIS_BRANCH" = "dev" ]; then
sh -c "$Cmd" || travis_terminate 1;
fi
fi

34
Makefile
View File

@ -128,6 +128,12 @@ ppc64test: clean
$(MAKE) -C $(TESTDIR) datagen # use native, faster
$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static"
arm-ppc-compilation:
$(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
$(MAKE) -C $(PRGDIR) clean zstd CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static"
$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static"
usan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"
@ -143,6 +149,34 @@ asan32: clean
uasan: clean
$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined"
uasan-%: clean
LDFLAGS=-fuse-ld=gold CFLAGS="-Og -fsanitize=address -fsanitize=undefined" $(MAKE) -C $(TESTDIR) $*
apt-install:
sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES)
apt-add-repo:
sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
sudo apt-get update -y -qq
ppcinstall:
APT_PACKAGES="qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu" $(MAKE) apt-install
arminstall:
APT_PACKAGES="qemu-system-arm qemu-user-static gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross" $(MAKE) apt-install
valgrindinstall:
APT_PACKAGES="valgrind" $(MAKE) apt-install
libc6install:
APT_PACKAGES="libc6-dev-i386 gcc-multilib" $(MAKE) apt-install
gcc6install: apt-add-repo
APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-6 gcc-6-multilib" $(MAKE) apt-install
gpp6install: apt-add-repo
APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install
endif

19
contrib/pzstd/Makefile
View File

@ -85,6 +85,23 @@ endif
.PHONY: default
default: all
.PHONY: test-pzstd
test-pzstd: TESTFLAGS=--gtest_filter=-*ExtremelyLarge*
test-pzstd: clean googletest pzstd tests check
.PHONY: test-pzstd32
test-pzstd32: clean googletest32 all32 check
.PHONY: test-pzstd-tsan
test-pzstd-tsan: LDFLAGS=-fuse-ld=gold
test-pzstd-tsan: TESTFLAGS=--gtest_filter=-*ExtremelyLarge*
test-pzstd-tsan: clean googletest tsan check
.PHONY: test-pzstd-asan
test-pzstd-asan: LDFLAGS=-fuse-ld=gold
test-pzstd-asan: TESTFLAGS=--gtest_filter=-*ExtremelyLarge*
test-pzstd-asan: clean asan check
.PHONY: check
check:
$(TESTPROG) ./utils/test/BufferTest$(EXT) $(TESTFLAGS)
@ -117,7 +134,7 @@ debug: pzstd$(EXT) tests roundtrip
.PHONY: tsan
tsan: PZSTD_CCXXFLAGS += -fsanitize=thread -fPIC
tsan: PZSTD_LDFLAGS += -fsanitize=thread -pie
tsan: PZSTD_LDFLAGS += -fsanitize=thread
tsan: debug
.PHONY: asan

1319
doc/zstd_compression_format.md
View File

File diff suppressed because it is too large Load Diff

17
examples/Makefile
View File

@ -9,7 +9,7 @@
# This Makefile presumes libzstd is installed, using `sudo make install`
LDFLAGS+= -lzstd
LDFLAGS += -lzstd
.PHONY: default all clean test
@ -52,16 +52,23 @@ clean:
test: all
cp README.md tmp
cp Makefile tmp2
@echo starting simple compression
@echo -- Simple compression tests
./simple_compression tmp
./simple_decompression tmp.zst
./streaming_decompression tmp.zst > /dev/null
@echo starting streaming compression
@echo -- Streaming compression tests
./streaming_compression tmp
./streaming_decompression tmp.zst > /dev/null
@echo starting multiple streaming compression
@echo -- Edge cases detection
! ./streaming_decompression tmp # invalid input, must fail
! ./simple_decompression tmp # invalid input, must fail
! ./simple_decompression tmp.zst # unknown input size, must fail
touch tmpNull # create 0-size file
./simple_compression tmpNull
./simple_decompression tmpNull.zst # 0-size frame : must work
@echo -- Multiple streaming tests
./multiple_streaming_compression *.c
@echo starting dictionary compression
@echo -- Dictionary compression tests
./dictionary_compression tmp2 tmp README.md
./dictionary_decompression tmp2.zst tmp.zst README.md
$(RM) tmp* *.zst

6
examples/dictionary_decompression.c
View File

@ -78,10 +78,14 @@ static void decompress(const char* fname, const ZSTD_DDict* ddict)
size_t cSize;
void* const cBuff = loadFile_orDie(fname, &cSize);
unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
if (rSize==0) {
if (rSize==ZSTD_CONTENTSIZE_ERROR) {
fprintf(stderr, "%s : it was not compressed by zstd.\n", fname);
exit(5);
} else if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) {
fprintf(stderr, "%s : original size unknown \n", fname);
exit(6);
}
void* const rBuff = malloc_orDie((size_t)rSize);
ZSTD_DCtx* const dctx = ZSTD_createDCtx();

21
examples/simple_decompression.c
View File

@ -20,7 +20,7 @@ static off_t fsize_orDie(const char *filename)
struct stat st;
if (stat(filename, &st) == 0) return st.st_size;
/* error */
printf("stat: %s : %s \n", filename, strerror(errno));
fprintf(stderr, "stat: %s : %s \n", filename, strerror(errno));
exit(1);
}
@ -29,16 +29,16 @@ static FILE* fopen_orDie(const char *filename, const char *instruction)
FILE* const inFile = fopen(filename, instruction);
if (inFile) return inFile;
/* error */
printf("fopen: %s : %s \n", filename, strerror(errno));
fprintf(stderr, "fopen: %s : %s \n", filename, strerror(errno));
exit(2);
}
static void* malloc_orDie(size_t size)
{
void* const buff = malloc(size);
void* const buff = malloc(size + !size); /* avoid allocating size of 0 : may return NULL (implementation dependent) */
if (buff) return buff;
/* error */
printf("malloc: %s \n", strerror(errno));
fprintf(stderr, "malloc: %s \n", strerror(errno));
exit(3);
}
@ -49,7 +49,7 @@ static void* loadFile_orDie(const char* fileName, size_t* size)
void* const buffer = malloc_orDie(buffSize);
size_t const readSize = fread(buffer, 1, buffSize, inFile);
if (readSize != (size_t)buffSize) {
printf("fread: %s : %s \n", fileName, strerror(errno));
fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
exit(4);
}
fclose(inFile); /* can't fail (read only) */
@ -63,16 +63,21 @@ static void decompress(const char* fname)
size_t cSize;
void* const cBuff = loadFile_orDie(fname, &cSize);
unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
if (rSize==0) {
printf("%s : original size unknown. Use streaming decompression instead. \n", fname);
if (rSize==ZSTD_CONTENTSIZE_ERROR) {
fprintf(stderr, "%s : it was not compressed by zstd.\n", fname);
exit(5);
} else if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) {
fprintf(stderr,
"%s : original size unknown. Use streaming decompression instead.\n", fname);
exit(6);
}
void* const rBuff = malloc_orDie((size_t)rSize);
size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize);
if (dSize != rSize) {
printf("error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
exit(7);
}

15
lib/compress/fse_compress.c
View File

@ -506,6 +506,7 @@ unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue)
{
short const NOT_YET_ASSIGNED = -2;
U32 s;
U32 distributed = 0;
U32 ToDistribute;
@ -531,7 +532,8 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
total -= count[s];
continue;
}
norm[s]=-2;
norm[s]=NOT_YET_ASSIGNED;
}
ToDistribute = (1 << tableLog) - distributed;
@ -539,7 +541,7 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
/* risk of rounding to zero */
lowOne = (U32)((total * 3) / (ToDistribute * 2));
for (s=0; s<=maxSymbolValue; s++) {
if ((norm[s] == -2) && (count[s] <= lowOne)) {
if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) {
norm[s] = 1;
distributed++;
total -= count[s];
@ -559,12 +561,19 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
return 0;
}
if (total == 0) {
/* all of the symbols were low enough for the lowOne or lowThreshold */
for (s=0; ToDistribute > 0; s = (s+1)%(maxSymbolValue+1))
if (norm[s] > 0) ToDistribute--, norm[s]++;
return 0;
}
{ U64 const vStepLog = 62 - tableLog;
U64 const mid = (1ULL << (vStepLog-1)) - 1;
U64 const rStep = ((((U64)1<<vStepLog) * ToDistribute) + mid) / total; /* scale on remaining */
U64 tmpTotal = mid;
for (s=0; s<=maxSymbolValue; s++) {
if (norm[s]==-2) {
if (norm[s]==NOT_YET_ASSIGNED) {
U64 const end = tmpTotal + (count[s] * rStep);
U32 const sStart = (U32)(tmpTotal >> vStepLog);
U32 const sEnd = (U32)(end >> vStepLog);

4
lib/compress/zstd_compress.c
View File

@ -2786,7 +2786,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, u
if (!cdict || !cctx) {
ZSTD_free(cdict, customMem);
ZSTD_free(cctx, customMem);
ZSTD_freeCCtx(cctx);
return NULL;
}
@ -2804,8 +2804,8 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, u
{ size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0);
if (ZSTD_isError(errorCode)) {
ZSTD_free(cdict->dictBuffer, customMem);
ZSTD_free(cctx, customMem);
ZSTD_free(cdict, customMem);
ZSTD_freeCCtx(cctx);
return NULL;
} }

2
programs/Makefile
View File

@ -148,7 +148,7 @@ generate_res:
windres/generate_res.bat
clean:
$(MAKE) -C ../lib clean
$(MAKE) -C $(ZSTDDIR) clean
@$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda
@$(RM) core *.o tmp* result* *.gcda dictionary *.zst \
zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \

4
programs/zstdcli.c
View File

@ -399,7 +399,7 @@ int main(int argCount, const char* argv[])
while (argument[0]!=0) {
if (lastCommand) {
DISPLAY("error : command must be followed by argument \n");
return 1;
CLEAN_RETURN(1);
}
#ifndef ZSTD_NOCOMPRESS
/* compression Level */
@ -555,7 +555,7 @@ int main(int argCount, const char* argv[])
filenameTable[filenameIdx++] = argument;
}
if (lastCommand) { DISPLAY("error : command must be followed by argument \n"); return 1; } /* forgotten argument */
if (lastCommand) { DISPLAY("error : command must be followed by argument \n"); CLEAN_RETURN(1); } /* forgotten argument */
/* Welcome message (if verbose) */
DISPLAYLEVEL(3, WELCOME_MESSAGE);

4
tests/Makefile
View File

@ -170,11 +170,11 @@ namespaceTest:
if $(CC) namespaceTest.c ../lib/common/xxhash.c -o $@ ; then echo compilation should fail; exit 1 ; fi
$(RM) $@
versionsTest:
versionsTest: clean
$(PYTHON) test-zstd-versions.py
clean:
$(MAKE) -C ../lib clean
$(MAKE) -C $(ZSTDDIR) clean
@$(RM) -fR $(TESTARTEFACT)
@$(RM) -f core *.o tmp* result* *.gcda dictionary *.zst \
$(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \

3
tests/legacy.c
View File

@ -65,6 +65,7 @@ int testSimpleAPI(void)
return 1;
}
free(output);
DISPLAY("Simple API OK\n");
return 0;
}
@ -118,6 +119,8 @@ int testStreamingAPI(void)
}
}
free(outBuff);
ZSTD_freeDStream(stream);
DISPLAY("Streaming API OK\n");
return 0;
}

2
tests/zstreamtest.c
View File

@ -496,6 +496,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
/* Bug will cause checksum to fail */
if (ZSTD_isError(r)) goto _output_error;
}
ZSTD_freeDStream(zds);
}
DISPLAYLEVEL(3, "OK \n");