AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

faster inflate() autodetection of zlib/zstd

Browse Source
This commit is contained in:
inikep 2016-09-23 14:59:46 +02:00
parent b88accfb5f
commit 57b9708054
3 changed files with 172 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
.travis.yml
View File

@ -3,9 +3,63 @@ compiler: gcc
matrix:
fast_finish: true
include:
# OS X Mavericks
- os: osx
env: PLATFORM="OS X Mavericks" CMD="make gnu90test && make clean && make test && make clean && make travis-install"
# Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" CMD="make test && make clean && make travis-install"
- os: linux
sudo: false
language: cpp
install:
- export CXX="g++-4.8" CC="gcc-4.8"
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- gcc-4.8
- g++-4.8
env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper && make clean && make -C tests test-zstd_nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd pzstd && make -C contrib/pzstd googletest && make -C contrib/pzstd test && make -C contrib/pzstd clean"
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" CMD="make usan"
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" CMD="make asan"
# Standard Ubuntu 12.04 LTS Server Edition 64 bit
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" CMD='make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest'
env: PLATFORM="Ubuntu 12.04" CMD="make armtest"
addons:
apt:
packages:
- gcc-arm-linux-gnueabi
- libc6-dev-armel-cross
- linux-libc-dev-armel-cross
- binfmt-support
- qemu
- qemu-user-static
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" CMD="make -C tests versionsTest"
- os: linux
sudo: required
env: PLATFORM="Ubuntu 12.04" CMD="make asan32"
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- libc6-dev-i386
- gcc-multilib
# Ubuntu 14.04 LTS Server Edition 64 bit
- os: linux
dist: trusty
sudo: required
env: PLATFORM="Ubuntu 14.04" CMD="make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest"
addons:
apt:
packages:
@ -13,7 +67,25 @@ matrix:
- os: linux
dist: trusty
sudo: required
env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper"
env: PLATFORM="Ubuntu 14.04" CMD="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest"
addons:
apt:
packages:
- libc6-dev-i386
- g++-multilib
- os: linux
dist: trusty
sudo: required
env: PLATFORM="Ubuntu 14.04" CMD="make -C tests test32"
addons:
apt:
packages:
- libc6-dev-i386
- gcc-multilib
- os: linux
dist: trusty
sudo: required
env: PLATFORM="Ubuntu 14.04" CMD="make gcc5test && make clean && make gcc6test && sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu && make clean && make ppctest"
addons:
apt:
sources:

15
zlibWrapper/Makefile
View File

@ -1,8 +1,8 @@
# Makefile for example of using zstd wrapper for zlib
#
# make - compiles statically and dynamically linked examples
# make LOC=-DZWRAP_USE_ZSTD=1 - compiles statically and dynamically linked examples with zstd compression turned on
# make test test_d - runs statically and dynamically linked examples
# make - compiles examples
# make LOC=-DZWRAP_USE_ZSTD=1 - compiles examples with zstd compression turned on
# make test - runs examples
# Paths to static and dynamic zlib and zstd libraries
@ -22,7 +22,7 @@ LDFLAGS = $(LOC)
RM = rm -f
all: clean fitblk example example_d zwrapbench
all: clean fitblk example zwrapbench
test: example fitblk example_zstd fitblk_zstd zwrapbench
./example
@ -34,11 +34,8 @@ test: example fitblk example_zstd fitblk_zstd zwrapbench
./zwrapbench -qb1e5 ../zstd_compression_format.md
./zwrapbench -qb1e5B1K ../zstd_compression_format.md
test_d: example_d
./example_d
#valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so
valgrindTest: VALGRIND = LD_LIBRARY_PATH=$(ZSTDLIBDIR) valgrind --track-origins=yes --leak-check=full --error-exitcode=1
valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so
valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench
@echo "\n ---- valgrind tests ----"
$(VALGRIND) ./example
@ -83,5 +80,5 @@ $(ZSTDLIBDIR)/libzstd.so:
$(MAKE) -C $(ZSTDLIBDIR) all
clean:
-$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_d example_zstd fitblk fitblk_zstd zwrapbench
-$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_zstd fitblk fitblk_zstd zwrapbench
@echo Cleaning completed

159
zlibWrapper/zstd_zlibwrapper.c
View File

@ -548,6 +548,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm,
if (zwd == NULL || zwd->zbd == NULL) return Z_STREAM_ERROR;
errorCode = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength);
if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0);
zwd->decompState = Z_NEED_DICT;
if (strm->total_in == ZSTD_HEADERSIZE) {
zwd->inBuffer.src = zwd->headerBuf;
@ -571,6 +572,7 @@ ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm,
ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
{
ZWRAP_DCtx* zwd;
int res;
if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) {
LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
@ -581,60 +583,79 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
if (strm->avail_in > 0) {
size_t errorCode, srcSize;
ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
if (zwd == NULL) return Z_STREAM_ERROR;
zwd = (ZWRAP_DCtx*) strm->state;
LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
if (zwd == NULL) return Z_STREAM_ERROR;
if (zwd->decompState == Z_STREAM_END) return Z_STREAM_END;
if (strm->total_in < ZLIB_HEADERSIZE)
{
srcSize = MIN(strm->avail_in, ZLIB_HEADERSIZE - strm->total_in);
memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
strm->total_in += srcSize;
strm->next_in += srcSize;
strm->avail_in -= srcSize;
if (strm->total_in < ZLIB_HEADERSIZE) return Z_OK;
if (strm->total_in < ZLIB_HEADERSIZE) {
if (strm->total_in == 0 && strm->avail_in >= ZLIB_HEADERSIZE) {
if (MEM_readLE32(strm->next_in) != ZSTD_MAGICNUMBER) {
if (zwd->windowBits)
errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size);
else
errorCode = inflateInit_(strm, zwd->version, zwd->stream_size);
if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) {
z_stream strm2;
strm2.next_in = strm->next_in;
strm2.avail_in = strm->avail_in;
strm2.next_out = strm->next_out;
strm2.avail_out = strm->avail_out;
strm->reserved = 0; /* mark as zlib stream */
errorCode = ZWRAP_freeDCtx(zwd);
if (ZSTD_isError(errorCode)) goto error;
if (zwd->windowBits)
errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size);
else
errorCode = inflateInit_(strm, zwd->version, zwd->stream_size);
LOG_WRAPPERD("ZLIB inflateInit errorCode=%d\n", (int)errorCode);
if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode);
if (flush == Z_INFLATE_SYNC) res = inflateSync(strm);
else res = inflate(strm, flush);
LOG_WRAPPERD("- inflate3 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
return res;
}
} else {
srcSize = MIN(strm->avail_in, ZLIB_HEADERSIZE - strm->total_in);
memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
strm->total_in += srcSize;
strm->next_in += srcSize;
strm->avail_in -= srcSize;
if (strm->total_in < ZLIB_HEADERSIZE) return Z_OK;
/* inflate header */
strm->next_in = (unsigned char*)zwd->headerBuf;
strm->avail_in = ZLIB_HEADERSIZE;
strm->avail_out = 0;
errorCode = inflate(strm, Z_NO_FLUSH);
LOG_WRAPPERD("ZLIB inflate errorCode=%d strm->avail_in=%d\n", (int)errorCode, (int)strm->avail_in);
if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode);
if (strm->avail_in > 0) goto error;
if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) {
z_stream strm2;
strm2.next_in = strm->next_in;
strm2.avail_in = strm->avail_in;
strm2.next_out = strm->next_out;
strm2.avail_out = strm->avail_out;
strm->next_in = strm2.next_in;
strm->avail_in = strm2.avail_in;
strm->next_out = strm2.next_out;
strm->avail_out = strm2.avail_out;
if (zwd->windowBits)
errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size);
else
errorCode = inflateInit_(strm, zwd->version, zwd->stream_size);
LOG_WRAPPERD("ZLIB inflateInit errorCode=%d\n", (int)errorCode);
if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode);
strm->reserved = 0; /* mark as zlib stream */
errorCode = ZWRAP_freeDCtx(zwd);
if (ZSTD_isError(errorCode)) goto error;
/* inflate header */
strm->next_in = (unsigned char*)zwd->headerBuf;
strm->avail_in = ZLIB_HEADERSIZE;
strm->avail_out = 0;
errorCode = inflate(strm, Z_NO_FLUSH);
LOG_WRAPPERD("ZLIB inflate errorCode=%d strm->avail_in=%d\n", (int)errorCode, (int)strm->avail_in);
if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode);
if (strm->avail_in > 0) goto error;
if (flush == Z_INFLATE_SYNC) res = inflateSync(strm);
else res = inflate(strm, flush);
LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
return res;
strm->next_in = strm2.next_in;
strm->avail_in = strm2.avail_in;
strm->next_out = strm2.next_out;
strm->avail_out = strm2.avail_out;
strm->reserved = 0; /* mark as zlib stream */
errorCode = ZWRAP_freeDCtx(zwd);
if (ZSTD_isError(errorCode)) goto error;
if (flush == Z_INFLATE_SYNC) res = inflateSync(strm);
else res = inflate(strm, flush);
LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
return res;
}
}
}
if (flush == Z_INFLATE_SYNC) { strm->msg = "inflateSync is not supported!"; goto error; }
if (!zwd->zbd) {
zwd->zbd = ZSTD_createDStream_advanced(zwd->customMem);
if (zwd->zbd == NULL) { LOG_WRAPPERD("ERROR: ZSTD_createDStream_advanced\n"); goto error; }
@ -642,31 +663,36 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
if (strm->total_in < ZSTD_HEADERSIZE)
{
srcSize = MIN(strm->avail_in, ZSTD_HEADERSIZE - strm->total_in);
memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
strm->total_in += srcSize;
strm->next_in += srcSize;
strm->avail_in -= srcSize;
if (strm->total_in < ZSTD_HEADERSIZE) return Z_OK;
if (strm->total_in == 0 && strm->avail_in >= ZSTD_HEADERSIZE) {
if (zwd->decompState != Z_NEED_DICT) {
errorCode = ZSTD_initDStream(zwd->zbd);
if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; }
}
} else {
srcSize = MIN(strm->avail_in, ZSTD_HEADERSIZE - strm->total_in);
memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
strm->total_in += srcSize;
strm->next_in += srcSize;
strm->avail_in -= srcSize;
if (strm->total_in < ZSTD_HEADERSIZE) return Z_OK;
errorCode = ZSTD_initDStream(zwd->zbd);
if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; }
errorCode = ZSTD_initDStream(zwd->zbd);
if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; }
if (flush == Z_INFLATE_SYNC) { strm->msg = "inflateSync is not supported!"; goto error; }
zwd->inBuffer.src = zwd->headerBuf;
zwd->inBuffer.size = ZSTD_HEADERSIZE;
zwd->inBuffer.pos = 0;
zwd->outBuffer.dst = strm->next_out;
zwd->outBuffer.size = 0;
zwd->outBuffer.pos = 0;
errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer);
LOG_WRAPPERD("inflate ZSTD_decompressStream1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size);
if (ZSTD_isError(errorCode)) {
LOG_WRAPPERD("ERROR: ZSTD_decompressStream1 %s\n", ZSTD_getErrorName(errorCode));
goto error;
zwd->inBuffer.src = zwd->headerBuf;
zwd->inBuffer.size = ZSTD_HEADERSIZE;
zwd->inBuffer.pos = 0;
zwd->outBuffer.dst = strm->next_out;
zwd->outBuffer.size = 0;
zwd->outBuffer.pos = 0;
errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer);
LOG_WRAPPERD("inflate ZSTD_decompressStream1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size);
if (ZSTD_isError(errorCode)) {
LOG_WRAPPERD("ERROR: ZSTD_decompressStream1 %s\n", ZSTD_getErrorName(errorCode));
goto error;
}
if (zwd->inBuffer.pos != zwd->inBuffer.size) return ZWRAPD_finishWithError(zwd, strm, 0); /* not consumed */
}
if (zwd->inBuffer.pos != zwd->inBuffer.size) return ZWRAPD_finishWithError(zwd, strm, 0); /* not consumed */
}
zwd->inBuffer.src = strm->next_in;
@ -694,13 +720,12 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
zwd->decompState = Z_STREAM_END;
return Z_STREAM_END;
}
goto finish;
error:
return ZWRAPD_finishWithError(zwd, strm, 0);
}
finish:
LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, Z_OK);
return Z_OK;
error:
return ZWRAPD_finishWithError(zwd, strm, 0);
}