AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11

Browse Source
This commit is contained in:
Przemyslaw Skibinski 2016-12-06 11:11:20 +01:00
commit 23a1e11360
20 changed files with 235 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
NEWS
View File

@ -7,6 +7,9 @@ cli : fixed : status displays total amount decoded, even for file consisting of
cli : fixed : zstdcat
API : changed : zbuff prototypes now generate deprecation warnings
API : changed : streaming decompression implicit reset on starting new frame
API : added experimental : dictID retrieval functions
zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski
Changed : zbuff source files moved to lib/deprecated
Changed : reduced stack memory use
v1.1.1

12
build/VS2005/fullbench/fullbench.vcproj
View File

@ -363,14 +363,6 @@
RelativePath="..\..\..\lib\decompress\huf_decompress.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\zbuff_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\decompress\zbuff_decompress.c"
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_common.c"
>
@ -425,10 +417,6 @@
RelativePath="..\..\..\lib\common\mem.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\zbuff_static.h"
>
</File>
<File
RelativePath="..\..\..\lib\zstd.h"
>

4
build/VS2005/fuzzer/fuzzer.vcproj
View File

@ -429,10 +429,6 @@
RelativePath="..\..\..\lib\common\xxhash.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\zbuff_static.h"
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
>

16
build/VS2005/zstd/zstd.vcproj
View File

@ -379,14 +379,6 @@
RelativePath="..\..\..\lib\common\xxhash.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\zbuff_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\decompress\zbuff_decompress.c"
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\zdict.c"
>
@ -481,14 +473,6 @@
RelativePath="..\..\..\lib\common\xxhash.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\zbuff.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\zbuff_static.h"
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
>

8
build/VS2005/zstdlib/zstdlib.vcproj
View File

@ -360,11 +360,11 @@
>
</File>
<File
RelativePath="..\..\..\lib\compress\zbuff_compress.c"
RelativePath="..\..\..\lib\deprecated\zbuff_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\decompress\zbuff_decompress.c"
RelativePath="..\..\..\lib\deprecated\zbuff_decompress.c"
>
</File>
<File
@ -461,10 +461,6 @@
RelativePath="..\..\..\lib\common\zbuff.h"
>
</File>
<File
RelativePath="..\..\..\lib\common\zbuff_static.h"
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
>

14
build/cmake/lib/CMakeLists.txt
View File

@ -64,25 +64,25 @@ SET(Sources
${LIBRARY_DIR}/common/fse_decompress.c
${LIBRARY_DIR}/compress/fse_compress.c
${LIBRARY_DIR}/compress/huf_compress.c
${LIBRARY_DIR}/compress/zbuff_compress.c
${LIBRARY_DIR}/compress/zstd_compress.c
${LIBRARY_DIR}/decompress/huf_decompress.c
${LIBRARY_DIR}/decompress/zbuff_decompress.c
${LIBRARY_DIR}/decompress/zstd_decompress.c
${LIBRARY_DIR}/dictBuilder/divsufsort.c
${LIBRARY_DIR}/dictBuilder/zdict.c)
${LIBRARY_DIR}/dictBuilder/zdict.c
${LIBRARY_DIR}/deprecated/zbuff_compress.c
${LIBRARY_DIR}/deprecated/zbuff_decompress.c)
SET(Headers
${LIBRARY_DIR}/zstd.h
${LIBRARY_DIR}/common/bitstream.h
${LIBRARY_DIR}/common/error_private.h
${LIBRARY_DIR}/common/zstd_errors.h
${LIBRARY_DIR}/common/fse.h
${LIBRARY_DIR}/common/huf.h
${LIBRARY_DIR}/common/mem.h
${LIBRARY_DIR}/common/zbuff.h
${LIBRARY_DIR}/common/zstd_internal.h
${LIBRARY_DIR}/zstd.h
${LIBRARY_DIR}/dictBuilder/zdict.h)
${LIBRARY_DIR}/dictBuilder/zdict.h
${LIBRARY_DIR}/deprecated/zbuff.h)
IF (ZSTD_LEGACY_SUPPORT)
SET(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
@ -165,7 +165,7 @@ IF (UNIX)
SET(INSTALL_INCLUDE_DIR ${PREFIX}/include)
# install target
INSTALL(FILES ${LIBRARY_DIR}/zstd.h ${LIBRARY_DIR}/common/zbuff.h ${LIBRARY_DIR}/dictBuilder/zdict.h DESTINATION ${INSTALL_INCLUDE_DIR})
INSTALL(FILES ${LIBRARY_DIR}/zstd.h ${LIBRARY_DIR}/deprecated/zbuff.h ${LIBRARY_DIR}/dictBuilder/zdict.h DESTINATION ${INSTALL_INCLUDE_DIR})
INSTALL(TARGETS libzstd_static DESTINATION ${INSTALL_LIBRARY_DIR})
INSTALL(TARGETS libzstd_shared LIBRARY DESTINATION ${INSTALL_LIBRARY_DIR})

3
build/cmake/tests/CMakeLists.txt
View File

@ -50,9 +50,6 @@ ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/fuzzer.c)
TARGET_LINK_LIBRARIES(fuzzer libzstd_static)
IF (UNIX)
ADD_EXECUTABLE(zbufftest ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/zbufftest.c)
TARGET_LINK_LIBRARIES(zbufftest libzstd_static)
ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/paramgrill.c)
TARGET_LINK_LIBRARIES(paramgrill libzstd_static m) #m is math library

26
lib/Makefile
View File

@ -23,24 +23,22 @@ PREFIX ?= /usr/local
LIBDIR ?= $(PREFIX)/lib
INCLUDEDIR=$(PREFIX)/include
CPPFLAGS= -I. -I./common -DXXH_NAMESPACE=XXH_
CFLAGS ?= -O3
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
-Wpointer-arith
FLAGS = $(CPPFLAGS) $(CFLAGS) $(MOREFLAGS)
CPPFLAGS+= -I. -I./common -DXXH_NAMESPACE=ZSTD_
CFLAGS ?= -O3
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
-Wpointer-arith
CFLAGS += $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS)
ZSTD_FILES := $(wildcard common/*.c compress/*.c decompress/*.c dictBuilder/*.c)
ZSTD_EXCLUDE := compress/zbuff_compress.c decompress/zbuff_decompress.c
ZSTD_FILES := $(filter-out $(ZSTD_EXCLUDE), $(ZSTD_FILES))
ZSTD_FILES := $(wildcard common/*.c compress/*.c decompress/*.c dictBuilder/*.c deprecated/*.c)
ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0
else
ZSTD_FILES+= legacy/*.c
CPPFLAGS += -I./legacy -DZSTD_LEGACY_SUPPORT=1
ZSTD_FILES+= $(wildcard legacy/*.c)
endif
# OS X linker doesn't support -soname, and use different extension
@ -90,8 +88,8 @@ libzstd : $(LIBZSTD)
lib: libzstd.a libzstd
clean:
@$(RM) -f core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc dll/libzstd.dll dll/libzstd.lib
@$(RM) -f decompress/*.o
@$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc dll/libzstd.dll dll/libzstd.lib
@$(RM) decompress/*.o
@echo Cleaning library completed
#------------------------------------------------------------------------
@ -116,7 +114,7 @@ install: libzstd.a libzstd libzstd.pc
@install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a
@install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h
@install -m 644 common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h
@install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h
@install -m 644 deprecated/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h # prototypes generate deprecation warnings
@install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h
@echo zstd static and shared library installed

8
lib/README.md
View File

@ -56,15 +56,15 @@ file it should be linked with `dll\libzstd.dll`. For example:
```
gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\libzstd.dll
```
The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`.
The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`.
#### Obsolete streaming API
Streaming is now provided within `zstd.h`.
Older streaming API is still available within `common/zbuff.h`.
It is now deprecated, and will be removed in a future version.
Consider migrating towards newer streaming API in `zstd.h`.
Older streaming API is still available within `deprecated/zbuff.h`.
It will be removed in a future version.
Consider migrating code towards newer streaming API in `zstd.h`.
#### Miscellaneous

8
lib/common/zstd_common.c
View File

@ -16,7 +16,6 @@
#include "error_private.h"
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */
#include "zbuff.h" /* declaration of ZBUFF_isError, ZBUFF_getErrorName */
/*-****************************************
@ -44,16 +43,11 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
* provides error code string from enum */
const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); }
/* **************************************************************
* ZBUFF Error Management
****************************************************************/
/* --- ZBUFF Error Management (deprecated) --- */
unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
/*=**************************************************************
* Custom allocator
****************************************************************/

39
lib/decompress/zstd_decompress.c
View File

@ -1778,6 +1778,45 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
return sizeof(*ddict) + sizeof(ddict->refContext) + ddict->dictSize;
}
/*! ZSTD_getDictID_fromDict() :
* Provides the dictID stored within dictionary.
* if @return == 0, the dictionary is not conformant with Zstandard specification.
* It can still be loaded, but as a content-only dictionary. */
unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
{
if (dictSize < 8) return 0;
if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return 0;
return MEM_readLE32((const char*)dict + 4);
}
/*! ZSTD_getDictID_fromDDict() :
* Provides the dictID of the dictionary loaded into `ddict`.
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0;
return ZSTD_getDictID_fromDict(ddict->dict, ddict->dictSize);
}
/*! ZSTD_getDictID_fromFrame() :
* Provides the dictID required to decompressed the frame stored within `src`.
* If @return == 0, the dictID could not be decoded.
* This could for one of the following reasons :
* - The frame does not require a dictionary to be decoded (most common case).
* - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
* Note : this use case also happens when using a non-conformant dictionary.
* - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
* - This is not a Zstandard frame.
* When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */
unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)
{
ZSTD_frameParams zfp = { 0 , 0 , 0 , 0 };
size_t const hError = ZSTD_getFrameParams(&zfp, src, srcSize);
if (ZSTD_isError(hError)) return 0;
return zfp.dictID;
}
/*! ZSTD_decompress_usingDDict() :
* Decompression using a pre-digested Dictionary

1
lib/common/zbuff.h → lib/deprecated/zbuff.h
View File

@ -202,6 +202,7 @@ ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_compressInit_adv
const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize);
#endif /* ZBUFF_STATIC_LINKING_ONLY */

0
lib/compress/zbuff_compress.c → lib/deprecated/zbuff_compress.c
View File

0
lib/decompress/zbuff_decompress.c → lib/deprecated/zbuff_decompress.c
View File

27
lib/zstd.h
View File

@ -53,8 +53,6 @@ extern "C" {
*********************************************************************************************************/
/*------ Version ------*/
ZSTDLIB_API unsigned ZSTD_versionNumber (void); /**< returns version number of ZSTD */
#define ZSTD_VERSION_MAJOR 1
#define ZSTD_VERSION_MINOR 1
#define ZSTD_VERSION_RELEASE 2
@ -65,6 +63,7 @@ ZSTDLIB_API unsigned ZSTD_versionNumber (void); /**< returns version number of
#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
ZSTDLIB_API unsigned ZSTD_versionNumber (void);
/***************************************
@ -462,6 +461,30 @@ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
* Gives the amount of memory used by a given ZSTD_DDict */
ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
/*! ZSTD_getDictID_fromDict() :
* Provides the dictID stored within dictionary.
* if @return == 0, the dictionary is not conformant with Zstandard specification.
* It can still be loaded, but as a content-only dictionary. */
unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
/*! ZSTD_getDictID_fromDDict() :
* Provides the dictID of the dictionary loaded into `ddict`.
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
/*! ZSTD_getDictID_fromFrame() :
* Provides the dictID required to decompressed the frame stored within `src`.
* If @return == 0, the dictID could not be decoded.
* This could for one of the following reasons :
* - The frame does not require a dictionary to be decoded (most common case).
* - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
* Note : this use case also happens when using a non-conformant dictionary.
* - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
* - This is not a Zstandard frame.
* When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */
unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
/********************************************************************
* Advanced streaming functions

169
programs/fileio.c
View File

@ -35,6 +35,9 @@
#include "zstd.h"
#ifdef ZSTD_GZDECOMPRESS
#include "zlib.h"
#if !defined(z_const)
#define z_const
#endif
#endif
@ -465,6 +468,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
****************************************************************************/
typedef struct {
void* srcBuffer;
size_t srcBufferLoaded;
size_t srcBufferSize;
void* dstBuffer;
size_t dstBufferSize;
@ -582,63 +586,18 @@ static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
} }
}
/** FIO_decompressFrame() :
@return : size of decoded frame
*/
unsigned long long FIO_decompressFrame(dRess_t ress,
FILE* finput, size_t alreadyLoaded,
U64 alreadyDecoded)
{
U64 frameSize = 0;
size_t readSize;
U32 storedSkips = 0;
ZSTD_resetDStream(ress.dctx);
/* Header loading (optional, saves one loop) */
{ size_t const toLoad = 9 - alreadyLoaded; /* assumption : 9 >= alreadyLoaded */
size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput);
readSize = alreadyLoaded + loadedSize;
}
/* Main decompression Loop */
while (1) {
ZSTD_inBuffer inBuff = { ress.srcBuffer, readSize, 0 };
ZSTD_outBuffer outBuff= { ress.dstBuffer, ress.dstBufferSize, 0 };
size_t const readSizeHint = ZSTD_decompressStream(ress.dctx, &outBuff, &inBuff );
if (ZSTD_isError(readSizeHint)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(readSizeHint));
/* Write block */
storedSkips = FIO_fwriteSparse(ress.dstFile, ress.dstBuffer, outBuff.pos, storedSkips);
frameSize += outBuff.pos;
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)((alreadyDecoded+frameSize)>>20) );
if (readSizeHint == 0) break; /* end of frame */
if (inBuff.size != inBuff.pos) EXM_THROW(37, "Decoding error : should consume entire input");
/* Fill input buffer */
{ size_t const toRead = MIN(readSizeHint, ress.srcBufferSize); /* support large skippable frames */
readSize = fread(ress.srcBuffer, 1, toRead, finput);
if (readSize < toRead) EXM_THROW(39, "Read error : premature end");
} }
FIO_fwriteSparseEnd(ress.dstFile, storedSkips);
return frameSize;
}
/** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode
@return : 0 (no error) */
static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_t bufferSize)
static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_t bufferSize, size_t alreadyLoaded)
{
size_t const blockSize = MIN(64 KB, bufferSize);
size_t readFromInput = 1;
unsigned storedSkips = 0;
/* assumption : first 4 bytes already loaded (magic number detection), and stored within buffer */
{ size_t const sizeCheck = fwrite(buffer, 1, 4, foutput);
if (sizeCheck != 4) EXM_THROW(50, "Pass-through write error"); }
/* assumption : ress->srcBufferLoaded bytes already loaded and stored within buffer */
{ size_t const sizeCheck = fwrite(buffer, 1, alreadyLoaded, foutput);
if (sizeCheck != alreadyLoaded) EXM_THROW(50, "Pass-through write error"); }
while (readFromInput) {
readFromInput = fread(buffer, 1, blockSize, finput);
@ -649,8 +608,60 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_
return 0;
}
/** FIO_decompressFrame() :
@return : size of decoded frame
*/
unsigned long long FIO_decompressFrame(dRess_t* ress,
FILE* finput,
U64 alreadyDecoded)
{
U64 frameSize = 0;
U32 storedSkips = 0;
ZSTD_resetDStream(ress->dctx);
/* Header loading (optional, saves one loop) */
{ size_t const toRead = 9;
if (ress->srcBufferLoaded < toRead)
ress->srcBufferLoaded += fread(((char*)ress->srcBuffer) + ress->srcBufferLoaded, 1, toRead - ress->srcBufferLoaded, finput);
}
/* Main decompression Loop */
while (1) {
ZSTD_inBuffer inBuff = { ress->srcBuffer, ress->srcBufferLoaded, 0 };
ZSTD_outBuffer outBuff= { ress->dstBuffer, ress->dstBufferSize, 0 };
size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff);
if (ZSTD_isError(readSizeHint)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(readSizeHint));
/* Write block */
storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, storedSkips);
frameSize += outBuff.pos;
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)((alreadyDecoded+frameSize)>>20) );
if (inBuff.pos > 0) {
memmove(ress->srcBuffer, (char*)ress->srcBuffer + inBuff.pos, inBuff.size - inBuff.pos);
ress->srcBufferLoaded -= inBuff.pos;
}
if (readSizeHint == 0) break; /* end of frame */
if (inBuff.size != inBuff.pos) EXM_THROW(37, "Decoding error : should consume entire input");
/* Fill input buffer */
{ size_t const toRead = MIN(readSizeHint, ress->srcBufferSize); /* support large skippable frames */
if (ress->srcBufferLoaded < toRead)
ress->srcBufferLoaded += fread(((char*)ress->srcBuffer) + ress->srcBufferLoaded, 1, toRead - ress->srcBufferLoaded, finput);
if (ress->srcBufferLoaded < toRead) EXM_THROW(39, "Read error : premature end");
} }
FIO_fwriteSparseEnd(ress->dstFile, storedSkips);
return frameSize;
}
#ifdef ZSTD_GZDECOMPRESS
static unsigned long long FIO_decompressGzFrame(dRess_t ress, FILE* srcFile, const char* srcFileName, size_t alreadyLoaded)
static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName)
{
unsigned long long outFileSize = 0;
z_stream strm;
@ -662,32 +673,34 @@ static unsigned long long FIO_decompressGzFrame(dRess_t ress, FILE* srcFile, con
strm.avail_in = Z_NULL;
if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) return 0; /* see http://www.zlib.net/manual.html */
strm.next_out = ress.dstBuffer;
strm.avail_out = ress.dstBufferSize;
strm.next_out = ress->dstBuffer;
strm.avail_out = ress->dstBufferSize;
strm.avail_in = ress->srcBufferLoaded;
strm.next_in = (z_const unsigned char*)ress->srcBuffer;
for ( ; ; ) {
if (alreadyLoaded) {
strm.avail_in = alreadyLoaded;
strm.next_in = (z_const unsigned char*)ress.srcBuffer;
alreadyLoaded = 0;
} else {
if (fread(ress.srcBuffer, 1, 1/*ress.srcBufferSize*/, srcFile) == 0) break;
strm.next_in = (z_const unsigned char*)ress.srcBuffer;
strm.avail_in = 1;
int ret;
if (strm.avail_in == 0) {
ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile);
if (ress->srcBufferLoaded == 0) break;
strm.next_in = (z_const unsigned char*)ress->srcBuffer;
strm.avail_in = ress->srcBufferLoaded;
}
{ int const ret = inflate(&strm, Z_NO_FLUSH);
if (ret == Z_STREAM_END) break;
if (ret != Z_OK) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; }
}
{ size_t const decompBytes = ress.dstBufferSize - strm.avail_out;
ret = inflate(&strm, Z_NO_FLUSH);
if (ret != Z_OK && ret != Z_STREAM_END) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; }
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
if (decompBytes) {
if (fwrite(ress.dstBuffer, 1, decompBytes, ress.dstFile) != decompBytes) EXM_THROW(31, "Write error : cannot write to output file");
if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(31, "Write error : cannot write to output file");
outFileSize += decompBytes;
strm.next_out = ress.dstBuffer;
strm.avail_out = ress.dstBufferSize;
} } }
strm.next_out = ress->dstBuffer;
strm.avail_out = ress->dstBufferSize;
}
}
if (ret == Z_STREAM_END) break;
}
if (strm.avail_in > 0) memmove(ress->srcBuffer, strm.next_in, strm.avail_in);
ress->srcBufferLoaded = strm.avail_in;
inflateEnd(&strm);
return outFileSize;
}
@ -717,19 +730,19 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
for ( ; ; ) {
/* check magic number -> version */
size_t const toRead = 4;
size_t const sizeCheck = fread(ress.srcBuffer, (size_t)1, toRead, srcFile);
const BYTE* buf = (const BYTE*)ress.srcBuffer;
if (sizeCheck==0) {
if (ress.srcBufferLoaded < toRead)
ress.srcBufferLoaded += fread((char*)ress.srcBuffer + ress.srcBufferLoaded, (size_t)1, toRead - ress.srcBufferLoaded, srcFile);
if (ress.srcBufferLoaded==0) {
if (readSomething==0) { DISPLAY("zstd: %s: unexpected end of file \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */
break; /* no more input */
}
readSomething = 1; /* there is at least >= 4 bytes in srcFile */
if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */
if (ress.srcBufferLoaded < toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */
if (buf[0] == 31 && buf[1] == 139) { /* gz header */
#ifdef ZSTD_GZDECOMPRESS
unsigned long long const result = FIO_decompressGzFrame(ress, srcFile, srcFileName, toRead);
unsigned long long const result = FIO_decompressGzFrame(&ress, srcFile, srcFileName);
if (result == 0) return 1;
printf("gzip=%d\n", (int)result);
filesize += result;
#else
DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without ZSTD_GZDECOMPRESS) -- ignored \n", srcFileName);
@ -738,8 +751,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
} else {
if (!ZSTD_isFrame(ress.srcBuffer, toRead)) {
if ((g_overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */
unsigned const result = FIO_passThrough(ress.dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize);
printf("pass-through=%d\n", (int)result);
unsigned const result = FIO_passThrough(ress.dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded);
if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */
return result;
} else {
@ -747,8 +759,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
fclose(srcFile);
return 1;
} }
filesize += FIO_decompressFrame(ress, srcFile, toRead, filesize);
printf("zstd filesize=%d\n", (int)filesize);
filesize += FIO_decompressFrame(&ress, srcFile, filesize);
}
}

45
tests/Makefile
View File

@ -37,10 +37,10 @@ FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c
ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTDDIR)/decompress/huf_decompress.c
ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c
ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c
ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
ZBUFF_FILES := $(ZSTDDIR)/compress/zbuff_compress.c $(ZSTDDIR)/decompress/zbuff_decompress.c
ZBUFF_FILES := $(ZSTDDIR)/deprecated/*.c
ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
@ -53,7 +53,7 @@ EXT =
endif
VOID = /dev/null
ZBUFFTEST = -T2mn
ZSTREAM_TESTTIME = -T2mn
FUZZERTEST= -T5mn
ZSTDRTTEST= --test-large-data
@ -61,9 +61,9 @@ ZSTDRTTEST= --test-large-data
default: fullbench
all: fullbench fuzzer zstreamtest paramgrill datagen
all: fullbench fuzzer zstreamtest paramgrill datagen zbufftest
all32: fullbench32 fuzzer32 zstreamtest32
all32: fullbench32 fuzzer32 zstreamtest32 zbufftest32
@ -93,19 +93,21 @@ fullbench-dll: $(PRGDIR)/datagen.c fullbench.c
$(MAKE) -C $(ZSTDDIR) libzstd
$(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll
fuzzer : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
fuzzer : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fuzzer32 : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
fuzzer32 : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
zbufftest : CPPFLAGS += -I$(ZSTDDIR)/deprecated
zbufftest : CFLAGS += -Wno-deprecated-declarations
zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
$(CC) $(FLAGS) $^ -o $@$(EXT) # flag required to silence deprecation warnings
zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
zbufftest32 : CPPFLAGS += -I$(ZSTDDIR)/deprecated
zbufftest32 : CFLAGS += -Wno-deprecated-declarations -m32
zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
zstreamtest : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
@ -194,9 +196,20 @@ test-zstd-nolegacy: ZSTD = $(PRGDIR)/zstd
test-zstd-nolegacy: zstd-nolegacy zstd-playTests
test-gzstd: gzstd
$(PRGDIR)/zstd README.md test-zstd-speed.py
gzip README.md test-zstd-speed.py
cat README.md.zst test-zstd-speed.py.gz >zstd_gz.zst
cat README.md.gz test-zstd-speed.py.zst >gz_zstd.gz
$(PRGDIR)/zstd -d README.md.gz -o README2.md
$(PRGDIR)/zstd -d README.md.gz test-zstd-speed.py.gz
$(PRGDIR)/zstd -d zstd_gz.zst gz_zstd.gz
diff -q zstd_gz gz_zstd
echo Hello World ZSTD | $(PRGDIR)/zstd -c - >hello.zst
echo Hello World GZIP | gzip -c - >hello.gz
echo Hello World TEXT >hello.txt
cat hello.zst hello.gz hello.txt >hello_zst_gz_txt.gz
$(PRGDIR)/zstd -dcf hello.*
$(PRGDIR)/zstd -dcf - <hello_zst_gz_txt.gz
test-fullbench: fullbench datagen
$(QEMU_SYS) ./fullbench -i1
@ -213,15 +226,15 @@ test-fuzzer32: fuzzer32
$(QEMU_SYS) ./fuzzer32 $(FUZZERTEST)
test-zbuff: zbufftest
$(QEMU_SYS) ./zbufftest $(ZBUFFTEST)
$(QEMU_SYS) ./zbufftest $(ZSTREAM_TESTTIME)
test-zbuff32: zbufftest32
$(QEMU_SYS) ./zbufftest32 $(ZBUFFTEST)
$(QEMU_SYS) ./zbufftest32 $(ZSTREAM_TESTTIME)
test-zstream: zstreamtest
$(QEMU_SYS) ./zstreamtest $(ZBUFFTEST)
$(QEMU_SYS) ./zstreamtest $(ZSTREAM_TESTTIME)
test-zstream32: zstreamtest32
$(QEMU_SYS) ./zstreamtest32 $(ZBUFFTEST)
$(QEMU_SYS) ./zstreamtest32 $(ZSTREAM_TESTTIME)
endif

22
tests/fuzzer.c
View File

@ -27,7 +27,7 @@
#include <time.h> /* clock_t */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
#include "zstd.h" /* ZSTD_VERSION_STRING */
#include "zstd_errors.h" /* ZSTD_getErrorCode */
#include "zstd_errors.h" /* ZSTD_getErrorCode */
#include "zdict.h" /* ZDICT_trainFromBuffer */
#include "datagen.h" /* RDG_genBuffer */
#include "mem.h"
@ -246,6 +246,7 @@ static int basicUnitTests(U32 seed, double compressibility)
size_t const sampleUnitSize = 8 KB;
U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
U32 dictID;
if (dictBuffer==NULL || samplesSizes==NULL) {
free(dictBuffer);
@ -261,10 +262,9 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)dictSize);
DISPLAYLEVEL(4, "test%3i : check dictID : ", testNb++);
{ U32 const dictID = ZDICT_getDictID(dictBuffer, dictSize);
if (dictID==0) goto _output_error;
DISPLAYLEVEL(4, "OK : %u \n", dictID);
}
dictID = ZDICT_getDictID(dictBuffer, dictSize);
if (dictID==0) goto _output_error;
DISPLAYLEVEL(4, "OK : %u \n", dictID);
DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++);
cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
@ -273,6 +273,18 @@ static int basicUnitTests(U32 seed, double compressibility)
if (ZSTD_isError(cSize)) goto _output_error;
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : retrieve dictID from dictionary : ", testNb++);
{ U32 const did = ZSTD_getDictID_fromDict(dictBuffer, dictSize);
if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : retrieve dictID from frame : ", testNb++);
{ U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
}
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
decodedBuffer, CNBuffSize,

3
zlibWrapper/.gitignore vendored
View File

@ -2,11 +2,14 @@
_*
example.*
example_zstd.*
example_gz.*
fitblk.*
fitblk_zstd.*
zwrapbench.*
foo.gz
minigzip
minigzip_zstd
example
example_zstd
fitblk

4
zlibWrapper/Makefile
View File

@ -13,12 +13,12 @@ ZLIB_PATH ?= .
ZSTDLIBDIR = ../lib
ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.a
ZLIBWRAPPER_PATH = .
GZFILES = gzclose.o gzlib.o gzread.o gzwrite.o
GZFILES = gzclose.o gzlib.o gzread.o gzwrite.o
EXAMPLE_PATH = examples
PROGRAMS_PATH = ../programs
TEST_FILE = ../doc/zstd_compression_format.md
CPPFLAGS = -DXXH_NAMESPACE=XXH_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH)
CPPFLAGS = -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH)
CFLAGS ?= $(MOREFLAGS) -O3 -std=gnu99
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wstrict-aliasing=1