Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11
This commit is contained in:
Przemyslaw Skibinski
commit
23a1e11360
3
NEWS
3
NEWS
@ -7,6 +7,9 @@ cli : fixed : status displays total amount decoded, even for file consisting of
|
||||
cli : fixed : zstdcat
|
||||
API : changed : zbuff prototypes now generate deprecation warnings
|
||||
API : changed : streaming decompression implicit reset on starting new frame
|
||||
API : added experimental : dictID retrieval functions
|
||||
zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski
|
||||
Changed : zbuff source files moved to lib/deprecated
|
||||
Changed : reduced stack memory use
|
||||
|
||||
v1.1.1
|
||||
|
||||
@ -363,14 +363,6 @@
|
||||
RelativePath="..\..\..\lib\decompress\huf_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zbuff_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zbuff_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zstd_common.c"
|
||||
>
|
||||
@ -425,10 +417,6 @@
|
||||
RelativePath="..\..\..\lib\common\mem.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zbuff_static.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\zstd.h"
|
||||
>
|
||||
|
||||
@ -429,10 +429,6 @@
|
||||
RelativePath="..\..\..\lib\common\xxhash.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zbuff_static.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
|
||||
>
|
||||
|
||||
@ -379,14 +379,6 @@
|
||||
RelativePath="..\..\..\lib\common\xxhash.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zbuff_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zbuff_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict.c"
|
||||
>
|
||||
@ -481,14 +473,6 @@
|
||||
RelativePath="..\..\..\lib\common\xxhash.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zbuff.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zbuff_static.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
|
||||
>
|
||||
|
||||
@ -360,11 +360,11 @@
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\compress\zbuff_compress.c"
|
||||
RelativePath="..\..\..\lib\deprecated\zbuff_compress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\decompress\zbuff_decompress.c"
|
||||
RelativePath="..\..\..\lib\deprecated\zbuff_decompress.c"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
@ -461,10 +461,6 @@
|
||||
RelativePath="..\..\..\lib\common\zbuff.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\common\zbuff_static.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
|
||||
>
|
||||
|
||||
@ -64,25 +64,25 @@ SET(Sources
|
||||
${LIBRARY_DIR}/common/fse_decompress.c
|
||||
${LIBRARY_DIR}/compress/fse_compress.c
|
||||
${LIBRARY_DIR}/compress/huf_compress.c
|
||||
${LIBRARY_DIR}/compress/zbuff_compress.c
|
||||
${LIBRARY_DIR}/compress/zstd_compress.c
|
||||
${LIBRARY_DIR}/decompress/huf_decompress.c
|
||||
${LIBRARY_DIR}/decompress/zbuff_decompress.c
|
||||
${LIBRARY_DIR}/decompress/zstd_decompress.c
|
||||
${LIBRARY_DIR}/dictBuilder/divsufsort.c
|
||||
${LIBRARY_DIR}/dictBuilder/zdict.c)
|
||||
${LIBRARY_DIR}/dictBuilder/zdict.c
|
||||
${LIBRARY_DIR}/deprecated/zbuff_compress.c
|
||||
${LIBRARY_DIR}/deprecated/zbuff_decompress.c)
|
||||
|
||||
SET(Headers
|
||||
${LIBRARY_DIR}/zstd.h
|
||||
${LIBRARY_DIR}/common/bitstream.h
|
||||
${LIBRARY_DIR}/common/error_private.h
|
||||
${LIBRARY_DIR}/common/zstd_errors.h
|
||||
${LIBRARY_DIR}/common/fse.h
|
||||
${LIBRARY_DIR}/common/huf.h
|
||||
${LIBRARY_DIR}/common/mem.h
|
||||
${LIBRARY_DIR}/common/zbuff.h
|
||||
${LIBRARY_DIR}/common/zstd_internal.h
|
||||
${LIBRARY_DIR}/zstd.h
|
||||
${LIBRARY_DIR}/dictBuilder/zdict.h)
|
||||
${LIBRARY_DIR}/dictBuilder/zdict.h
|
||||
${LIBRARY_DIR}/deprecated/zbuff.h)
|
||||
|
||||
IF (ZSTD_LEGACY_SUPPORT)
|
||||
SET(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
|
||||
@ -165,7 +165,7 @@ IF (UNIX)
|
||||
SET(INSTALL_INCLUDE_DIR ${PREFIX}/include)
|
||||
|
||||
# install target
|
||||
INSTALL(FILES ${LIBRARY_DIR}/zstd.h ${LIBRARY_DIR}/common/zbuff.h ${LIBRARY_DIR}/dictBuilder/zdict.h DESTINATION ${INSTALL_INCLUDE_DIR})
|
||||
INSTALL(FILES ${LIBRARY_DIR}/zstd.h ${LIBRARY_DIR}/deprecated/zbuff.h ${LIBRARY_DIR}/dictBuilder/zdict.h DESTINATION ${INSTALL_INCLUDE_DIR})
|
||||
INSTALL(TARGETS libzstd_static DESTINATION ${INSTALL_LIBRARY_DIR})
|
||||
INSTALL(TARGETS libzstd_shared LIBRARY DESTINATION ${INSTALL_LIBRARY_DIR})
|
||||
|
||||
|
||||
@ -50,9 +50,6 @@ ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/fuzzer.c)
|
||||
TARGET_LINK_LIBRARIES(fuzzer libzstd_static)
|
||||
|
||||
IF (UNIX)
|
||||
ADD_EXECUTABLE(zbufftest ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/zbufftest.c)
|
||||
TARGET_LINK_LIBRARIES(zbufftest libzstd_static)
|
||||
|
||||
ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/paramgrill.c)
|
||||
TARGET_LINK_LIBRARIES(paramgrill libzstd_static m) #m is math library
|
||||
|
||||
|
||||
26
lib/Makefile
26
lib/Makefile
@ -23,24 +23,22 @@ PREFIX ?= /usr/local
|
||||
LIBDIR ?= $(PREFIX)/lib
|
||||
INCLUDEDIR=$(PREFIX)/include
|
||||
|
||||
CPPFLAGS= -I. -I./common -DXXH_NAMESPACE=XXH_
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
|
||||
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
|
||||
-Wpointer-arith
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(MOREFLAGS)
|
||||
CPPFLAGS+= -I. -I./common -DXXH_NAMESPACE=ZSTD_
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
|
||||
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
|
||||
-Wpointer-arith
|
||||
CFLAGS += $(MOREFLAGS)
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS)
|
||||
|
||||
|
||||
ZSTD_FILES := $(wildcard common/*.c compress/*.c decompress/*.c dictBuilder/*.c)
|
||||
ZSTD_EXCLUDE := compress/zbuff_compress.c decompress/zbuff_decompress.c
|
||||
ZSTD_FILES := $(filter-out $(ZSTD_EXCLUDE), $(ZSTD_FILES))
|
||||
|
||||
ZSTD_FILES := $(wildcard common/*.c compress/*.c decompress/*.c dictBuilder/*.c deprecated/*.c)
|
||||
|
||||
ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
|
||||
CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0
|
||||
else
|
||||
ZSTD_FILES+= legacy/*.c
|
||||
CPPFLAGS += -I./legacy -DZSTD_LEGACY_SUPPORT=1
|
||||
ZSTD_FILES+= $(wildcard legacy/*.c)
|
||||
endif
|
||||
|
||||
# OS X linker doesn't support -soname, and use different extension
|
||||
@ -90,8 +88,8 @@ libzstd : $(LIBZSTD)
|
||||
lib: libzstd.a libzstd
|
||||
|
||||
clean:
|
||||
@$(RM) -f core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc dll/libzstd.dll dll/libzstd.lib
|
||||
@$(RM) -f decompress/*.o
|
||||
@$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc dll/libzstd.dll dll/libzstd.lib
|
||||
@$(RM) decompress/*.o
|
||||
@echo Cleaning library completed
|
||||
|
||||
#------------------------------------------------------------------------
|
||||
@ -116,7 +114,7 @@ install: libzstd.a libzstd libzstd.pc
|
||||
@install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a
|
||||
@install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h
|
||||
@install -m 644 common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h
|
||||
@install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h
|
||||
@install -m 644 deprecated/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h # prototypes generate deprecation warnings
|
||||
@install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h
|
||||
@echo zstd static and shared library installed
|
||||
|
||||
|
||||
@ -62,9 +62,9 @@ The compiled executable will require ZSTD DLL which is available at `dll\libzstd
|
||||
#### Obsolete streaming API
|
||||
|
||||
Streaming is now provided within `zstd.h`.
|
||||
Older streaming API is still available within `common/zbuff.h`.
|
||||
It is now deprecated, and will be removed in a future version.
|
||||
Consider migrating towards newer streaming API in `zstd.h`.
|
||||
Older streaming API is still available within `deprecated/zbuff.h`.
|
||||
It will be removed in a future version.
|
||||
Consider migrating code towards newer streaming API in `zstd.h`.
|
||||
|
||||
|
||||
#### Miscellaneous
|
||||
|
||||
@ -16,7 +16,6 @@
|
||||
#include "error_private.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */
|
||||
#include "zbuff.h" /* declaration of ZBUFF_isError, ZBUFF_getErrorName */
|
||||
|
||||
|
||||
/*-****************************************
|
||||
@ -44,16 +43,11 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
|
||||
* provides error code string from enum */
|
||||
const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); }
|
||||
|
||||
|
||||
/* **************************************************************
|
||||
* ZBUFF Error Management
|
||||
****************************************************************/
|
||||
/* --- ZBUFF Error Management (deprecated) --- */
|
||||
unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
|
||||
|
||||
const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
|
||||
|
||||
|
||||
|
||||
/*=**************************************************************
|
||||
* Custom allocator
|
||||
****************************************************************/
|
||||
|
||||
@ -1778,6 +1778,45 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
|
||||
return sizeof(*ddict) + sizeof(ddict->refContext) + ddict->dictSize;
|
||||
}
|
||||
|
||||
/*! ZSTD_getDictID_fromDict() :
|
||||
* Provides the dictID stored within dictionary.
|
||||
* if @return == 0, the dictionary is not conformant with Zstandard specification.
|
||||
* It can still be loaded, but as a content-only dictionary. */
|
||||
unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
|
||||
{
|
||||
if (dictSize < 8) return 0;
|
||||
if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return 0;
|
||||
return MEM_readLE32((const char*)dict + 4);
|
||||
}
|
||||
|
||||
/*! ZSTD_getDictID_fromDDict() :
|
||||
* Provides the dictID of the dictionary loaded into `ddict`.
|
||||
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
|
||||
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
|
||||
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
|
||||
{
|
||||
if (ddict==NULL) return 0;
|
||||
return ZSTD_getDictID_fromDict(ddict->dict, ddict->dictSize);
|
||||
}
|
||||
|
||||
/*! ZSTD_getDictID_fromFrame() :
|
||||
* Provides the dictID required to decompressed the frame stored within `src`.
|
||||
* If @return == 0, the dictID could not be decoded.
|
||||
* This could for one of the following reasons :
|
||||
* - The frame does not require a dictionary to be decoded (most common case).
|
||||
* - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
|
||||
* Note : this use case also happens when using a non-conformant dictionary.
|
||||
* - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
|
||||
* - This is not a Zstandard frame.
|
||||
* When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */
|
||||
unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)
|
||||
{
|
||||
ZSTD_frameParams zfp = { 0 , 0 , 0 , 0 };
|
||||
size_t const hError = ZSTD_getFrameParams(&zfp, src, srcSize);
|
||||
if (ZSTD_isError(hError)) return 0;
|
||||
return zfp.dictID;
|
||||
}
|
||||
|
||||
|
||||
/*! ZSTD_decompress_usingDDict() :
|
||||
* Decompression using a pre-digested Dictionary
|
||||
|
||||
@ -202,6 +202,7 @@ ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_compressInit_adv
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, unsigned long long pledgedSrcSize);
|
||||
|
||||
|
||||
#endif /* ZBUFF_STATIC_LINKING_ONLY */
|
||||
|
||||
|
||||
27
lib/zstd.h
27
lib/zstd.h
@ -53,8 +53,6 @@ extern "C" {
|
||||
*********************************************************************************************************/
|
||||
|
||||
/*------ Version ------*/
|
||||
ZSTDLIB_API unsigned ZSTD_versionNumber (void); /**< returns version number of ZSTD */
|
||||
|
||||
#define ZSTD_VERSION_MAJOR 1
|
||||
#define ZSTD_VERSION_MINOR 1
|
||||
#define ZSTD_VERSION_RELEASE 2
|
||||
@ -65,6 +63,7 @@ ZSTDLIB_API unsigned ZSTD_versionNumber (void); /**< returns version number of
|
||||
#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
|
||||
|
||||
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
|
||||
ZSTDLIB_API unsigned ZSTD_versionNumber (void);
|
||||
|
||||
|
||||
/***************************************
|
||||
@ -462,6 +461,30 @@ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
|
||||
* Gives the amount of memory used by a given ZSTD_DDict */
|
||||
ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
|
||||
|
||||
/*! ZSTD_getDictID_fromDict() :
|
||||
* Provides the dictID stored within dictionary.
|
||||
* if @return == 0, the dictionary is not conformant with Zstandard specification.
|
||||
* It can still be loaded, but as a content-only dictionary. */
|
||||
unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
|
||||
|
||||
/*! ZSTD_getDictID_fromDDict() :
|
||||
* Provides the dictID of the dictionary loaded into `ddict`.
|
||||
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
|
||||
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
|
||||
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
|
||||
|
||||
/*! ZSTD_getDictID_fromFrame() :
|
||||
* Provides the dictID required to decompressed the frame stored within `src`.
|
||||
* If @return == 0, the dictID could not be decoded.
|
||||
* This could for one of the following reasons :
|
||||
* - The frame does not require a dictionary to be decoded (most common case).
|
||||
* - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
|
||||
* Note : this use case also happens when using a non-conformant dictionary.
|
||||
* - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
|
||||
* - This is not a Zstandard frame.
|
||||
* When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */
|
||||
unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
|
||||
|
||||
|
||||
/********************************************************************
|
||||
* Advanced streaming functions
|
||||
|
||||
@ -35,6 +35,9 @@
|
||||
#include "zstd.h"
|
||||
#ifdef ZSTD_GZDECOMPRESS
|
||||
#include "zlib.h"
|
||||
#if !defined(z_const)
|
||||
#define z_const
|
||||
#endif
|
||||
#endif
|
||||
|
||||
|
||||
@ -465,6 +468,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
|
||||
****************************************************************************/
|
||||
typedef struct {
|
||||
void* srcBuffer;
|
||||
size_t srcBufferLoaded;
|
||||
size_t srcBufferSize;
|
||||
void* dstBuffer;
|
||||
size_t dstBufferSize;
|
||||
@ -582,63 +586,18 @@ static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
|
||||
} }
|
||||
}
|
||||
|
||||
/** FIO_decompressFrame() :
|
||||
@return : size of decoded frame
|
||||
*/
|
||||
unsigned long long FIO_decompressFrame(dRess_t ress,
|
||||
FILE* finput, size_t alreadyLoaded,
|
||||
U64 alreadyDecoded)
|
||||
{
|
||||
U64 frameSize = 0;
|
||||
size_t readSize;
|
||||
U32 storedSkips = 0;
|
||||
|
||||
ZSTD_resetDStream(ress.dctx);
|
||||
|
||||
/* Header loading (optional, saves one loop) */
|
||||
{ size_t const toLoad = 9 - alreadyLoaded; /* assumption : 9 >= alreadyLoaded */
|
||||
size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput);
|
||||
readSize = alreadyLoaded + loadedSize;
|
||||
}
|
||||
|
||||
/* Main decompression Loop */
|
||||
while (1) {
|
||||
ZSTD_inBuffer inBuff = { ress.srcBuffer, readSize, 0 };
|
||||
ZSTD_outBuffer outBuff= { ress.dstBuffer, ress.dstBufferSize, 0 };
|
||||
size_t const readSizeHint = ZSTD_decompressStream(ress.dctx, &outBuff, &inBuff );
|
||||
if (ZSTD_isError(readSizeHint)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(readSizeHint));
|
||||
|
||||
/* Write block */
|
||||
storedSkips = FIO_fwriteSparse(ress.dstFile, ress.dstBuffer, outBuff.pos, storedSkips);
|
||||
frameSize += outBuff.pos;
|
||||
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)((alreadyDecoded+frameSize)>>20) );
|
||||
|
||||
if (readSizeHint == 0) break; /* end of frame */
|
||||
if (inBuff.size != inBuff.pos) EXM_THROW(37, "Decoding error : should consume entire input");
|
||||
|
||||
/* Fill input buffer */
|
||||
{ size_t const toRead = MIN(readSizeHint, ress.srcBufferSize); /* support large skippable frames */
|
||||
readSize = fread(ress.srcBuffer, 1, toRead, finput);
|
||||
if (readSize < toRead) EXM_THROW(39, "Read error : premature end");
|
||||
} }
|
||||
|
||||
FIO_fwriteSparseEnd(ress.dstFile, storedSkips);
|
||||
|
||||
return frameSize;
|
||||
}
|
||||
|
||||
|
||||
/** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode
|
||||
@return : 0 (no error) */
|
||||
static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_t bufferSize)
|
||||
static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_t bufferSize, size_t alreadyLoaded)
|
||||
{
|
||||
size_t const blockSize = MIN(64 KB, bufferSize);
|
||||
size_t readFromInput = 1;
|
||||
unsigned storedSkips = 0;
|
||||
|
||||
/* assumption : first 4 bytes already loaded (magic number detection), and stored within buffer */
|
||||
{ size_t const sizeCheck = fwrite(buffer, 1, 4, foutput);
|
||||
if (sizeCheck != 4) EXM_THROW(50, "Pass-through write error"); }
|
||||
/* assumption : ress->srcBufferLoaded bytes already loaded and stored within buffer */
|
||||
{ size_t const sizeCheck = fwrite(buffer, 1, alreadyLoaded, foutput);
|
||||
if (sizeCheck != alreadyLoaded) EXM_THROW(50, "Pass-through write error"); }
|
||||
|
||||
while (readFromInput) {
|
||||
readFromInput = fread(buffer, 1, blockSize, finput);
|
||||
@ -649,8 +608,60 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/** FIO_decompressFrame() :
|
||||
@return : size of decoded frame
|
||||
*/
|
||||
unsigned long long FIO_decompressFrame(dRess_t* ress,
|
||||
FILE* finput,
|
||||
U64 alreadyDecoded)
|
||||
{
|
||||
U64 frameSize = 0;
|
||||
U32 storedSkips = 0;
|
||||
|
||||
ZSTD_resetDStream(ress->dctx);
|
||||
|
||||
/* Header loading (optional, saves one loop) */
|
||||
{ size_t const toRead = 9;
|
||||
if (ress->srcBufferLoaded < toRead)
|
||||
ress->srcBufferLoaded += fread(((char*)ress->srcBuffer) + ress->srcBufferLoaded, 1, toRead - ress->srcBufferLoaded, finput);
|
||||
}
|
||||
|
||||
/* Main decompression Loop */
|
||||
while (1) {
|
||||
ZSTD_inBuffer inBuff = { ress->srcBuffer, ress->srcBufferLoaded, 0 };
|
||||
ZSTD_outBuffer outBuff= { ress->dstBuffer, ress->dstBufferSize, 0 };
|
||||
size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff);
|
||||
if (ZSTD_isError(readSizeHint)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(readSizeHint));
|
||||
|
||||
/* Write block */
|
||||
storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, storedSkips);
|
||||
frameSize += outBuff.pos;
|
||||
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)((alreadyDecoded+frameSize)>>20) );
|
||||
|
||||
if (inBuff.pos > 0) {
|
||||
memmove(ress->srcBuffer, (char*)ress->srcBuffer + inBuff.pos, inBuff.size - inBuff.pos);
|
||||
ress->srcBufferLoaded -= inBuff.pos;
|
||||
}
|
||||
|
||||
if (readSizeHint == 0) break; /* end of frame */
|
||||
if (inBuff.size != inBuff.pos) EXM_THROW(37, "Decoding error : should consume entire input");
|
||||
|
||||
/* Fill input buffer */
|
||||
{ size_t const toRead = MIN(readSizeHint, ress->srcBufferSize); /* support large skippable frames */
|
||||
if (ress->srcBufferLoaded < toRead)
|
||||
ress->srcBufferLoaded += fread(((char*)ress->srcBuffer) + ress->srcBufferLoaded, 1, toRead - ress->srcBufferLoaded, finput);
|
||||
if (ress->srcBufferLoaded < toRead) EXM_THROW(39, "Read error : premature end");
|
||||
} }
|
||||
|
||||
FIO_fwriteSparseEnd(ress->dstFile, storedSkips);
|
||||
|
||||
return frameSize;
|
||||
}
|
||||
|
||||
|
||||
#ifdef ZSTD_GZDECOMPRESS
|
||||
static unsigned long long FIO_decompressGzFrame(dRess_t ress, FILE* srcFile, const char* srcFileName, size_t alreadyLoaded)
|
||||
static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName)
|
||||
{
|
||||
unsigned long long outFileSize = 0;
|
||||
z_stream strm;
|
||||
@ -662,32 +673,34 @@ static unsigned long long FIO_decompressGzFrame(dRess_t ress, FILE* srcFile, con
|
||||
strm.avail_in = Z_NULL;
|
||||
if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) return 0; /* see http://www.zlib.net/manual.html */
|
||||
|
||||
strm.next_out = ress.dstBuffer;
|
||||
strm.avail_out = ress.dstBufferSize;
|
||||
strm.next_out = ress->dstBuffer;
|
||||
strm.avail_out = ress->dstBufferSize;
|
||||
strm.avail_in = ress->srcBufferLoaded;
|
||||
strm.next_in = (z_const unsigned char*)ress->srcBuffer;
|
||||
|
||||
for ( ; ; ) {
|
||||
if (alreadyLoaded) {
|
||||
strm.avail_in = alreadyLoaded;
|
||||
strm.next_in = (z_const unsigned char*)ress.srcBuffer;
|
||||
alreadyLoaded = 0;
|
||||
} else {
|
||||
if (fread(ress.srcBuffer, 1, 1/*ress.srcBufferSize*/, srcFile) == 0) break;
|
||||
strm.next_in = (z_const unsigned char*)ress.srcBuffer;
|
||||
strm.avail_in = 1;
|
||||
int ret;
|
||||
if (strm.avail_in == 0) {
|
||||
ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile);
|
||||
if (ress->srcBufferLoaded == 0) break;
|
||||
strm.next_in = (z_const unsigned char*)ress->srcBuffer;
|
||||
strm.avail_in = ress->srcBufferLoaded;
|
||||
}
|
||||
{ int const ret = inflate(&strm, Z_NO_FLUSH);
|
||||
if (ret == Z_STREAM_END) break;
|
||||
if (ret != Z_OK) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; }
|
||||
}
|
||||
|
||||
{ size_t const decompBytes = ress.dstBufferSize - strm.avail_out;
|
||||
ret = inflate(&strm, Z_NO_FLUSH);
|
||||
if (ret != Z_OK && ret != Z_STREAM_END) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; }
|
||||
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
|
||||
if (decompBytes) {
|
||||
if (fwrite(ress.dstBuffer, 1, decompBytes, ress.dstFile) != decompBytes) EXM_THROW(31, "Write error : cannot write to output file");
|
||||
if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(31, "Write error : cannot write to output file");
|
||||
outFileSize += decompBytes;
|
||||
strm.next_out = ress.dstBuffer;
|
||||
strm.avail_out = ress.dstBufferSize;
|
||||
} } }
|
||||
strm.next_out = ress->dstBuffer;
|
||||
strm.avail_out = ress->dstBufferSize;
|
||||
}
|
||||
}
|
||||
if (ret == Z_STREAM_END) break;
|
||||
}
|
||||
|
||||
if (strm.avail_in > 0) memmove(ress->srcBuffer, strm.next_in, strm.avail_in);
|
||||
ress->srcBufferLoaded = strm.avail_in;
|
||||
inflateEnd(&strm);
|
||||
return outFileSize;
|
||||
}
|
||||
@ -717,19 +730,19 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
|
||||
for ( ; ; ) {
|
||||
/* check magic number -> version */
|
||||
size_t const toRead = 4;
|
||||
size_t const sizeCheck = fread(ress.srcBuffer, (size_t)1, toRead, srcFile);
|
||||
const BYTE* buf = (const BYTE*)ress.srcBuffer;
|
||||
if (sizeCheck==0) {
|
||||
if (ress.srcBufferLoaded < toRead)
|
||||
ress.srcBufferLoaded += fread((char*)ress.srcBuffer + ress.srcBufferLoaded, (size_t)1, toRead - ress.srcBufferLoaded, srcFile);
|
||||
if (ress.srcBufferLoaded==0) {
|
||||
if (readSomething==0) { DISPLAY("zstd: %s: unexpected end of file \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */
|
||||
break; /* no more input */
|
||||
}
|
||||
readSomething = 1; /* there is at least >= 4 bytes in srcFile */
|
||||
if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */
|
||||
if (ress.srcBufferLoaded < toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */
|
||||
if (buf[0] == 31 && buf[1] == 139) { /* gz header */
|
||||
#ifdef ZSTD_GZDECOMPRESS
|
||||
unsigned long long const result = FIO_decompressGzFrame(ress, srcFile, srcFileName, toRead);
|
||||
unsigned long long const result = FIO_decompressGzFrame(&ress, srcFile, srcFileName);
|
||||
if (result == 0) return 1;
|
||||
printf("gzip=%d\n", (int)result);
|
||||
filesize += result;
|
||||
#else
|
||||
DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without ZSTD_GZDECOMPRESS) -- ignored \n", srcFileName);
|
||||
@ -738,8 +751,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
|
||||
} else {
|
||||
if (!ZSTD_isFrame(ress.srcBuffer, toRead)) {
|
||||
if ((g_overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */
|
||||
unsigned const result = FIO_passThrough(ress.dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize);
|
||||
printf("pass-through=%d\n", (int)result);
|
||||
unsigned const result = FIO_passThrough(ress.dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded);
|
||||
if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */
|
||||
return result;
|
||||
} else {
|
||||
@ -747,8 +759,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
|
||||
fclose(srcFile);
|
||||
return 1;
|
||||
} }
|
||||
filesize += FIO_decompressFrame(ress, srcFile, toRead, filesize);
|
||||
printf("zstd filesize=%d\n", (int)filesize);
|
||||
filesize += FIO_decompressFrame(&ress, srcFile, filesize);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -37,10 +37,10 @@ FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
|
||||
|
||||
|
||||
ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
|
||||
ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c
|
||||
ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTDDIR)/decompress/huf_decompress.c
|
||||
ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c
|
||||
ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c
|
||||
ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
|
||||
ZBUFF_FILES := $(ZSTDDIR)/compress/zbuff_compress.c $(ZSTDDIR)/decompress/zbuff_decompress.c
|
||||
ZBUFF_FILES := $(ZSTDDIR)/deprecated/*.c
|
||||
ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
|
||||
|
||||
|
||||
@ -53,7 +53,7 @@ EXT =
|
||||
endif
|
||||
|
||||
VOID = /dev/null
|
||||
ZBUFFTEST = -T2mn
|
||||
ZSTREAM_TESTTIME = -T2mn
|
||||
FUZZERTEST= -T5mn
|
||||
ZSTDRTTEST= --test-large-data
|
||||
|
||||
@ -61,9 +61,9 @@ ZSTDRTTEST= --test-large-data
|
||||
|
||||
default: fullbench
|
||||
|
||||
all: fullbench fuzzer zstreamtest paramgrill datagen
|
||||
all: fullbench fuzzer zstreamtest paramgrill datagen zbufftest
|
||||
|
||||
all32: fullbench32 fuzzer32 zstreamtest32
|
||||
all32: fullbench32 fuzzer32 zstreamtest32 zbufftest32
|
||||
|
||||
|
||||
|
||||
@ -93,19 +93,21 @@ fullbench-dll: $(PRGDIR)/datagen.c fullbench.c
|
||||
$(MAKE) -C $(ZSTDDIR) libzstd
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll
|
||||
|
||||
fuzzer : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
|
||||
fuzzer : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fuzzer32 : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
|
||||
fuzzer32 : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
zbufftest : CPPFLAGS += -I$(ZSTDDIR)/deprecated
|
||||
zbufftest : CFLAGS += -Wno-deprecated-declarations
|
||||
zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT) # flag required to silence deprecation warnings
|
||||
|
||||
zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
zbufftest32 : CPPFLAGS += -I$(ZSTDDIR)/deprecated
|
||||
zbufftest32 : CFLAGS += -Wno-deprecated-declarations -m32
|
||||
zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
zstreamtest : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
@ -194,9 +196,20 @@ test-zstd-nolegacy: ZSTD = $(PRGDIR)/zstd
|
||||
test-zstd-nolegacy: zstd-nolegacy zstd-playTests
|
||||
|
||||
test-gzstd: gzstd
|
||||
$(PRGDIR)/zstd README.md test-zstd-speed.py
|
||||
gzip README.md test-zstd-speed.py
|
||||
cat README.md.zst test-zstd-speed.py.gz >zstd_gz.zst
|
||||
cat README.md.gz test-zstd-speed.py.zst >gz_zstd.gz
|
||||
$(PRGDIR)/zstd -d README.md.gz -o README2.md
|
||||
$(PRGDIR)/zstd -d README.md.gz test-zstd-speed.py.gz
|
||||
$(PRGDIR)/zstd -d zstd_gz.zst gz_zstd.gz
|
||||
diff -q zstd_gz gz_zstd
|
||||
echo Hello World ZSTD | $(PRGDIR)/zstd -c - >hello.zst
|
||||
echo Hello World GZIP | gzip -c - >hello.gz
|
||||
echo Hello World TEXT >hello.txt
|
||||
cat hello.zst hello.gz hello.txt >hello_zst_gz_txt.gz
|
||||
$(PRGDIR)/zstd -dcf hello.*
|
||||
$(PRGDIR)/zstd -dcf - <hello_zst_gz_txt.gz
|
||||
|
||||
test-fullbench: fullbench datagen
|
||||
$(QEMU_SYS) ./fullbench -i1
|
||||
@ -213,15 +226,15 @@ test-fuzzer32: fuzzer32
|
||||
$(QEMU_SYS) ./fuzzer32 $(FUZZERTEST)
|
||||
|
||||
test-zbuff: zbufftest
|
||||
$(QEMU_SYS) ./zbufftest $(ZBUFFTEST)
|
||||
$(QEMU_SYS) ./zbufftest $(ZSTREAM_TESTTIME)
|
||||
|
||||
test-zbuff32: zbufftest32
|
||||
$(QEMU_SYS) ./zbufftest32 $(ZBUFFTEST)
|
||||
$(QEMU_SYS) ./zbufftest32 $(ZSTREAM_TESTTIME)
|
||||
|
||||
test-zstream: zstreamtest
|
||||
$(QEMU_SYS) ./zstreamtest $(ZBUFFTEST)
|
||||
$(QEMU_SYS) ./zstreamtest $(ZSTREAM_TESTTIME)
|
||||
|
||||
test-zstream32: zstreamtest32
|
||||
$(QEMU_SYS) ./zstreamtest32 $(ZBUFFTEST)
|
||||
$(QEMU_SYS) ./zstreamtest32 $(ZSTREAM_TESTTIME)
|
||||
|
||||
endif
|
||||
|
||||
@ -27,7 +27,7 @@
|
||||
#include <time.h> /* clock_t */
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
|
||||
#include "zstd.h" /* ZSTD_VERSION_STRING */
|
||||
#include "zstd_errors.h" /* ZSTD_getErrorCode */
|
||||
#include "zstd_errors.h" /* ZSTD_getErrorCode */
|
||||
#include "zdict.h" /* ZDICT_trainFromBuffer */
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
#include "mem.h"
|
||||
@ -246,6 +246,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
size_t const sampleUnitSize = 8 KB;
|
||||
U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
|
||||
size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
|
||||
U32 dictID;
|
||||
|
||||
if (dictBuffer==NULL || samplesSizes==NULL) {
|
||||
free(dictBuffer);
|
||||
@ -261,10 +262,9 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)dictSize);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : check dictID : ", testNb++);
|
||||
{ U32 const dictID = ZDICT_getDictID(dictBuffer, dictSize);
|
||||
if (dictID==0) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK : %u \n", dictID);
|
||||
}
|
||||
dictID = ZDICT_getDictID(dictBuffer, dictSize);
|
||||
if (dictID==0) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK : %u \n", dictID);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++);
|
||||
cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
|
||||
@ -273,6 +273,18 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
if (ZSTD_isError(cSize)) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : retrieve dictID from dictionary : ", testNb++);
|
||||
{ U32 const did = ZSTD_getDictID_fromDict(dictBuffer, dictSize);
|
||||
if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : retrieve dictID from frame : ", testNb++);
|
||||
{ U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
|
||||
if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
|
||||
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
|
||||
decodedBuffer, CNBuffSize,
|
||||
|
||||
3
zlibWrapper/.gitignore
vendored
3
zlibWrapper/.gitignore
vendored
@ -2,11 +2,14 @@
|
||||
_*
|
||||
example.*
|
||||
example_zstd.*
|
||||
example_gz.*
|
||||
fitblk.*
|
||||
fitblk_zstd.*
|
||||
zwrapbench.*
|
||||
foo.gz
|
||||
|
||||
minigzip
|
||||
minigzip_zstd
|
||||
example
|
||||
example_zstd
|
||||
fitblk
|
||||
|
||||
@ -18,7 +18,7 @@ EXAMPLE_PATH = examples
|
||||
PROGRAMS_PATH = ../programs
|
||||
TEST_FILE = ../doc/zstd_compression_format.md
|
||||
|
||||
CPPFLAGS = -DXXH_NAMESPACE=XXH_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH)
|
||||
CPPFLAGS = -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH)
|
||||
CFLAGS ?= $(MOREFLAGS) -O3 -std=gnu99
|
||||
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wstrict-aliasing=1
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user