Yann Collet
commit
62e1b5f5b3
15
Makefile
15
Makefile
@ -32,12 +32,19 @@
|
||||
# ################################################################
|
||||
|
||||
# Version number
|
||||
export VERSION := 0.3.6
|
||||
export VERSION := 0.4.0
|
||||
|
||||
PRGDIR = programs
|
||||
ZSTDDIR = lib
|
||||
|
||||
.PHONY: clean
|
||||
# Define nul output
|
||||
ifneq (,$(filter Windows%,$(OS)))
|
||||
VOID = nul
|
||||
else
|
||||
VOID = /dev/null
|
||||
endif
|
||||
|
||||
.PHONY: default all zstdprogram clean install uninstall travis-install test clangtest gpptest armtest usan asan uasan
|
||||
|
||||
default: zstdprogram
|
||||
|
||||
@ -49,8 +56,8 @@ zstdprogram:
|
||||
$(MAKE) -C $(PRGDIR)
|
||||
|
||||
clean:
|
||||
$(MAKE) -C $(ZSTDDIR) $@
|
||||
$(MAKE) -C $(PRGDIR) $@
|
||||
@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
|
||||
@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
|
||||
@echo Cleaning completed
|
||||
|
||||
|
||||
|
||||
6
NEWS
6
NEWS
@ -1,3 +1,9 @@
|
||||
v0.4.0
|
||||
Command line utility is now compatible with high compression levels
|
||||
Removed zstdhc => merged into zstd
|
||||
Added : ZBUFF API (see zstd_buffered.h)
|
||||
Rolling buffer support
|
||||
|
||||
v0.3.6
|
||||
small blocks params
|
||||
|
||||
|
||||
13
lib/Makefile
13
lib/Makefile
@ -27,8 +27,8 @@
|
||||
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
#
|
||||
# You can contact the author at :
|
||||
# - ZSTD homepage : http://www.zstd.net
|
||||
# - ZSTD source repository : https://github.com/Cyan4973/zstd
|
||||
# - Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
# ################################################################
|
||||
|
||||
# Version numbers
|
||||
@ -63,14 +63,17 @@ else
|
||||
SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
|
||||
endif
|
||||
|
||||
default: libzstd
|
||||
|
||||
all: libzstd
|
||||
.PHONY: default all clean install uninstall
|
||||
|
||||
libzstd: zstd.c huff0.c fse.c
|
||||
default: clean libzstd
|
||||
|
||||
all: clean libzstd
|
||||
|
||||
libzstd: zstd_compress.c zstd_decompress.c huff0.c fse.c legacy/zstd_v01.c legacy/zstd_v02.c
|
||||
@echo compiling static library
|
||||
@$(CC) $(FLAGS) -c $^
|
||||
@$(AR) rcs libzstd.a zstd.o huff0.o fse.o
|
||||
@$(AR) rcs libzstd.a *.o
|
||||
@echo compiling dynamic library $(LIBVER)
|
||||
@$(CC) $(FLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER)
|
||||
@echo creating versioned links
|
||||
|
||||
@ -68,6 +68,7 @@ extern "C" {
|
||||
|
||||
#define ERROR_LIST(ITEM) \
|
||||
ITEM(PREFIX(No_Error)) ITEM(PREFIX(GENERIC)) \
|
||||
ITEM(PREFIX(mode_unsupported)) ITEM(PREFIX(init_missing))\
|
||||
ITEM(PREFIX(memory_allocation)) \
|
||||
ITEM(PREFIX(dstSize_tooSmall)) ITEM(PREFIX(srcSize_wrong)) \
|
||||
ITEM(PREFIX(prefix_unknown)) ITEM(PREFIX(corruption_detected)) \
|
||||
|
||||
@ -475,7 +475,7 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
|
||||
MEM_STATIC unsigned BIT_highbit32 (register U32 val)
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
unsigned long r;
|
||||
unsigned long r=0;
|
||||
_BitScanReverse ( &r, val );
|
||||
return (unsigned) r;
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
||||
|
||||
@ -49,12 +49,12 @@ extern "C" {
|
||||
/******************************************
|
||||
* Compiler-specific
|
||||
******************************************/
|
||||
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
||||
#if defined(__GNUC__)
|
||||
# define MEM_STATIC static __attribute__((unused))
|
||||
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
||||
# define MEM_STATIC static inline
|
||||
#elif defined(_MSC_VER)
|
||||
# define MEM_STATIC static __inline
|
||||
#elif defined(__GNUC__)
|
||||
# define MEM_STATIC static __attribute__((unused))
|
||||
#else
|
||||
# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
|
||||
#endif
|
||||
|
||||
@ -47,8 +47,8 @@ extern "C" {
|
||||
* Version
|
||||
***************************************/
|
||||
#define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */
|
||||
#define ZSTD_VERSION_MINOR 3 /* for new (non-breaking) interface capabilities */
|
||||
#define ZSTD_VERSION_RELEASE 6 /* for tweaks, bug-fixes, or development */
|
||||
#define ZSTD_VERSION_MINOR 4 /* for new (non-breaking) interface capabilities */
|
||||
#define ZSTD_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */
|
||||
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
|
||||
unsigned ZSTD_versionNumber (void);
|
||||
|
||||
@ -57,7 +57,8 @@ unsigned ZSTD_versionNumber (void);
|
||||
* Simple functions
|
||||
***************************************/
|
||||
size_t ZSTD_compress( void* dst, size_t maxDstSize,
|
||||
const void* src, size_t srcSize);
|
||||
const void* src, size_t srcSize,
|
||||
int compressionLevel);
|
||||
|
||||
size_t ZSTD_decompress( void* dst, size_t maxOriginalSize,
|
||||
const void* src, size_t compressedSize);
|
||||
@ -100,7 +101,7 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
|
||||
ZSTD_compressCCtx() :
|
||||
Same as ZSTD_compress(), but requires a ZSTD_CCtx working space already allocated
|
||||
*/
|
||||
size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
|
||||
size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
|
||||
542
lib/zstd_buffered.c
Normal file
542
lib/zstd_buffered.c
Normal file
@ -0,0 +1,542 @@
|
||||
/*
|
||||
Buffered version of Zstd compression library
|
||||
Copyright (C) 2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd source repository : https://github.com/Cyan4973/zstd
|
||||
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
|
||||
/* The objects defined into this file should be considered experimental.
|
||||
* They are not labelled stable, as their prototype may change in the future.
|
||||
* You can use them for tests, provide feedback, or if you can endure risk of future changes.
|
||||
*/
|
||||
|
||||
/* *************************************
|
||||
* Includes
|
||||
***************************************/
|
||||
#include <stdlib.h>
|
||||
#include "error.h"
|
||||
#include "zstd_static.h"
|
||||
#include "zstd_buffered_static.h"
|
||||
|
||||
|
||||
/** ************************************************
|
||||
* Streaming compression
|
||||
*
|
||||
* A ZBUFF_CCtx object is required to track streaming operation.
|
||||
* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources.
|
||||
* Use ZBUFF_compressInit() to start a new compression operation.
|
||||
* ZBUFF_CCtx objects can be reused multiple times.
|
||||
*
|
||||
* Use ZBUFF_compressContinue() repetitively to consume your input.
|
||||
* *srcSizePtr and *maxDstSizePtr can be any size.
|
||||
* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr.
|
||||
* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input.
|
||||
* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst .
|
||||
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer.
|
||||
* Note that it will not output more than *maxDstSizePtr.
|
||||
* Therefore, some content might still be left into its internal buffer if dst buffer is too small.
|
||||
* @return : nb of bytes still present into internal buffer (0 if it's empty)
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* ZBUFF_compressEnd() instructs to finish a frame.
|
||||
* It will perform a flush and write frame epilogue.
|
||||
* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *maxDstSizePtr is too small.
|
||||
* @return : nb of bytes still present into internal buffer (0 if it's empty)
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* Hint : recommended buffer sizes (not compulsory)
|
||||
* input : 128 KB block size is the internal unit, it improves latency to use this value.
|
||||
* output : ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block at best speed.
|
||||
* **************************************************/
|
||||
|
||||
typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage;
|
||||
|
||||
/* *** Ressources *** */
|
||||
struct ZBUFF_CCtx_s {
|
||||
ZSTD_CCtx* zc;
|
||||
char* inBuff;
|
||||
size_t inBuffSize;
|
||||
size_t inToCompress;
|
||||
size_t inBuffPos;
|
||||
size_t inBuffTarget;
|
||||
size_t blockSize;
|
||||
char* outBuff;
|
||||
size_t outBuffSize;
|
||||
size_t outBuffContentSize;
|
||||
size_t outBuffFlushedSize;
|
||||
ZBUFF_cStage stage;
|
||||
}; /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */
|
||||
|
||||
ZBUFF_CCtx* ZBUFF_createCCtx(void)
|
||||
{
|
||||
ZBUFF_CCtx* zbc = (ZBUFF_CCtx*)malloc(sizeof(ZBUFF_CCtx));
|
||||
if (zbc==NULL) return NULL;
|
||||
memset(zbc, 0, sizeof(*zbc));
|
||||
zbc->zc = ZSTD_createCCtx();
|
||||
return zbc;
|
||||
}
|
||||
|
||||
size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc)
|
||||
{
|
||||
if (zbc==NULL) return 0; /* support free on NULL */
|
||||
ZSTD_freeCCtx(zbc->zc);
|
||||
free(zbc->inBuff);
|
||||
free(zbc->outBuff);
|
||||
free(zbc);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/* *** Initialization *** */
|
||||
|
||||
#define MIN(a,b) ( ((a)<(b)) ? (a) : (b) )
|
||||
#define BLOCKSIZE (128 * 1024) /* a bit too "magic", should come from reference */
|
||||
size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, ZSTD_parameters params)
|
||||
{
|
||||
size_t neededInBuffSize;
|
||||
|
||||
ZSTD_validateParams(¶ms);
|
||||
neededInBuffSize = (size_t)1 << params.windowLog;
|
||||
|
||||
/* allocate buffers */
|
||||
if (zbc->inBuffSize < neededInBuffSize)
|
||||
{
|
||||
zbc->inBuffSize = neededInBuffSize;
|
||||
free(zbc->inBuff); /* should not be necessary */
|
||||
zbc->inBuff = (char*)malloc(neededInBuffSize);
|
||||
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
|
||||
}
|
||||
zbc->blockSize = MIN(BLOCKSIZE, zbc->inBuffSize);
|
||||
if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1)
|
||||
{
|
||||
zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1;
|
||||
free(zbc->outBuff); /* should not be necessary */
|
||||
zbc->outBuff = (char*)malloc(zbc->outBuffSize);
|
||||
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
|
||||
}
|
||||
|
||||
zbc->outBuffContentSize = ZSTD_compressBegin_advanced(zbc->zc, zbc->outBuff, zbc->outBuffSize, params);
|
||||
if (ZSTD_isError(zbc->outBuffContentSize)) return zbc->outBuffContentSize;
|
||||
|
||||
zbc->inToCompress = 0;
|
||||
zbc->inBuffPos = 0;
|
||||
zbc->inBuffTarget = zbc->blockSize;
|
||||
zbc->outBuffFlushedSize = 0;
|
||||
zbc->stage = ZBUFFcs_flush; /* starts by flushing the header */
|
||||
return 0; /* ready to go */
|
||||
}
|
||||
|
||||
size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel)
|
||||
{
|
||||
return ZBUFF_compressInit_advanced(zbc, ZSTD_getParams(compressionLevel, 0));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* *** Compression *** */
|
||||
|
||||
static size_t ZBUFF_limitCopy(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
size_t length = MIN(maxDstSize, srcSize);
|
||||
memcpy(dst, src, length);
|
||||
return length;
|
||||
}
|
||||
|
||||
static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
|
||||
void* dst, size_t* maxDstSizePtr,
|
||||
const void* src, size_t* srcSizePtr,
|
||||
int flush) /* aggregate : wait for full block before compressing */
|
||||
{
|
||||
U32 notDone = 1;
|
||||
const char* const istart = (const char*)src;
|
||||
const char* ip = istart;
|
||||
const char* const iend = istart + *srcSizePtr;
|
||||
char* const ostart = (char*)dst;
|
||||
char* op = ostart;
|
||||
char* const oend = ostart + *maxDstSizePtr;
|
||||
|
||||
while (notDone)
|
||||
{
|
||||
switch(zbc->stage)
|
||||
{
|
||||
case ZBUFFcs_init: return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */
|
||||
|
||||
case ZBUFFcs_load:
|
||||
/* complete inBuffer */
|
||||
{
|
||||
size_t toLoad = zbc->inBuffTarget - zbc->inBuffPos;
|
||||
size_t loaded = ZBUFF_limitCopy(zbc->inBuff + zbc->inBuffPos, toLoad, ip, iend-ip);
|
||||
zbc->inBuffPos += loaded;
|
||||
ip += loaded;
|
||||
if ( (zbc->inBuffPos==zbc->inToCompress) || (!flush && (toLoad != loaded)) )
|
||||
{ notDone = 0; break; } /* not enough input to get a full block : stop there, wait for more */
|
||||
}
|
||||
/* compress current block (note : this stage cannot be stopped in the middle) */
|
||||
{
|
||||
void* cDst;
|
||||
size_t cSize;
|
||||
size_t iSize = zbc->inBuffPos - zbc->inToCompress;
|
||||
if ((size_t)(oend-op) > ZSTD_compressBound(iSize))
|
||||
cDst = op; /* compress directly into output buffer (avoid flush stage) */
|
||||
else
|
||||
cDst = zbc->outBuff;
|
||||
cSize = ZSTD_compressContinue(zbc->zc, cDst, oend-op, zbc->inBuff + zbc->inToCompress, iSize);
|
||||
if (ZSTD_isError(cSize)) return cSize;
|
||||
/* prepare next block */
|
||||
zbc->inBuffTarget = zbc->inBuffPos + zbc->blockSize;
|
||||
if (zbc->inBuffTarget > zbc->inBuffSize)
|
||||
{ zbc->inBuffPos = 0; zbc->inBuffTarget = zbc->blockSize; } /* note : inBuffSize >= blockSize */
|
||||
zbc->inToCompress = zbc->inBuffPos;
|
||||
if (cDst == op) { op += cSize; break; } /* no need to flush */
|
||||
zbc->outBuffContentSize = cSize;
|
||||
zbc->outBuffFlushedSize = 0;
|
||||
zbc->stage = ZBUFFcs_flush;
|
||||
// break; /* flush stage follows */
|
||||
}
|
||||
|
||||
case ZBUFFcs_flush:
|
||||
/* flush into dst */
|
||||
{
|
||||
size_t toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
|
||||
size_t flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
|
||||
op += flushed;
|
||||
zbc->outBuffFlushedSize += flushed;
|
||||
if (toFlush!=flushed)
|
||||
{ notDone = 0; break; } /* not enough space within dst to store compressed block : stop there */
|
||||
zbc->outBuffContentSize = 0;
|
||||
zbc->outBuffFlushedSize = 0;
|
||||
zbc->stage = ZBUFFcs_load;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
*srcSizePtr = ip - istart;
|
||||
*maxDstSizePtr = op - ostart;
|
||||
{
|
||||
size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos;
|
||||
if (hintInSize==0) hintInSize = zbc->blockSize;
|
||||
return hintInSize;
|
||||
}
|
||||
}
|
||||
|
||||
size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc,
|
||||
void* dst, size_t* maxDstSizePtr,
|
||||
const void* src, size_t* srcSizePtr)
|
||||
{ return ZBUFF_compressContinue_generic(zbc, dst, maxDstSizePtr, src, srcSizePtr, 0); }
|
||||
|
||||
|
||||
|
||||
/* *** Finalize *** */
|
||||
|
||||
size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr)
|
||||
{
|
||||
size_t srcSize = 0;
|
||||
ZBUFF_compressContinue_generic(zbc, dst, maxDstSizePtr, NULL, &srcSize, 1);
|
||||
return zbc->outBuffContentSize - zbc->outBuffFlushedSize;
|
||||
}
|
||||
|
||||
|
||||
size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* maxDstSizePtr)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* op = ostart;
|
||||
BYTE* const oend = ostart + *maxDstSizePtr;
|
||||
size_t outSize = *maxDstSizePtr;
|
||||
size_t epilogueSize, remaining;
|
||||
ZBUFF_compressFlush(zbc, dst, &outSize); /* flush any remaining inBuff */
|
||||
op += outSize;
|
||||
epilogueSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff + zbc->outBuffContentSize, zbc->outBuffSize - zbc->outBuffContentSize); /* epilogue into outBuff */
|
||||
zbc->outBuffContentSize += epilogueSize;
|
||||
outSize = oend-op;
|
||||
zbc->stage = ZBUFFcs_flush;
|
||||
remaining = ZBUFF_compressFlush(zbc, op, &outSize); /* attempt to flush epilogue into dst */
|
||||
op += outSize;
|
||||
if (!remaining) zbc->stage = ZBUFFcs_init; /* close only if nothing left to flush */
|
||||
*maxDstSizePtr = op-ostart; /* tells how many bytes were written */
|
||||
return remaining;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/** ************************************************
|
||||
* Streaming decompression
|
||||
*
|
||||
* A ZBUFF_DCtx object is required to track streaming operation.
|
||||
* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
|
||||
* Use ZBUFF_decompressInit() to start a new decompression operation.
|
||||
* ZBUFF_DCtx objects can be reused multiple times.
|
||||
*
|
||||
* Use ZBUFF_decompressContinue() repetitively to consume your input.
|
||||
* *srcSizePtr and *maxDstSizePtr can be any size.
|
||||
* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr.
|
||||
* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input.
|
||||
* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst .
|
||||
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
|
||||
* or 0 when a frame is completely decoded
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* Hint : recommended buffer sizes (not compulsory)
|
||||
* output : 128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded.
|
||||
* input : just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
|
||||
* **************************************************/
|
||||
|
||||
typedef enum { ZBUFFds_init, ZBUFFds_readHeader, ZBUFFds_loadHeader, ZBUFFds_decodeHeader,
|
||||
ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage;
|
||||
|
||||
/* *** Resource management *** */
|
||||
|
||||
#define ZSTD_frameHeaderSize_max 5 /* too magical, should come from reference */
|
||||
struct ZBUFF_DCtx_s {
|
||||
ZSTD_DCtx* zc;
|
||||
ZSTD_parameters params;
|
||||
char* inBuff;
|
||||
size_t inBuffSize;
|
||||
size_t inPos;
|
||||
char* outBuff;
|
||||
size_t outBuffSize;
|
||||
size_t outStart;
|
||||
size_t outEnd;
|
||||
size_t hPos;
|
||||
ZBUFF_dStage stage;
|
||||
unsigned char headerBuffer[ZSTD_frameHeaderSize_max];
|
||||
}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */
|
||||
|
||||
|
||||
ZBUFF_DCtx* ZBUFF_createDCtx(void)
|
||||
{
|
||||
ZBUFF_DCtx* zbc = (ZBUFF_DCtx*)malloc(sizeof(ZBUFF_DCtx));
|
||||
if (zbc==NULL) return NULL;
|
||||
memset(zbc, 0, sizeof(*zbc));
|
||||
zbc->zc = ZSTD_createDCtx();
|
||||
zbc->stage = ZBUFFds_init;
|
||||
return zbc;
|
||||
}
|
||||
|
||||
size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbc)
|
||||
{
|
||||
if (zbc==NULL) return 0; /* support free on null */
|
||||
ZSTD_freeDCtx(zbc->zc);
|
||||
free(zbc->inBuff);
|
||||
free(zbc->outBuff);
|
||||
free(zbc);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* *** Initialization *** */
|
||||
|
||||
size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbc)
|
||||
{
|
||||
zbc->stage = ZBUFFds_readHeader;
|
||||
zbc->hPos = zbc->inPos = zbc->outStart = zbc->outEnd = 0;
|
||||
return ZSTD_resetDCtx(zbc->zc);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* *** Decompression *** */
|
||||
|
||||
size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr)
|
||||
{
|
||||
const char* const istart = (const char*)src;
|
||||
const char* ip = istart;
|
||||
const char* const iend = istart + *srcSizePtr;
|
||||
char* const ostart = (char*)dst;
|
||||
char* op = ostart;
|
||||
char* const oend = ostart + *maxDstSizePtr;
|
||||
U32 notDone = 1;
|
||||
|
||||
while (notDone)
|
||||
{
|
||||
switch(zbc->stage)
|
||||
{
|
||||
|
||||
case ZBUFFds_init :
|
||||
return ERROR(init_missing);
|
||||
|
||||
case ZBUFFds_readHeader :
|
||||
/* read header from src */
|
||||
{
|
||||
size_t headerSize = ZSTD_getFrameParams(&(zbc->params), src, *srcSizePtr);
|
||||
if (ZSTD_isError(headerSize)) return headerSize;
|
||||
if (headerSize)
|
||||
{
|
||||
/* not enough input to decode header : tell how many bytes would be necessary */
|
||||
memcpy(zbc->headerBuffer+zbc->hPos, src, *srcSizePtr);
|
||||
zbc->hPos += *srcSizePtr;
|
||||
*maxDstSizePtr = 0;
|
||||
zbc->stage = ZBUFFds_loadHeader;
|
||||
return headerSize - zbc->hPos;
|
||||
}
|
||||
zbc->stage = ZBUFFds_decodeHeader;
|
||||
break;
|
||||
}
|
||||
|
||||
case ZBUFFds_loadHeader:
|
||||
/* complete header from src */
|
||||
{
|
||||
size_t headerSize = ZBUFF_limitCopy(
|
||||
zbc->headerBuffer + zbc->hPos, ZSTD_frameHeaderSize_max - zbc->hPos,
|
||||
src, *srcSizePtr);
|
||||
zbc->hPos += headerSize;
|
||||
ip += headerSize;
|
||||
headerSize = ZSTD_getFrameParams(&(zbc->params), zbc->headerBuffer, zbc->hPos);
|
||||
if (ZSTD_isError(headerSize)) return headerSize;
|
||||
if (headerSize)
|
||||
{
|
||||
/* not enough input to decode header : tell how many bytes would be necessary */
|
||||
*maxDstSizePtr = 0;
|
||||
return headerSize - zbc->hPos;
|
||||
}
|
||||
// zbc->stage = ZBUFFds_decodeHeader; break; /* useless : stage follows */
|
||||
}
|
||||
|
||||
case ZBUFFds_decodeHeader:
|
||||
/* apply header to create / resize buffers */
|
||||
{
|
||||
size_t neededOutSize = (size_t)1 << zbc->params.windowLog;
|
||||
size_t neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */
|
||||
if (zbc->inBuffSize < neededInSize)
|
||||
{
|
||||
free(zbc->inBuff);
|
||||
zbc->inBuffSize = neededInSize;
|
||||
zbc->inBuff = (char*)malloc(neededInSize);
|
||||
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
|
||||
}
|
||||
if (zbc->outBuffSize < neededOutSize)
|
||||
{
|
||||
free(zbc->outBuff);
|
||||
zbc->outBuffSize = neededOutSize;
|
||||
zbc->outBuff = (char*)malloc(neededOutSize);
|
||||
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
|
||||
}
|
||||
}
|
||||
if (zbc->hPos)
|
||||
{
|
||||
/* some data already loaded into headerBuffer : transfer into inBuff */
|
||||
memcpy(zbc->inBuff, zbc->headerBuffer, zbc->hPos);
|
||||
zbc->inPos = zbc->hPos;
|
||||
zbc->hPos = 0;
|
||||
zbc->stage = ZBUFFds_load;
|
||||
break;
|
||||
}
|
||||
zbc->stage = ZBUFFds_read;
|
||||
|
||||
case ZBUFFds_read:
|
||||
{
|
||||
size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
|
||||
if (neededInSize==0) /* end of frame */
|
||||
{
|
||||
zbc->stage = ZBUFFds_init;
|
||||
notDone = 0;
|
||||
break;
|
||||
}
|
||||
if ((size_t)(iend-ip) >= neededInSize)
|
||||
{
|
||||
/* directly decode from src */
|
||||
size_t decodedSize = ZSTD_decompressContinue(zbc->zc,
|
||||
zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart,
|
||||
ip, neededInSize);
|
||||
if (ZSTD_isError(decodedSize)) return decodedSize;
|
||||
ip += neededInSize;
|
||||
if (!decodedSize) break; /* this was just a header */
|
||||
zbc->outEnd = zbc->outStart + decodedSize;
|
||||
zbc->stage = ZBUFFds_flush;
|
||||
break;
|
||||
}
|
||||
if (ip==iend) { notDone = 0; break; } /* no more input */
|
||||
zbc->stage = ZBUFFds_load;
|
||||
}
|
||||
|
||||
case ZBUFFds_load:
|
||||
{
|
||||
size_t neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
|
||||
size_t toLoad = neededInSize - zbc->inPos; /* should always be <= remaining space within inBuff */
|
||||
size_t loadedSize;
|
||||
if (toLoad > zbc->inBuffSize - zbc->inPos) return ERROR(corruption_detected); /* should never happen */
|
||||
loadedSize = ZBUFF_limitCopy(zbc->inBuff + zbc->inPos, toLoad, ip, iend-ip);
|
||||
ip += loadedSize;
|
||||
zbc->inPos += loadedSize;
|
||||
if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */
|
||||
{
|
||||
size_t decodedSize = ZSTD_decompressContinue(zbc->zc,
|
||||
zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart,
|
||||
zbc->inBuff, neededInSize);
|
||||
if (ZSTD_isError(decodedSize)) return decodedSize;
|
||||
zbc->inPos = 0; /* input is consumed */
|
||||
if (!decodedSize) { zbc->stage = ZBUFFds_read; break; } /* this was just a header */
|
||||
zbc->outEnd = zbc->outStart + decodedSize;
|
||||
zbc->stage = ZBUFFds_flush;
|
||||
// break; /* ZBUFFds_flush follows */
|
||||
}
|
||||
}
|
||||
case ZBUFFds_flush:
|
||||
{
|
||||
size_t toFlushSize = zbc->outEnd - zbc->outStart;
|
||||
size_t flushedSize = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outStart, toFlushSize);
|
||||
op += flushedSize;
|
||||
zbc->outStart += flushedSize;
|
||||
if (flushedSize == toFlushSize)
|
||||
{
|
||||
zbc->stage = ZBUFFds_read;
|
||||
if (zbc->outStart + BLOCKSIZE > zbc->outBuffSize)
|
||||
zbc->outStart = zbc->outEnd = 0;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
*srcSizePtr = ip-istart;
|
||||
*maxDstSizePtr = op-ostart;
|
||||
|
||||
return ZSTD_nextSrcSizeToDecompress(zbc->zc) - zbc->inPos;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Tool functions
|
||||
***************************************/
|
||||
unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
|
||||
const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
|
||||
|
||||
size_t ZBUFF_recommendedCInSize() { return BLOCKSIZE; }
|
||||
size_t ZBUFF_recommendedCOutSize() { return ZSTD_compressBound(BLOCKSIZE) + 6; }
|
||||
size_t ZBUFF_recommendedDInSize() { return BLOCKSIZE + 3; }
|
||||
size_t ZBUFF_recommendedDOutSize() { return BLOCKSIZE; }
|
||||
146
lib/zstd_buffered.h
Normal file
146
lib/zstd_buffered.h
Normal file
@ -0,0 +1,146 @@
|
||||
/*
|
||||
Buffered version of Zstd compression library
|
||||
Copyright (C) 2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd source repository : https://github.com/Cyan4973/zstd
|
||||
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
#ifndef ZSTD_BUFFERED_H
|
||||
#define ZSTD_BUFFERED_H
|
||||
|
||||
/* The objects defined into this file should be considered experimental.
|
||||
* They are not labelled stable, as their prototype may change in the future.
|
||||
* You can use them for tests, provide feedback, or if you can endure risk of future changes.
|
||||
*/
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* *************************************
|
||||
* Includes
|
||||
***************************************/
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Streaming functions
|
||||
***************************************/
|
||||
typedef struct ZBUFF_CCtx_s ZBUFF_CCtx;
|
||||
ZBUFF_CCtx* ZBUFF_createCCtx(void);
|
||||
size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx);
|
||||
|
||||
size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel);
|
||||
size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr);
|
||||
size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr);
|
||||
size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* maxDstSizePtr);
|
||||
|
||||
/** ************************************************
|
||||
* Streaming compression
|
||||
*
|
||||
* A ZBUFF_CCtx object is required to track streaming operation.
|
||||
* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources.
|
||||
* Use ZBUFF_compressInit() to start a new compression operation.
|
||||
* ZBUFF_CCtx objects can be reused multiple times.
|
||||
*
|
||||
* Use ZBUFF_compressContinue() repetitively to consume input stream.
|
||||
* *srcSizePtr and *maxDstSizePtr can be any size.
|
||||
* The function will report how many bytes were read or written within *srcSizePtr and *maxDstSizePtr.
|
||||
* Note that it may not consume the entire input, in which case it's up to the caller to present again remaining data.
|
||||
* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or move dst .
|
||||
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer.
|
||||
* Note that it will not output more than *maxDstSizePtr.
|
||||
* Therefore, some content might still be left into its internal buffer if dst buffer is too small.
|
||||
* @return : nb of bytes still present into internal buffer (0 if it's empty)
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* ZBUFF_compressEnd() instructs to finish a frame.
|
||||
* It will perform a flush and write frame epilogue.
|
||||
* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *maxDstSizePtr is too small.
|
||||
* In which case, call again ZBUFF_compressFlush() to complete the flush.
|
||||
* @return : nb of bytes still present into internal buffer (0 if it's empty)
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedCInSize / ZBUFF_recommendedCOutSize
|
||||
* input : ZBUFF_recommendedCInSize==128 KB block size is the internal unit, it improves latency to use this value.
|
||||
* output : ZBUFF_recommendedCOutSize==ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block. Skip some buffering.
|
||||
* By using both, you ensure that input will be entirely consumed, and output will always contain the result.
|
||||
* **************************************************/
|
||||
|
||||
|
||||
typedef struct ZBUFF_DCtx_s ZBUFF_DCtx;
|
||||
ZBUFF_DCtx* ZBUFF_createDCtx(void);
|
||||
size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx);
|
||||
|
||||
size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx);
|
||||
size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr);
|
||||
|
||||
/** ************************************************
|
||||
* Streaming decompression
|
||||
*
|
||||
* A ZBUFF_DCtx object is required to track streaming operation.
|
||||
* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
|
||||
* Use ZBUFF_decompressInit() to start a new decompression operation.
|
||||
* ZBUFF_DCtx objects can be reused multiple times.
|
||||
*
|
||||
* Use ZBUFF_decompressContinue() repetitively to consume your input.
|
||||
* *srcSizePtr and *maxDstSizePtr can be any size.
|
||||
* The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr.
|
||||
* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
|
||||
* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst.
|
||||
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
|
||||
* or 0 when a frame is completely decoded
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize / ZBUFF_recommendedDOutSize
|
||||
* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded.
|
||||
* input : ZBUFF_recommendedDInSize==128Kb+3; just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
|
||||
* **************************************************/
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Tool functions
|
||||
***************************************/
|
||||
unsigned ZBUFF_isError(size_t errorCode);
|
||||
const char* ZBUFF_getErrorName(size_t errorCode);
|
||||
|
||||
/** The below functions provide recommended buffer sizes for Compression or Decompression operations.
|
||||
* These sizes are not compulsory, they just tend to offer better latency */
|
||||
size_t ZBUFF_recommendedCInSize(void);
|
||||
size_t ZBUFF_recommendedCOutSize(void);
|
||||
size_t ZBUFF_recommendedDInSize(void);
|
||||
size_t ZBUFF_recommendedDOutSize(void);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_BUFFERED_H */
|
||||
@ -1,6 +1,6 @@
|
||||
/*
|
||||
zstdhc - high compression variant
|
||||
Header File
|
||||
zstd - buffered version of compression library
|
||||
experimental complementary API, for static linking only
|
||||
Copyright (C) 2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
@ -27,9 +27,16 @@
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd source repository : http://www.zstd.net
|
||||
- zstd source repository : https://github.com/Cyan4973/zstd
|
||||
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
#pragma once
|
||||
#ifndef ZSTD_BUFFERED_STATIC_H
|
||||
#define ZSTD_BUFFERED_STATIC_H
|
||||
|
||||
/* The objects defined into this file should be considered experimental.
|
||||
* They are not labelled stable, as their prototype may change in the future.
|
||||
* You can use them for tests, provide feedback, or if you can endure risk of future changes.
|
||||
*/
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
@ -38,39 +45,18 @@ extern "C" {
|
||||
/* *************************************
|
||||
* Includes
|
||||
***************************************/
|
||||
#include <stddef.h> /* size_t */
|
||||
#include "zstd_static.h"
|
||||
#include "zstd_buffered.h"
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Simple function
|
||||
* Advanced Streaming functions
|
||||
***************************************/
|
||||
/**
|
||||
ZSTD_HC_compress() :
|
||||
Compresses 'srcSize' bytes from buffer 'src' into buffer 'dst', of maximum size 'dstSize'.
|
||||
Destination buffer must be already allocated.
|
||||
Compression runs faster if maxDstSize >= ZSTD_compressBound(srcSize).
|
||||
@return : the number of bytes written into buffer 'dst'
|
||||
or an error code if it fails (which can be tested using ZSTD_isError())
|
||||
*/
|
||||
size_t ZSTD_HC_compress(void* dst, size_t maxDstSize,
|
||||
const void* src, size_t srcSize,
|
||||
int compressionLevel);
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Advanced functions
|
||||
***************************************/
|
||||
typedef struct ZSTD_HC_CCtx_s ZSTD_HC_CCtx; /* incomplete type */
|
||||
ZSTD_HC_CCtx* ZSTD_HC_createCCtx(void);
|
||||
size_t ZSTD_HC_freeCCtx(ZSTD_HC_CCtx* cctx);
|
||||
|
||||
/**
|
||||
ZSTD_HC_compressCCtx() :
|
||||
Same as ZSTD_compress(), but requires a ZSTD_HC_CCtx working space already allocated
|
||||
*/
|
||||
size_t ZSTD_HC_compressCCtx(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel);
|
||||
size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx, ZSTD_parameters params);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_BUFFERED_STATIC_H */
|
||||
2145
lib/zstd_compress.c
Normal file
2145
lib/zstd_compress.c
Normal file
File diff suppressed because it is too large
Load Diff
873
lib/zstd_decompress.c
Normal file
873
lib/zstd_decompress.c
Normal file
@ -0,0 +1,873 @@
|
||||
/*
|
||||
zstd - standard compression library
|
||||
Copyright (C) 2014-2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd source repository : https://github.com/Cyan4973/zstd
|
||||
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
|
||||
/* ***************************************************************
|
||||
* Tuning parameters
|
||||
*****************************************************************/
|
||||
/*!
|
||||
* HEAPMODE :
|
||||
* Select how default compression functions will allocate memory for their hash table,
|
||||
* in memory stack (0, fastest), or in memory heap (1, requires malloc())
|
||||
* Note that compression context is fairly large, as a consequence heap memory is recommended.
|
||||
*/
|
||||
#ifndef ZSTD_HEAPMODE
|
||||
# define ZSTD_HEAPMODE 1
|
||||
#endif /* ZSTD_HEAPMODE */
|
||||
|
||||
/*!
|
||||
* LEGACY_SUPPORT :
|
||||
* ZSTD_decompress() can decode older formats (starting from zstd 0.1+)
|
||||
*/
|
||||
#ifndef ZSTD_LEGACY_SUPPORT
|
||||
# define ZSTD_LEGACY_SUPPORT 1
|
||||
#endif
|
||||
|
||||
|
||||
/* *******************************************************
|
||||
* Includes
|
||||
*********************************************************/
|
||||
#include <stdlib.h> /* calloc */
|
||||
#include <string.h> /* memcpy, memmove */
|
||||
#include <stdio.h> /* debug : printf */
|
||||
#include "mem.h" /* low level memory routines */
|
||||
#include "zstd_static.h"
|
||||
#include "zstd_internal.h"
|
||||
#include "fse_static.h"
|
||||
#include "huff0.h"
|
||||
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
# include "zstd_legacy.h"
|
||||
#endif
|
||||
|
||||
/* *******************************************************
|
||||
* Compiler specifics
|
||||
*********************************************************/
|
||||
#ifdef _MSC_VER /* Visual Studio */
|
||||
# define FORCE_INLINE static __forceinline
|
||||
# include <intrin.h> /* For Visual 2005 */
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
# pragma warning(disable : 4324) /* disable: C4324: padded structure */
|
||||
#else
|
||||
# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
||||
# ifdef __GNUC__
|
||||
# define FORCE_INLINE static inline __attribute__((always_inline))
|
||||
# else
|
||||
# define FORCE_INLINE static inline
|
||||
# endif
|
||||
#endif
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Local types
|
||||
***************************************/
|
||||
typedef struct
|
||||
{
|
||||
blockType_t blockType;
|
||||
U32 origSize;
|
||||
} blockProperties_t;
|
||||
|
||||
|
||||
/* *******************************************************
|
||||
* Memory operations
|
||||
**********************************************************/
|
||||
static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Error Management
|
||||
***************************************/
|
||||
unsigned ZSTD_versionNumber (void) { return ZSTD_VERSION_NUMBER; }
|
||||
|
||||
/*! ZSTD_isError
|
||||
* tells if a return value is an error code */
|
||||
unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
|
||||
|
||||
/*! ZSTD_getErrorName
|
||||
* provides error code string (useful for debugging) */
|
||||
const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
||||
|
||||
|
||||
/* *************************************************************
|
||||
* Context management
|
||||
***************************************************************/
|
||||
typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
|
||||
ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock } ZSTD_dStage;
|
||||
|
||||
struct ZSTD_DCtx_s
|
||||
{
|
||||
U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
|
||||
U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
|
||||
U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
|
||||
void* previousDstEnd;
|
||||
void* base;
|
||||
void* vBase;
|
||||
void* dictEnd;
|
||||
size_t expected;
|
||||
size_t headerSize;
|
||||
ZSTD_parameters params;
|
||||
blockType_t bType;
|
||||
ZSTD_dStage stage;
|
||||
const BYTE* litPtr;
|
||||
size_t litBufSize;
|
||||
size_t litSize;
|
||||
BYTE litBuffer[BLOCKSIZE + 8 /* margin for wildcopy */];
|
||||
BYTE headerBuffer[ZSTD_frameHeaderSize_max];
|
||||
}; /* typedef'd to ZSTD_Dctx within "zstd_static.h" */
|
||||
|
||||
size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx)
|
||||
{
|
||||
dctx->expected = ZSTD_frameHeaderSize_min;
|
||||
dctx->stage = ZSTDds_getFrameHeaderSize;
|
||||
dctx->previousDstEnd = NULL;
|
||||
dctx->base = NULL;
|
||||
dctx->vBase = NULL;
|
||||
dctx->dictEnd = NULL;
|
||||
return 0;
|
||||
}
|
||||
|
||||
ZSTD_DCtx* ZSTD_createDCtx(void)
|
||||
{
|
||||
ZSTD_DCtx* dctx = (ZSTD_DCtx*)malloc(sizeof(ZSTD_DCtx));
|
||||
if (dctx==NULL) return NULL;
|
||||
ZSTD_resetDCtx(dctx);
|
||||
return dctx;
|
||||
}
|
||||
|
||||
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
|
||||
{
|
||||
free(dctx);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/* *************************************************************
|
||||
* Decompression section
|
||||
***************************************************************/
|
||||
/** ZSTD_decodeFrameHeader_Part1
|
||||
* decode the 1st part of the Frame Header, which tells Frame Header size.
|
||||
* srcSize must be == ZSTD_frameHeaderSize_min
|
||||
* @return : the full size of the Frame Header */
|
||||
static size_t ZSTD_decodeFrameHeader_Part1(ZSTD_DCtx* zc, const void* src, size_t srcSize)
|
||||
{
|
||||
U32 magicNumber;
|
||||
if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
|
||||
magicNumber = MEM_readLE32(src);
|
||||
if (magicNumber != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
|
||||
zc->headerSize = ZSTD_frameHeaderSize_min;
|
||||
return zc->headerSize;
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize)
|
||||
{
|
||||
U32 magicNumber;
|
||||
if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_max;
|
||||
magicNumber = MEM_readLE32(src);
|
||||
if (magicNumber != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
|
||||
memset(params, 0, sizeof(*params));
|
||||
params->windowLog = (((const BYTE*)src)[4] & 15) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/** ZSTD_decodeFrameHeader_Part2
|
||||
* decode the full Frame Header
|
||||
* srcSize must be the size provided by ZSTD_decodeFrameHeader_Part1
|
||||
* @return : 0, or an error code, which can be tested using ZSTD_isError() */
|
||||
static size_t ZSTD_decodeFrameHeader_Part2(ZSTD_DCtx* zc, const void* src, size_t srcSize)
|
||||
{
|
||||
if (srcSize != zc->headerSize) return ERROR(srcSize_wrong);
|
||||
return ZSTD_getFrameParams(&(zc->params), src, srcSize);
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
||||
{
|
||||
const BYTE* const in = (const BYTE* const)src;
|
||||
BYTE headerFlags;
|
||||
U32 cSize;
|
||||
|
||||
if (srcSize < 3) return ERROR(srcSize_wrong);
|
||||
|
||||
headerFlags = *in;
|
||||
cSize = in[2] + (in[1]<<8) + ((in[0] & 7)<<16);
|
||||
|
||||
bpPtr->blockType = (blockType_t)(headerFlags >> 6);
|
||||
bpPtr->origSize = (bpPtr->blockType == bt_rle) ? cSize : 0;
|
||||
|
||||
if (bpPtr->blockType == bt_end) return 0;
|
||||
if (bpPtr->blockType == bt_rle) return 1;
|
||||
return cSize;
|
||||
}
|
||||
|
||||
static size_t ZSTD_copyRawBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
||||
memcpy(dst, src, srcSize);
|
||||
return srcSize;
|
||||
}
|
||||
|
||||
|
||||
/** ZSTD_decompressLiterals
|
||||
@return : nb of bytes read from src, or an error code*/
|
||||
static size_t ZSTD_decompressLiterals(void* dst, size_t* maxDstSizePtr,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
|
||||
const size_t litSize = (MEM_readLE32(src) & 0x1FFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
|
||||
const size_t litCSize = (MEM_readLE32(ip+2) & 0xFFFFFF) >> 5; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
|
||||
|
||||
if (litSize > *maxDstSizePtr) return ERROR(corruption_detected);
|
||||
if (litCSize + 5 > srcSize) return ERROR(corruption_detected);
|
||||
|
||||
if (HUF_isError(HUF_decompress(dst, litSize, ip+5, litCSize))) return ERROR(corruption_detected);
|
||||
|
||||
*maxDstSizePtr = litSize;
|
||||
return litCSize + 5;
|
||||
}
|
||||
|
||||
|
||||
/** ZSTD_decodeLiteralsBlock
|
||||
@return : nb of bytes read from src (< srcSize ) */
|
||||
size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
||||
const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */
|
||||
{
|
||||
const BYTE* const istart = (const BYTE*) src;
|
||||
|
||||
/* any compressed block with literals segment must be at least this size */
|
||||
if (srcSize < MIN_CBLOCK_SIZE) return ERROR(corruption_detected);
|
||||
|
||||
switch(*istart & 3)
|
||||
{
|
||||
/* compressed */
|
||||
case 0:
|
||||
{
|
||||
size_t litSize = BLOCKSIZE;
|
||||
const size_t readSize = ZSTD_decompressLiterals(dctx->litBuffer, &litSize, src, srcSize);
|
||||
dctx->litPtr = dctx->litBuffer;
|
||||
dctx->litBufSize = BLOCKSIZE+8;
|
||||
dctx->litSize = litSize;
|
||||
return readSize; /* works if it's an error too */
|
||||
}
|
||||
case IS_RAW:
|
||||
{
|
||||
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
|
||||
if (litSize > srcSize-11) /* risk of reading too far with wildcopy */
|
||||
{
|
||||
if (litSize > srcSize-3) return ERROR(corruption_detected);
|
||||
memcpy(dctx->litBuffer, istart, litSize);
|
||||
dctx->litPtr = dctx->litBuffer;
|
||||
dctx->litBufSize = BLOCKSIZE+8;
|
||||
dctx->litSize = litSize;
|
||||
return litSize+3;
|
||||
}
|
||||
/* direct reference into compressed stream */
|
||||
dctx->litPtr = istart+3;
|
||||
dctx->litBufSize = srcSize-3;
|
||||
dctx->litSize = litSize;
|
||||
return litSize+3; }
|
||||
case IS_RLE:
|
||||
{
|
||||
const size_t litSize = (MEM_readLE32(istart) & 0xFFFFFF) >> 2; /* no buffer issue : srcSize >= MIN_CBLOCK_SIZE */
|
||||
if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
|
||||
memset(dctx->litBuffer, istart[3], litSize);
|
||||
dctx->litPtr = dctx->litBuffer;
|
||||
dctx->litBufSize = BLOCKSIZE+8;
|
||||
dctx->litSize = litSize;
|
||||
return 4;
|
||||
}
|
||||
default:
|
||||
return ERROR(corruption_detected); /* forbidden nominal case */
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
|
||||
FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* const istart = (const BYTE* const)src;
|
||||
const BYTE* ip = istart;
|
||||
const BYTE* const iend = istart + srcSize;
|
||||
U32 LLtype, Offtype, MLtype;
|
||||
U32 LLlog, Offlog, MLlog;
|
||||
size_t dumpsLength;
|
||||
|
||||
/* check */
|
||||
if (srcSize < 5) return ERROR(srcSize_wrong);
|
||||
|
||||
/* SeqHead */
|
||||
*nbSeq = MEM_readLE16(ip); ip+=2;
|
||||
LLtype = *ip >> 6;
|
||||
Offtype = (*ip >> 4) & 3;
|
||||
MLtype = (*ip >> 2) & 3;
|
||||
if (*ip & 2)
|
||||
{
|
||||
dumpsLength = ip[2];
|
||||
dumpsLength += ip[1] << 8;
|
||||
ip += 3;
|
||||
}
|
||||
else
|
||||
{
|
||||
dumpsLength = ip[1];
|
||||
dumpsLength += (ip[0] & 1) << 8;
|
||||
ip += 2;
|
||||
}
|
||||
*dumpsPtr = ip;
|
||||
ip += dumpsLength;
|
||||
*dumpsLengthPtr = dumpsLength;
|
||||
|
||||
/* check */
|
||||
if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
|
||||
|
||||
/* sequences */
|
||||
{
|
||||
S16 norm[MaxML+1]; /* assumption : MaxML >= MaxLL >= MaxOff */
|
||||
size_t headerSize;
|
||||
|
||||
/* Build DTables */
|
||||
switch(LLtype)
|
||||
{
|
||||
U32 max;
|
||||
case bt_rle :
|
||||
LLlog = 0;
|
||||
FSE_buildDTable_rle(DTableLL, *ip++); break;
|
||||
case bt_raw :
|
||||
LLlog = LLbits;
|
||||
FSE_buildDTable_raw(DTableLL, LLbits); break;
|
||||
default :
|
||||
max = MaxLL;
|
||||
headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
|
||||
if (FSE_isError(headerSize)) return ERROR(GENERIC);
|
||||
if (LLlog > LLFSELog) return ERROR(corruption_detected);
|
||||
ip += headerSize;
|
||||
FSE_buildDTable(DTableLL, norm, max, LLlog);
|
||||
}
|
||||
|
||||
switch(Offtype)
|
||||
{
|
||||
U32 max;
|
||||
case bt_rle :
|
||||
Offlog = 0;
|
||||
if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */
|
||||
FSE_buildDTable_rle(DTableOffb, *ip++ & MaxOff); /* if *ip > MaxOff, data is corrupted */
|
||||
break;
|
||||
case bt_raw :
|
||||
Offlog = Offbits;
|
||||
FSE_buildDTable_raw(DTableOffb, Offbits); break;
|
||||
default :
|
||||
max = MaxOff;
|
||||
headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
|
||||
if (FSE_isError(headerSize)) return ERROR(GENERIC);
|
||||
if (Offlog > OffFSELog) return ERROR(corruption_detected);
|
||||
ip += headerSize;
|
||||
FSE_buildDTable(DTableOffb, norm, max, Offlog);
|
||||
}
|
||||
|
||||
switch(MLtype)
|
||||
{
|
||||
U32 max;
|
||||
case bt_rle :
|
||||
MLlog = 0;
|
||||
if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */
|
||||
FSE_buildDTable_rle(DTableML, *ip++); break;
|
||||
case bt_raw :
|
||||
MLlog = MLbits;
|
||||
FSE_buildDTable_raw(DTableML, MLbits); break;
|
||||
default :
|
||||
max = MaxML;
|
||||
headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
|
||||
if (FSE_isError(headerSize)) return ERROR(GENERIC);
|
||||
if (MLlog > MLFSELog) return ERROR(corruption_detected);
|
||||
ip += headerSize;
|
||||
FSE_buildDTable(DTableML, norm, max, MLlog);
|
||||
}
|
||||
}
|
||||
|
||||
return ip-istart;
|
||||
}
|
||||
|
||||
|
||||
typedef struct {
|
||||
size_t litLength;
|
||||
size_t offset;
|
||||
size_t matchLength;
|
||||
} seq_t;
|
||||
|
||||
typedef struct {
|
||||
BIT_DStream_t DStream;
|
||||
FSE_DState_t stateLL;
|
||||
FSE_DState_t stateOffb;
|
||||
FSE_DState_t stateML;
|
||||
size_t prevOffset;
|
||||
const BYTE* dumps;
|
||||
const BYTE* dumpsEnd;
|
||||
} seqState_t;
|
||||
|
||||
|
||||
static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
||||
{
|
||||
size_t litLength;
|
||||
size_t prevOffset;
|
||||
size_t offset;
|
||||
size_t matchLength;
|
||||
const BYTE* dumps = seqState->dumps;
|
||||
const BYTE* const de = seqState->dumpsEnd;
|
||||
|
||||
/* Literal length */
|
||||
litLength = FSE_decodeSymbol(&(seqState->stateLL), &(seqState->DStream));
|
||||
prevOffset = litLength ? seq->offset : seqState->prevOffset;
|
||||
if (litLength == MaxLL)
|
||||
{
|
||||
U32 add = *dumps++;
|
||||
if (add < 255) litLength += add;
|
||||
else
|
||||
{
|
||||
litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
|
||||
dumps += 3;
|
||||
}
|
||||
if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
|
||||
}
|
||||
|
||||
/* Offset */
|
||||
{
|
||||
static const U32 offsetPrefix[MaxOff+1] = {
|
||||
1 /*fake*/, 1, 2, 4, 8, 16, 32, 64, 128, 256,
|
||||
512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144,
|
||||
524288, 1048576, 2097152, 4194304, 8388608, 16777216, 33554432, /*fake*/ 1, 1, 1, 1, 1 };
|
||||
U32 offsetCode, nbBits;
|
||||
offsetCode = FSE_decodeSymbol(&(seqState->stateOffb), &(seqState->DStream)); /* <= maxOff, by table construction */
|
||||
if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream));
|
||||
nbBits = offsetCode - 1;
|
||||
if (offsetCode==0) nbBits = 0; /* cmove */
|
||||
offset = offsetPrefix[offsetCode] + BIT_readBits(&(seqState->DStream), nbBits);
|
||||
if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream));
|
||||
if (offsetCode==0) offset = prevOffset; /* cmove */
|
||||
if (offsetCode | !litLength) seqState->prevOffset = seq->offset; /* cmove */
|
||||
}
|
||||
|
||||
/* MatchLength */
|
||||
matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
|
||||
if (matchLength == MaxML)
|
||||
{
|
||||
U32 add = *dumps++;
|
||||
if (add < 255) matchLength += add;
|
||||
else
|
||||
{
|
||||
matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
|
||||
dumps += 3;
|
||||
}
|
||||
if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */
|
||||
}
|
||||
matchLength += MINMATCH;
|
||||
|
||||
/* save result */
|
||||
seq->litLength = litLength;
|
||||
seq->offset = offset;
|
||||
seq->matchLength = matchLength;
|
||||
seqState->dumps = dumps;
|
||||
}
|
||||
|
||||
|
||||
FORCE_INLINE size_t ZSTD_execSequence(BYTE* op,
|
||||
BYTE* const oend, seq_t sequence,
|
||||
const BYTE** litPtr, const BYTE* const litLimit_8,
|
||||
BYTE* const base, BYTE* const vBase, BYTE* const dictEnd)
|
||||
{
|
||||
static const int dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
|
||||
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */
|
||||
BYTE* const oLitEnd = op + sequence.litLength;
|
||||
const size_t sequenceLength = sequence.litLength + sequence.matchLength;
|
||||
BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
|
||||
BYTE* const oend_8 = oend-8;
|
||||
const BYTE* const litEnd = *litPtr + sequence.litLength;
|
||||
const BYTE* match = oLitEnd - sequence.offset;
|
||||
|
||||
/* check */
|
||||
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
|
||||
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
||||
if (litEnd > litLimit_8) return ERROR(corruption_detected); /* risk read beyond lit buffer */
|
||||
|
||||
/* copy Literals */
|
||||
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */
|
||||
op = oLitEnd;
|
||||
*litPtr = litEnd; /* update for next sequence */
|
||||
|
||||
/* copy Match */
|
||||
/* check */
|
||||
//if (match > oLitEnd) return ERROR(corruption_detected); /* address space overflow test (is clang optimizer wrongly removing this test ?) */
|
||||
if (sequence.offset > (size_t)oLitEnd) return ERROR(corruption_detected); /* address space overflow test (this test seems preserved by clang optimizer) */
|
||||
|
||||
if (match < base)
|
||||
{
|
||||
/* offset beyond prefix */
|
||||
if (match < vBase) return ERROR(corruption_detected);
|
||||
match = dictEnd - (base-match);
|
||||
if (match + sequence.matchLength <= dictEnd)
|
||||
{
|
||||
memcpy(oLitEnd, match, sequence.matchLength);
|
||||
return sequenceLength;
|
||||
}
|
||||
/* span extDict & currentPrefixSegment */
|
||||
{
|
||||
size_t length1 = dictEnd - match;
|
||||
memcpy(oLitEnd, match, length1);
|
||||
op = oLitEnd + length1;
|
||||
sequence.matchLength -= length1;
|
||||
match = base;
|
||||
}
|
||||
}
|
||||
|
||||
/* match within prefix */
|
||||
if (sequence.offset < 8)
|
||||
{
|
||||
/* close range match, overlap */
|
||||
const int sub2 = dec64table[sequence.offset];
|
||||
op[0] = match[0];
|
||||
op[1] = match[1];
|
||||
op[2] = match[2];
|
||||
op[3] = match[3];
|
||||
match += dec32table[sequence.offset];
|
||||
ZSTD_copy4(op+4, match);
|
||||
match -= sub2;
|
||||
}
|
||||
else
|
||||
{
|
||||
ZSTD_copy8(op, match);
|
||||
}
|
||||
op += 8; match += 8;
|
||||
|
||||
if (oMatchEnd > oend-12)
|
||||
{
|
||||
if (op < oend_8)
|
||||
{
|
||||
ZSTD_wildcopy(op, match, oend_8 - op);
|
||||
match += oend_8 - op;
|
||||
op = oend_8;
|
||||
}
|
||||
while (op < oMatchEnd) *op++ = *match++;
|
||||
}
|
||||
else
|
||||
{
|
||||
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
|
||||
}
|
||||
return sequenceLength;
|
||||
}
|
||||
|
||||
|
||||
static size_t ZSTD_decompressSequences(
|
||||
ZSTD_DCtx* dctx,
|
||||
void* dst, size_t maxDstSize,
|
||||
const void* seqStart, size_t seqSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)seqStart;
|
||||
const BYTE* const iend = ip + seqSize;
|
||||
BYTE* const ostart = (BYTE* const)dst;
|
||||
BYTE* op = ostart;
|
||||
BYTE* const oend = ostart + maxDstSize;
|
||||
size_t errorCode, dumpsLength;
|
||||
const BYTE* litPtr = dctx->litPtr;
|
||||
const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
|
||||
const BYTE* const litEnd = litPtr + dctx->litSize;
|
||||
int nbSeq;
|
||||
const BYTE* dumps;
|
||||
U32* DTableLL = dctx->LLTable;
|
||||
U32* DTableML = dctx->MLTable;
|
||||
U32* DTableOffb = dctx->OffTable;
|
||||
BYTE* const base = (BYTE*) (dctx->base);
|
||||
BYTE* const vBase = (BYTE*) (dctx->vBase);
|
||||
BYTE* const dictEnd = (BYTE*) (dctx->dictEnd);
|
||||
|
||||
/* Build Decoding Tables */
|
||||
errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength,
|
||||
DTableLL, DTableML, DTableOffb,
|
||||
ip, iend-ip);
|
||||
if (ZSTD_isError(errorCode)) return errorCode;
|
||||
ip += errorCode;
|
||||
|
||||
/* Regen sequences */
|
||||
{
|
||||
seq_t sequence;
|
||||
seqState_t seqState;
|
||||
|
||||
memset(&sequence, 0, sizeof(sequence));
|
||||
sequence.offset = 4;
|
||||
seqState.dumps = dumps;
|
||||
seqState.dumpsEnd = dumps + dumpsLength;
|
||||
seqState.prevOffset = 4;
|
||||
errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip);
|
||||
if (ERR_isError(errorCode)) return ERROR(corruption_detected);
|
||||
FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL);
|
||||
FSE_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb);
|
||||
FSE_initDState(&(seqState.stateML), &(seqState.DStream), DTableML);
|
||||
|
||||
for ( ; (BIT_reloadDStream(&(seqState.DStream)) < BIT_DStream_completed) ; )
|
||||
{
|
||||
size_t oneSeqSize;
|
||||
nbSeq--;
|
||||
ZSTD_decodeSequence(&sequence, &seqState);
|
||||
oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
|
||||
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
||||
op += oneSeqSize;
|
||||
}
|
||||
|
||||
if (nbSeq<0) return ERROR(corruption_detected); /* requested too many sequences : data is corrupted */
|
||||
|
||||
/* now BIT_reloadDStream(&(seqState.DStream)) >= BIT_DStream_completed) */
|
||||
for ( ; (BIT_reloadDStream(&(seqState.DStream)) == BIT_DStream_completed) && nbSeq ; )
|
||||
{
|
||||
size_t oneSeqSize;
|
||||
nbSeq--;
|
||||
ZSTD_decodeSequence(&sequence, &seqState);
|
||||
oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
|
||||
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
||||
op += oneSeqSize;
|
||||
}
|
||||
|
||||
/* check if reached exact end */
|
||||
if ( !BIT_endOfDStream(&(seqState.DStream)) ) return ERROR(corruption_detected); /* DStream should be entirely and precisely consumed; otherwise data is corrupted */
|
||||
|
||||
/* last literal segment */
|
||||
{
|
||||
size_t lastLLSize = litEnd - litPtr;
|
||||
if (litPtr > litEnd) return ERROR(corruption_detected);
|
||||
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
|
||||
if (op != litPtr) memcpy(op, litPtr, lastLLSize);
|
||||
op += lastLLSize;
|
||||
}
|
||||
}
|
||||
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
|
||||
static size_t ZSTD_decompressBlock(
|
||||
ZSTD_DCtx* dctx,
|
||||
void* dst, size_t maxDstSize,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
/* blockType == blockCompressed */
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
|
||||
/* Decode literals sub-block */
|
||||
size_t litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
|
||||
if (ZSTD_isError(litCSize)) return litCSize;
|
||||
ip += litCSize;
|
||||
srcSize -= litCSize;
|
||||
|
||||
return ZSTD_decompressSequences(dctx, dst, maxDstSize, ip, srcSize);
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
const BYTE* iend = ip + srcSize;
|
||||
BYTE* const ostart = (BYTE* const)dst;
|
||||
BYTE* op = ostart;
|
||||
BYTE* const oend = ostart + maxDstSize;
|
||||
size_t remainingSize = srcSize;
|
||||
blockProperties_t blockProperties;
|
||||
|
||||
|
||||
/* init */
|
||||
ctx->base = ctx->vBase = ctx->dictEnd = dst;
|
||||
|
||||
/* Frame Header */
|
||||
{
|
||||
size_t frameHeaderSize;
|
||||
if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
{
|
||||
const U32 magicNumber = MEM_readLE32(src);
|
||||
if (ZSTD_isLegacy(magicNumber))
|
||||
return ZSTD_decompressLegacy(dst, maxDstSize, src, srcSize, magicNumber);
|
||||
}
|
||||
#endif
|
||||
frameHeaderSize = ZSTD_decodeFrameHeader_Part1(ctx, src, ZSTD_frameHeaderSize_min);
|
||||
if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
|
||||
if (srcSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
ip += frameHeaderSize; remainingSize -= frameHeaderSize;
|
||||
frameHeaderSize = ZSTD_decodeFrameHeader_Part2(ctx, src, frameHeaderSize);
|
||||
if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
|
||||
}
|
||||
|
||||
/* Loop on each block */
|
||||
while (1)
|
||||
{
|
||||
size_t decodedSize=0;
|
||||
size_t cBlockSize = ZSTD_getcBlockSize(ip, iend-ip, &blockProperties);
|
||||
if (ZSTD_isError(cBlockSize)) return cBlockSize;
|
||||
|
||||
ip += ZSTD_blockHeaderSize;
|
||||
remainingSize -= ZSTD_blockHeaderSize;
|
||||
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
|
||||
|
||||
switch(blockProperties.blockType)
|
||||
{
|
||||
case bt_compressed:
|
||||
decodedSize = ZSTD_decompressBlock(ctx, op, oend-op, ip, cBlockSize);
|
||||
break;
|
||||
case bt_raw :
|
||||
decodedSize = ZSTD_copyRawBlock(op, oend-op, ip, cBlockSize);
|
||||
break;
|
||||
case bt_rle :
|
||||
return ERROR(GENERIC); /* not yet supported */
|
||||
break;
|
||||
case bt_end :
|
||||
/* end of frame */
|
||||
if (remainingSize) return ERROR(srcSize_wrong);
|
||||
break;
|
||||
default:
|
||||
return ERROR(GENERIC); /* impossible */
|
||||
}
|
||||
if (cBlockSize == 0) break; /* bt_end */
|
||||
|
||||
if (ZSTD_isError(decodedSize)) return decodedSize;
|
||||
op += decodedSize;
|
||||
ip += cBlockSize;
|
||||
remainingSize -= cBlockSize;
|
||||
}
|
||||
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
ZSTD_DCtx ctx;
|
||||
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
||||
}
|
||||
|
||||
|
||||
/* ******************************
|
||||
* Streaming Decompression API
|
||||
********************************/
|
||||
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx)
|
||||
{
|
||||
return dctx->expected;
|
||||
}
|
||||
|
||||
size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
/* Sanity check */
|
||||
if (srcSize != ctx->expected) return ERROR(srcSize_wrong);
|
||||
if (dst != ctx->previousDstEnd) /* not contiguous */
|
||||
{
|
||||
ctx->dictEnd = ctx->previousDstEnd;
|
||||
if ((dst > ctx->base) && (dst < ctx->previousDstEnd)) /* rolling buffer : new segment right into tracked memory */
|
||||
ctx->base = (char*)dst + maxDstSize; /* temporary affectation, for vBase calculation */
|
||||
ctx->vBase = (char*)dst - ((char*)(ctx->dictEnd) - (char*)(ctx->base));
|
||||
ctx->base = dst;
|
||||
}
|
||||
|
||||
/* Decompress : frame header; part 1 */
|
||||
switch (ctx->stage)
|
||||
{
|
||||
case ZSTDds_getFrameHeaderSize :
|
||||
{
|
||||
/* get frame header size */
|
||||
if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */
|
||||
ctx->headerSize = ZSTD_decodeFrameHeader_Part1(ctx, src, ZSTD_frameHeaderSize_min);
|
||||
if (ZSTD_isError(ctx->headerSize)) return ctx->headerSize;
|
||||
memcpy(ctx->headerBuffer, src, ZSTD_frameHeaderSize_min);
|
||||
if (ctx->headerSize > ZSTD_frameHeaderSize_min)
|
||||
{
|
||||
ctx->expected = ctx->headerSize - ZSTD_frameHeaderSize_min;
|
||||
ctx->stage = ZSTDds_decodeFrameHeader;
|
||||
return 0;
|
||||
}
|
||||
ctx->expected = 0; /* not necessary to copy more */
|
||||
}
|
||||
case ZSTDds_decodeFrameHeader:
|
||||
{
|
||||
/* get frame header */
|
||||
size_t result;
|
||||
memcpy(ctx->headerBuffer + ZSTD_frameHeaderSize_min, src, ctx->expected);
|
||||
result = ZSTD_decodeFrameHeader_Part2(ctx, ctx->headerBuffer, ctx->headerSize);
|
||||
if (ZSTD_isError(result)) return result;
|
||||
ctx->expected = ZSTD_blockHeaderSize;
|
||||
ctx->stage = ZSTDds_decodeBlockHeader;
|
||||
return 0;
|
||||
}
|
||||
case ZSTDds_decodeBlockHeader:
|
||||
{
|
||||
/* Decode block header */
|
||||
blockProperties_t bp;
|
||||
size_t blockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
|
||||
if (ZSTD_isError(blockSize)) return blockSize;
|
||||
if (bp.blockType == bt_end)
|
||||
{
|
||||
ctx->expected = 0;
|
||||
ctx->stage = ZSTDds_getFrameHeaderSize;
|
||||
}
|
||||
else
|
||||
{
|
||||
ctx->expected = blockSize;
|
||||
ctx->bType = bp.blockType;
|
||||
ctx->stage = ZSTDds_decompressBlock;
|
||||
}
|
||||
|
||||
ctx->previousDstEnd = dst;
|
||||
return 0;
|
||||
}
|
||||
case 3:
|
||||
{
|
||||
/* Decompress : block content */
|
||||
size_t rSize;
|
||||
switch(ctx->bType)
|
||||
{
|
||||
case bt_compressed:
|
||||
rSize = ZSTD_decompressBlock(ctx, dst, maxDstSize, src, srcSize);
|
||||
break;
|
||||
case bt_raw :
|
||||
rSize = ZSTD_copyRawBlock(dst, maxDstSize, src, srcSize);
|
||||
break;
|
||||
case bt_rle :
|
||||
return ERROR(GENERIC); /* not yet handled */
|
||||
break;
|
||||
case bt_end : /* should never happen (filtered at phase 1) */
|
||||
rSize = 0;
|
||||
break;
|
||||
default:
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
ctx->stage = ZSTDds_decodeBlockHeader;
|
||||
ctx->expected = ZSTD_blockHeaderSize;
|
||||
ctx->previousDstEnd = (char*)dst + rSize;
|
||||
return rSize;
|
||||
}
|
||||
default:
|
||||
return ERROR(GENERIC); /* impossible */
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -44,123 +44,61 @@ extern "C" {
|
||||
#include "error.h"
|
||||
|
||||
|
||||
/* **************************************
|
||||
* Function body to include for inlining
|
||||
****************************************/
|
||||
static size_t ZSTD_read_ARCH(const void* p) { size_t r; memcpy(&r, p, sizeof(r)); return r; }
|
||||
|
||||
/* *************************************
|
||||
* Common macros
|
||||
***************************************/
|
||||
#define MIN(a,b) ((a)<(b) ? (a) : (b))
|
||||
|
||||
static unsigned ZSTD_highbit(U32 val)
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
unsigned long r=0;
|
||||
_BitScanReverse(&r, val);
|
||||
return (unsigned)r;
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
|
||||
return 31 - __builtin_clz(val);
|
||||
# else /* Software version */
|
||||
static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
||||
U32 v = val;
|
||||
int r;
|
||||
v |= v >> 1;
|
||||
v |= v >> 2;
|
||||
v |= v >> 4;
|
||||
v |= v >> 8;
|
||||
v |= v >> 16;
|
||||
r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
|
||||
return r;
|
||||
# endif
|
||||
}
|
||||
|
||||
MEM_STATIC unsigned ZSTD_NbCommonBytes (register size_t val)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
{
|
||||
if (MEM_64bits())
|
||||
{
|
||||
# if defined(_MSC_VER) && defined(_WIN64)
|
||||
unsigned long r = 0;
|
||||
_BitScanForward64( &r, (U64)val );
|
||||
return (int)(r>>3);
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
return (__builtin_ctzll((U64)val) >> 3);
|
||||
# else
|
||||
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
|
||||
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
|
||||
# endif
|
||||
}
|
||||
else /* 32 bits */
|
||||
{
|
||||
# if defined(_MSC_VER)
|
||||
unsigned long r=0;
|
||||
_BitScanForward( &r, (U32)val );
|
||||
return (int)(r>>3);
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
return (__builtin_ctz((U32)val) >> 3);
|
||||
# else
|
||||
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
|
||||
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
|
||||
# endif
|
||||
}
|
||||
}
|
||||
else /* Big Endian CPU */
|
||||
{
|
||||
if (MEM_32bits())
|
||||
{
|
||||
# if defined(_MSC_VER) && defined(_WIN64)
|
||||
unsigned long r = 0;
|
||||
_BitScanReverse64( &r, val );
|
||||
return (unsigned)(r>>3);
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
return (__builtin_clzll(val) >> 3);
|
||||
# else
|
||||
unsigned r;
|
||||
const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
|
||||
if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
|
||||
if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
|
||||
r += (!val);
|
||||
return r;
|
||||
# endif
|
||||
}
|
||||
else /* 32 bits */
|
||||
{
|
||||
# if defined(_MSC_VER)
|
||||
unsigned long r = 0;
|
||||
_BitScanReverse( &r, (unsigned long)val );
|
||||
return (unsigned)(r>>3);
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
return (__builtin_clz((U32)val) >> 3);
|
||||
# else
|
||||
unsigned r;
|
||||
if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
|
||||
r += (!val);
|
||||
return r;
|
||||
# endif
|
||||
}
|
||||
}
|
||||
}
|
||||
#define MAX(a,b) ((a)>(b) ? (a) : (b))
|
||||
|
||||
|
||||
MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
|
||||
{
|
||||
const BYTE* const pStart = pIn;
|
||||
/* *************************************
|
||||
* Common constants
|
||||
***************************************/
|
||||
#define ZSTD_MAGICNUMBER 0xFD2FB524 /* v0.4 */
|
||||
|
||||
while ((pIn<pInLimit-(sizeof(size_t)-1)))
|
||||
{
|
||||
size_t diff = ZSTD_read_ARCH(pMatch) ^ ZSTD_read_ARCH(pIn);
|
||||
if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
|
||||
pIn += ZSTD_NbCommonBytes(diff);
|
||||
return (size_t)(pIn - pStart);
|
||||
}
|
||||
#define KB *(1 <<10)
|
||||
#define MB *(1 <<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
if (MEM_64bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
|
||||
if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
|
||||
if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
|
||||
return (size_t)(pIn - pStart);
|
||||
}
|
||||
#define BLOCKSIZE (128 KB) /* define, for static allocation */
|
||||
|
||||
static const size_t ZSTD_blockHeaderSize = 3;
|
||||
static const size_t ZSTD_frameHeaderSize_min = 5;
|
||||
#define ZSTD_frameHeaderSize_max 5 /* define, for static allocation */
|
||||
|
||||
#define BIT7 128
|
||||
#define BIT6 64
|
||||
#define BIT5 32
|
||||
#define BIT4 16
|
||||
#define BIT1 2
|
||||
#define BIT0 1
|
||||
|
||||
#define IS_RAW BIT0
|
||||
#define IS_RLE BIT1
|
||||
|
||||
#define MINMATCH 4
|
||||
#define REPCODE_STARTVALUE 4
|
||||
|
||||
#define MLbits 7
|
||||
#define LLbits 6
|
||||
#define Offbits 5
|
||||
#define MaxML ((1<<MLbits) - 1)
|
||||
#define MaxLL ((1<<LLbits) - 1)
|
||||
#define MaxOff ((1<<Offbits)- 1)
|
||||
#define MLFSELog 10
|
||||
#define LLFSELog 10
|
||||
#define OffFSELog 9
|
||||
#define MaxSeq MAX(MaxLL, MaxML)
|
||||
|
||||
#define MIN_SEQUENCES_SIZE (2 /*seqNb*/ + 2 /*dumps*/ + 3 /*seqTables*/ + 1 /*bitStream*/)
|
||||
#define MIN_CBLOCK_SIZE (3 /*litCSize*/ + MIN_SEQUENCES_SIZE)
|
||||
|
||||
typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
|
||||
|
||||
|
||||
/* ******************************************
|
||||
* Shared functions to include for inlining
|
||||
********************************************/
|
||||
static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
|
||||
|
||||
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
|
||||
@ -177,104 +115,6 @@ static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
|
||||
}
|
||||
|
||||
|
||||
typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
|
||||
|
||||
typedef struct
|
||||
{
|
||||
blockType_t blockType;
|
||||
U32 origSize;
|
||||
} blockProperties_t;
|
||||
|
||||
size_t ZSTD_noCompressBlock(void* op, size_t maxDstSize, const void* ip, size_t blockSize);
|
||||
|
||||
|
||||
typedef struct {
|
||||
void* buffer;
|
||||
U32* offsetStart;
|
||||
U32* offset;
|
||||
BYTE* offCodeStart;
|
||||
BYTE* offCode;
|
||||
BYTE* litStart;
|
||||
BYTE* lit;
|
||||
BYTE* litLengthStart;
|
||||
BYTE* litLength;
|
||||
BYTE* matchLengthStart;
|
||||
BYTE* matchLength;
|
||||
BYTE* dumpsStart;
|
||||
BYTE* dumps;
|
||||
} seqStore_t;
|
||||
|
||||
void ZSTD_resetSeqStore(seqStore_t* ssPtr);
|
||||
|
||||
static const U32 g_searchStrength = 8;
|
||||
|
||||
#define REPCODE_STARTVALUE 4
|
||||
#define MLbits 7
|
||||
#define LLbits 6
|
||||
#define Offbits 5
|
||||
#define MaxML ((1<<MLbits) - 1)
|
||||
#define MaxLL ((1<<LLbits) - 1)
|
||||
#define MaxOff 31
|
||||
|
||||
#define MIN_SEQUENCES_SIZE (2 /*seqNb*/ + 2 /*dumps*/ + 3 /*seqTables*/ + 1 /*bitStream*/)
|
||||
#define MIN_CBLOCK_SIZE (3 /*litCSize*/ + MIN_SEQUENCES_SIZE)
|
||||
|
||||
/** ZSTD_storeSeq
|
||||
Store a sequence (literal length, literals, offset code and match length) into seqStore_t
|
||||
@offsetCode : distance to match, or 0 == repCode
|
||||
@matchCode : matchLength - MINMATCH
|
||||
*/
|
||||
MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, size_t offsetCode, size_t matchCode)
|
||||
{
|
||||
#if 0
|
||||
static const BYTE* g_start = NULL;
|
||||
if (g_start==NULL) g_start = literals;
|
||||
if (literals - g_start == 8695)
|
||||
printf("pos %6u : %3u literals & match %3u bytes at distance %6u \n",
|
||||
(U32)(literals - g_start), (U32)litLength, (U32)matchCode+4, (U32)offsetCode);
|
||||
#endif
|
||||
|
||||
/* copy Literals */
|
||||
ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
|
||||
seqStorePtr->lit += litLength;
|
||||
|
||||
/* literal Length */
|
||||
if (litLength >= MaxLL)
|
||||
{
|
||||
*(seqStorePtr->litLength++) = MaxLL;
|
||||
if (litLength<255 + MaxLL)
|
||||
*(seqStorePtr->dumps++) = (BYTE)(litLength - MaxLL);
|
||||
else
|
||||
{
|
||||
*(seqStorePtr->dumps++) = 255;
|
||||
MEM_writeLE32(seqStorePtr->dumps, (U32)litLength); seqStorePtr->dumps += 3;
|
||||
}
|
||||
}
|
||||
else *(seqStorePtr->litLength++) = (BYTE)litLength;
|
||||
|
||||
/* match offset */
|
||||
*(seqStorePtr->offset++) = (U32)offsetCode;
|
||||
|
||||
/* match Length */
|
||||
if (matchCode >= MaxML)
|
||||
{
|
||||
*(seqStorePtr->matchLength++) = MaxML;
|
||||
if (matchCode < 255+MaxML)
|
||||
*(seqStorePtr->dumps++) = (BYTE)(matchCode - MaxML);
|
||||
else
|
||||
{
|
||||
*(seqStorePtr->dumps++) = 255;
|
||||
MEM_writeLE32(seqStorePtr->dumps, (U32)matchCode); seqStorePtr->dumps += 3;
|
||||
}
|
||||
}
|
||||
else *(seqStorePtr->matchLength++) = (BYTE)matchCode;
|
||||
}
|
||||
|
||||
|
||||
/* prototype, body into zstd.c */
|
||||
size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize, const seqStore_t* seqStorePtr, size_t srcSize);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -46,12 +46,65 @@ extern "C" {
|
||||
* Includes
|
||||
***************************************/
|
||||
#include "zstd.h"
|
||||
#include "mem.h"
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Streaming functions
|
||||
* Types
|
||||
***************************************/
|
||||
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize);
|
||||
#define ZSTD_WINDOWLOG_MAX 26
|
||||
#define ZSTD_WINDOWLOG_MIN 18
|
||||
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 11
|
||||
#define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
|
||||
#define ZSTD_CONTENTLOG_MIN 4
|
||||
#define ZSTD_HASHLOG_MAX 28
|
||||
#define ZSTD_HASHLOG_MIN 4
|
||||
#define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1)
|
||||
#define ZSTD_SEARCHLOG_MIN 1
|
||||
#define ZSTD_SEARCHLENGTH_MAX 7
|
||||
#define ZSTD_SEARCHLENGTH_MIN 4
|
||||
|
||||
/** from faster to stronger */
|
||||
typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2 } ZSTD_strategy;
|
||||
|
||||
typedef struct
|
||||
{
|
||||
U64 srcSize; /* optional : tells how much bytes are present in the frame. Use 0 if not known. */
|
||||
U32 windowLog; /* largest match distance : larger == more compression, more memory needed during decompression */
|
||||
U32 contentLog; /* full search segment : larger == more compression, slower, more memory (useless for fast) */
|
||||
U32 hashLog; /* dispatch table : larger == more memory, faster */
|
||||
U32 searchLog; /* nb of searches : larger == more compression, slower */
|
||||
U32 searchLength; /* size of matches : larger == faster decompression, sometimes less compression */
|
||||
ZSTD_strategy strategy;
|
||||
} ZSTD_parameters;
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Advanced function
|
||||
***************************************/
|
||||
/** ZSTD_getParams
|
||||
* return ZSTD_parameters structure for a selected compression level and srcSize.
|
||||
* srcSizeHint value is optional, select 0 if not known */
|
||||
ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSizeHint);
|
||||
|
||||
/** ZSTD_validateParams
|
||||
* correct params value to remain within authorized range */
|
||||
void ZSTD_validateParams(ZSTD_parameters* params);
|
||||
|
||||
/** ZSTD_compress_advanced
|
||||
* Same as ZSTD_compressCCtx(), with fine-tune control of each compression parameter */
|
||||
size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
|
||||
void* dst, size_t maxDstSize,
|
||||
const void* src, size_t srcSize,
|
||||
ZSTD_parameters params);
|
||||
|
||||
|
||||
/* **************************************
|
||||
* Streaming functions (bufferless mode)
|
||||
****************************************/
|
||||
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint);
|
||||
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, ZSTD_parameters params);
|
||||
|
||||
size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
|
||||
size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize);
|
||||
|
||||
@ -61,20 +114,91 @@ ZSTD_DCtx* ZSTD_createDCtx(void);
|
||||
size_t ZSTD_resetDCtx(ZSTD_DCtx* dctx);
|
||||
size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
|
||||
|
||||
size_t ZSTD_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize);
|
||||
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
|
||||
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
|
||||
/*
|
||||
Use above functions alternatively.
|
||||
/**
|
||||
Streaming decompression, bufferless mode
|
||||
|
||||
A ZSTD_DCtx object is required to track streaming operations.
|
||||
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
|
||||
A ZSTD_DCtx object can be re-used multiple times. Use ZSTD_resetDCtx() to return to fresh status.
|
||||
|
||||
First operation is to retrieve frame parameters, using ZSTD_getFrameParams().
|
||||
This function doesn't consume its input. It needs enough input data to properly decode the frame header.
|
||||
The objective is to retrieve *params.windowlog, to know how much memory is required during decoding.
|
||||
Result : 0 if successfull, it means the ZSTD_parameters structure has been filled.
|
||||
>0 : means there is not enough data into src. Provides the expected size to successfully decode header.
|
||||
errorCode, which can be tested using ZSTD_isError() (For example, if it's not a ZSTD header)
|
||||
|
||||
Then it's possible to start decompression.
|
||||
Use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively.
|
||||
ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
|
||||
ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block.
|
||||
Result is the number of bytes regenerated within 'dst'.
|
||||
ZSTD_decompressContinue() will use previous data blocks during decompress.
|
||||
They should be located contiguously prior to current block. Alternatively, a round buffer is possible.
|
||||
Just make sure that the combined of current and accessible past blocks is a minimum of (1 << windowlog).
|
||||
|
||||
@result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst'.
|
||||
It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
|
||||
|
||||
A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.
|
||||
*/
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Prefix - version detection
|
||||
* Pre-defined compression levels
|
||||
***************************************/
|
||||
#define ZSTD_magicNumber 0xFD2FB523 /* v0.3 (current)*/
|
||||
#define ZSTD_MAX_CLEVEL 20
|
||||
static const ZSTD_parameters ZSTD_defaultParameters[2][ZSTD_MAX_CLEVEL+1] = {
|
||||
{ /* "default" */
|
||||
/* W, C, H, S, L, strat */
|
||||
{ 0, 18, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */
|
||||
{ 0, 19, 13, 14, 1, 7, ZSTD_fast }, /* level 1 */
|
||||
{ 0, 19, 15, 16, 1, 6, ZSTD_fast }, /* level 2 */
|
||||
{ 0, 20, 18, 20, 1, 6, ZSTD_fast }, /* level 3 */
|
||||
{ 0, 21, 19, 21, 1, 6, ZSTD_fast }, /* level 4 */
|
||||
{ 0, 20, 14, 18, 3, 5, ZSTD_greedy }, /* level 5 */
|
||||
{ 0, 20, 18, 19, 3, 5, ZSTD_greedy }, /* level 6 */
|
||||
{ 0, 21, 17, 20, 3, 5, ZSTD_lazy }, /* level 7 */
|
||||
{ 0, 21, 19, 20, 3, 5, ZSTD_lazy }, /* level 8 */
|
||||
{ 0, 21, 20, 20, 3, 5, ZSTD_lazy2 }, /* level 9 */
|
||||
{ 0, 21, 19, 21, 4, 5, ZSTD_lazy2 }, /* level 10 */
|
||||
{ 0, 22, 20, 22, 4, 5, ZSTD_lazy2 }, /* level 11 */
|
||||
{ 0, 22, 20, 22, 5, 5, ZSTD_lazy2 }, /* level 12 */
|
||||
{ 0, 22, 21, 22, 5, 5, ZSTD_lazy2 }, /* level 13 */
|
||||
{ 0, 22, 22, 23, 5, 5, ZSTD_lazy2 }, /* level 14 */
|
||||
{ 0, 23, 23, 23, 5, 5, ZSTD_lazy2 }, /* level 15 */
|
||||
{ 0, 23, 21, 22, 5, 5, ZSTD_btlazy2 }, /* level 16 */
|
||||
{ 0, 23, 24, 23, 4, 5, ZSTD_btlazy2 }, /* level 17 */
|
||||
{ 0, 25, 24, 23, 5, 5, ZSTD_btlazy2 }, /* level 18 */
|
||||
{ 0, 25, 26, 23, 5, 5, ZSTD_btlazy2 }, /* level 19 */
|
||||
{ 0, 25, 26, 25, 6, 5, ZSTD_btlazy2 }, /* level 20 */
|
||||
},
|
||||
{ /* for srcSize <= 128 KB */
|
||||
/* W, C, H, S, L, strat */
|
||||
{ 0, 17, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */
|
||||
{ 0, 17, 12, 13, 1, 6, ZSTD_fast }, /* level 1 */
|
||||
{ 0, 17, 15, 16, 1, 5, ZSTD_fast }, /* level 2 */
|
||||
{ 0, 17, 16, 17, 1, 5, ZSTD_fast }, /* level 3 */
|
||||
{ 0, 17, 13, 15, 2, 4, ZSTD_greedy }, /* level 4 */
|
||||
{ 0, 17, 15, 17, 3, 4, ZSTD_greedy }, /* level 5 */
|
||||
{ 0, 17, 14, 17, 3, 4, ZSTD_lazy }, /* level 6 */
|
||||
{ 0, 17, 16, 17, 4, 4, ZSTD_lazy }, /* level 7 */
|
||||
{ 0, 17, 16, 17, 4, 4, ZSTD_lazy2 }, /* level 8 */
|
||||
{ 0, 17, 17, 16, 5, 4, ZSTD_lazy2 }, /* level 9 */
|
||||
{ 0, 17, 17, 16, 6, 4, ZSTD_lazy2 }, /* level 10 */
|
||||
{ 0, 17, 17, 16, 7, 4, ZSTD_lazy2 }, /* level 11 */
|
||||
{ 0, 17, 17, 16, 8, 4, ZSTD_lazy2 }, /* level 12 */
|
||||
{ 0, 17, 18, 16, 4, 4, ZSTD_btlazy2 }, /* level 13 */
|
||||
{ 0, 17, 18, 16, 5, 4, ZSTD_btlazy2 }, /* level 14 */
|
||||
{ 0, 17, 18, 16, 6, 4, ZSTD_btlazy2 }, /* level 15 */
|
||||
{ 0, 17, 18, 16, 7, 4, ZSTD_btlazy2 }, /* level 16 */
|
||||
{ 0, 17, 18, 16, 8, 4, ZSTD_btlazy2 }, /* level 17 */
|
||||
{ 0, 17, 18, 16, 9, 4, ZSTD_btlazy2 }, /* level 18 */
|
||||
{ 0, 17, 18, 16, 10, 4, ZSTD_btlazy2 }, /* level 19 */
|
||||
{ 0, 17, 18, 18, 12, 4, ZSTD_btlazy2 }, /* level 20 */
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
/* *************************************
|
||||
|
||||
1036
lib/zstdhc.c
1036
lib/zstdhc.c
File diff suppressed because it is too large
Load Diff
@ -1,156 +0,0 @@
|
||||
/*
|
||||
zstdhc - high compression variant
|
||||
Header File - Experimental API, static linking only
|
||||
Copyright (C) 2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd source repository : http://www.zstd.net
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* *************************************
|
||||
* Includes
|
||||
***************************************/
|
||||
#include "mem.h"
|
||||
#include "zstdhc.h"
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Types
|
||||
***************************************/
|
||||
/** from faster to stronger */
|
||||
typedef enum { ZSTD_HC_fast, ZSTD_HC_greedy, ZSTD_HC_lazy, ZSTD_HC_lazy2, ZSTD_HC_btlazy2 } ZSTD_HC_strategy;
|
||||
|
||||
typedef struct
|
||||
{
|
||||
U32 windowLog; /* largest match distance : impact decompression buffer size */
|
||||
U32 contentLog; /* full search segment : larger == more compression, slower, more memory (useless for fast) */
|
||||
U32 hashLog; /* dispatch table : larger == more memory, faster*/
|
||||
U32 searchLog; /* nb of searches : larger == more compression, slower*/
|
||||
U32 searchLength; /* size of matches : larger == faster decompression */
|
||||
ZSTD_HC_strategy strategy;
|
||||
} ZSTD_HC_parameters;
|
||||
|
||||
/* parameters boundaries */
|
||||
#define ZSTD_HC_WINDOWLOG_MAX 26
|
||||
#define ZSTD_HC_WINDOWLOG_MIN 18
|
||||
#define ZSTD_HC_CONTENTLOG_MAX (ZSTD_HC_WINDOWLOG_MAX+1)
|
||||
#define ZSTD_HC_CONTENTLOG_MIN 4
|
||||
#define ZSTD_HC_HASHLOG_MAX 28
|
||||
#define ZSTD_HC_HASHLOG_MIN 4
|
||||
#define ZSTD_HC_SEARCHLOG_MAX (ZSTD_HC_CONTENTLOG_MAX-1)
|
||||
#define ZSTD_HC_SEARCHLOG_MIN 1
|
||||
#define ZSTD_HC_SEARCHLENGTH_MAX 7
|
||||
#define ZSTD_HC_SEARCHLENGTH_MIN 4
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Advanced function
|
||||
***************************************/
|
||||
/** ZSTD_HC_compress_advanced
|
||||
* Same as ZSTD_HC_compressCCtx(), with fine-tune control of each compression parameter */
|
||||
size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx,
|
||||
void* dst, size_t maxDstSize,
|
||||
const void* src, size_t srcSize,
|
||||
ZSTD_HC_parameters params);
|
||||
|
||||
/** ZSTD_HC_validateParams
|
||||
correct params value to remain within authorized range
|
||||
srcSizeHint value is optional, select 0 if not known */
|
||||
void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, U64 srcSizeHint);
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Streaming functions
|
||||
***************************************/
|
||||
size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint);
|
||||
size_t ZSTD_HC_compressContinue(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
|
||||
size_t ZSTD_HC_compressEnd(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize);
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Pre-defined compression levels
|
||||
***************************************/
|
||||
#define ZSTD_HC_MAX_CLEVEL 20
|
||||
static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[2][ZSTD_HC_MAX_CLEVEL+1] = {
|
||||
{ /* for <= 128 KB */
|
||||
/* W, C, H, S, L, strat */
|
||||
{ 17, 12, 12, 1, 4, ZSTD_HC_fast }, /* level 0 - never used */
|
||||
{ 17, 12, 13, 1, 6, ZSTD_HC_fast }, /* level 1 */
|
||||
{ 17, 15, 16, 1, 5, ZSTD_HC_fast }, /* level 2 */
|
||||
{ 17, 16, 17, 1, 5, ZSTD_HC_fast }, /* level 3 */
|
||||
{ 17, 13, 15, 2, 4, ZSTD_HC_greedy }, /* level 4 */
|
||||
{ 17, 15, 17, 3, 4, ZSTD_HC_greedy }, /* level 5 */
|
||||
{ 17, 14, 17, 3, 4, ZSTD_HC_lazy }, /* level 6 */
|
||||
{ 17, 16, 17, 4, 4, ZSTD_HC_lazy }, /* level 7 */
|
||||
{ 17, 16, 17, 4, 4, ZSTD_HC_lazy2 }, /* level 8 */
|
||||
{ 17, 17, 16, 5, 4, ZSTD_HC_lazy2 }, /* level 9 */
|
||||
{ 17, 17, 16, 6, 4, ZSTD_HC_lazy2 }, /* level 10 */
|
||||
{ 17, 17, 16, 7, 4, ZSTD_HC_lazy2 }, /* level 11 */
|
||||
{ 17, 17, 16, 8, 4, ZSTD_HC_lazy2 }, /* level 12 */
|
||||
{ 17, 18, 16, 4, 4, ZSTD_HC_btlazy2 }, /* level 13 */
|
||||
{ 17, 18, 16, 5, 4, ZSTD_HC_btlazy2 }, /* level 14 */
|
||||
{ 17, 18, 16, 6, 4, ZSTD_HC_btlazy2 }, /* level 15 */
|
||||
{ 17, 18, 16, 7, 4, ZSTD_HC_btlazy2 }, /* level 16 */
|
||||
{ 17, 18, 16, 8, 4, ZSTD_HC_btlazy2 }, /* level 17 */
|
||||
{ 17, 18, 16, 9, 4, ZSTD_HC_btlazy2 }, /* level 18 */
|
||||
{ 17, 18, 16, 10, 4, ZSTD_HC_btlazy2 }, /* level 19 */
|
||||
{ 17, 18, 18, 12, 4, ZSTD_HC_btlazy2 }, /* level 20 */
|
||||
},
|
||||
{ /* for > 128 KB */
|
||||
/* W, C, H, S, L, strat */
|
||||
{ 18, 12, 12, 1, 4, ZSTD_HC_fast }, /* level 0 - never used */
|
||||
{ 18, 14, 14, 1, 7, ZSTD_HC_fast }, /* level 1 - in fact redirected towards zstd fast */
|
||||
{ 19, 15, 16, 1, 6, ZSTD_HC_fast }, /* level 2 */
|
||||
{ 20, 18, 20, 1, 6, ZSTD_HC_fast }, /* level 3 */
|
||||
{ 21, 19, 21, 1, 6, ZSTD_HC_fast }, /* level 4 */
|
||||
{ 20, 13, 18, 5, 5, ZSTD_HC_greedy }, /* level 5 */
|
||||
{ 20, 17, 19, 3, 5, ZSTD_HC_greedy }, /* level 6 */
|
||||
{ 21, 17, 20, 3, 5, ZSTD_HC_lazy }, /* level 7 */
|
||||
{ 21, 19, 20, 3, 5, ZSTD_HC_lazy }, /* level 8 */
|
||||
{ 21, 20, 20, 3, 5, ZSTD_HC_lazy2 }, /* level 9 */
|
||||
{ 21, 19, 20, 4, 5, ZSTD_HC_lazy2 }, /* level 10 */
|
||||
{ 22, 20, 22, 4, 5, ZSTD_HC_lazy2 }, /* level 11 */
|
||||
{ 22, 20, 22, 5, 5, ZSTD_HC_lazy2 }, /* level 12 */
|
||||
{ 22, 21, 22, 5, 5, ZSTD_HC_lazy2 }, /* level 13 */
|
||||
{ 22, 22, 23, 5, 5, ZSTD_HC_lazy2 }, /* level 14 */
|
||||
{ 23, 23, 23, 5, 5, ZSTD_HC_lazy2 }, /* level 15 */
|
||||
{ 23, 21, 22, 5, 5, ZSTD_HC_btlazy2 }, /* level 16 */
|
||||
{ 23, 24, 23, 4, 5, ZSTD_HC_btlazy2 }, /* level 17 */
|
||||
{ 25, 24, 23, 5, 5, ZSTD_HC_btlazy2 }, /* level 18 */
|
||||
{ 25, 26, 23, 5, 5, ZSTD_HC_btlazy2 }, /* level 19 */
|
||||
{ 26, 27, 24, 6, 5, ZSTD_HC_btlazy2 }, /* level 20 */
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
@ -30,12 +30,12 @@
|
||||
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
|
||||
# ##########################################################################
|
||||
|
||||
VERSION?= 0.3.6
|
||||
VERSION?= 0.4.0
|
||||
|
||||
DESTDIR?=
|
||||
PREFIX ?= /usr/local
|
||||
CPPFLAGS= -I../lib -I../lib/legacy -I./legacy -DZSTD_VERSION=\"$(VERSION)\" -DZSTD_LEGACY_SUPPORT=1
|
||||
CFLAGS ?= -O3 # -falign-loops=32 # not always positive
|
||||
CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial
|
||||
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
|
||||
|
||||
@ -53,42 +53,55 @@ EXT =
|
||||
VOID = /dev/null
|
||||
endif
|
||||
|
||||
ZBUFFTEST = -T2mn
|
||||
|
||||
.PHONY: default all clean install uninstall test test32 test-all
|
||||
|
||||
default: zstd
|
||||
|
||||
all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 paramgrill datagen
|
||||
all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 zbufftest zbufftest32 paramgrill datagen
|
||||
|
||||
zstd: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
zstd: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/zstd_buffered.c \
|
||||
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
|
||||
xxhash.c bench.c fileio.c zstdcli.c legacy/fileio_legacy.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
zstd32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
zstd32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/zstd_buffered.c \
|
||||
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
|
||||
xxhash.c bench.c fileio.c zstdcli.c legacy/fileio_legacy.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fullbench : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
fullbench : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
|
||||
datagen.c fullbench.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fullbench32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
fullbench32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
|
||||
datagen.c fullbench.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fuzzer : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
fuzzer : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
|
||||
datagen.c xxhash.c fuzzer.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fuzzer32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
fuzzer32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
|
||||
datagen.c xxhash.c fuzzer.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
paramgrill : $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
zbufftest : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/zstd_buffered.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
|
||||
datagen.c xxhash.c zbufftest.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
zbufftest32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/zstd_buffered.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
|
||||
datagen.c xxhash.c zbufftest.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
paramgrill : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
|
||||
datagen.c xxhash.c paramgrill.c
|
||||
$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
|
||||
@ -97,10 +110,10 @@ datagen : datagen.c datagencli.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
clean:
|
||||
@rm -f core *.o tmp \
|
||||
@rm -f core *.o tmp* \
|
||||
zstd$(EXT) zstd32$(EXT) \
|
||||
fullbench$(EXT) fullbench32$(EXT) \
|
||||
fuzzer$(EXT) fuzzer32$(EXT) \
|
||||
fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
|
||||
datagen$(EXT) paramgrill$(EXT)
|
||||
@echo Cleaning completed
|
||||
|
||||
@ -130,9 +143,9 @@ uninstall:
|
||||
[ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstd.1
|
||||
@echo zstd programs successfully uninstalled
|
||||
|
||||
test: test-zstd test-fullbench test-fuzzer
|
||||
test: test-zstd test-fullbench test-fuzzer test-zbuff
|
||||
|
||||
test32: test-zstd32 test-fullbench32 test-fuzzer32
|
||||
test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32
|
||||
|
||||
test-all: test test32 valgrindTest
|
||||
|
||||
@ -153,11 +166,61 @@ test-zstd: zstd datagen
|
||||
echo foo | ./zstd > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
|
||||
echo foo | ./zstd | ./zstd -d > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
|
||||
@echo "**** zstd round-trip tests **** "
|
||||
./datagen | ./zstd -v | ./zstd -d > $(VOID)
|
||||
./datagen | ./zstd -6 -v | ./zstd -d > $(VOID)
|
||||
./datagen -g256MB | ./zstd -v | ./zstd -d > $(VOID)
|
||||
./datagen -g256MB | ./zstd -3 -v | ./zstd -d > $(VOID)
|
||||
./datagen -g6GB -P99 | ./zstd -vq | ./zstd -d > $(VOID)
|
||||
@./datagen | md5sum > tmp1
|
||||
./datagen | ./zstd -v | ./zstd -d | md5sum > tmp2
|
||||
@diff tmp1 tmp2
|
||||
./datagen | ./zstd -6 -v | ./zstd -d | md5sum > tmp2
|
||||
@diff tmp1 tmp2
|
||||
@./datagen -g257MB | md5sum > tmp1
|
||||
./datagen -g257MB | ./zstd -v | ./zstd -d | md5sum > tmp2
|
||||
@diff tmp1 tmp2
|
||||
./datagen -g257MB | ./zstd -v2 | ./zstd -d | md5sum > tmp2
|
||||
@diff tmp1 tmp2
|
||||
./datagen -g257MB | ./zstd -v3 | ./zstd -d | md5sum > tmp2
|
||||
@diff tmp1 tmp2
|
||||
@./datagen -g129MB -P60| md5sum > tmp1
|
||||
./datagen -g129MB -P60 | ./zstd -v4 | ./zstd -d | md5sum > tmp2
|
||||
@diff tmp1 tmp2
|
||||
./datagen -g129MB -P60 | ./zstd -v5 | ./zstd -d | md5sum > tmp2
|
||||
@diff tmp1 tmp2
|
||||
./datagen -g129MB -P60 | ./zstd -v6 | ./zstd -d | md5sum > tmp2
|
||||
@diff tmp1 tmp2
|
||||
@./datagen -g65MB -P70 | md5sum > tmp1
|
||||
./datagen -g65MB -P70 | ./zstd -v7 | ./zstd -d | md5sum > tmp2
|
||||
@diff tmp1 tmp2
|
||||
./datagen -g65MB -P70 | ./zstd -v8 | ./zstd -d | md5sum > tmp2
|
||||
@diff tmp1 tmp2
|
||||
./datagen -g65MB -P70 | ./zstd -v9 | ./zstd -d | md5sum > tmp2
|
||||
@diff tmp1 tmp2
|
||||
@./datagen -g33MB -P75 | md5sum > tmp1
|
||||
./datagen -g33MB -P75 | ./zstd -v10 | ./zstd -d | md5sum > tmp2
|
||||
@diff tmp1 tmp2
|
||||
./datagen -g33MB -P75 | ./zstd -v11 | ./zstd -d | md5sum > tmp2
|
||||
@diff tmp1 tmp2
|
||||
./datagen -g33MB -P75 | ./zstd -v12 | ./zstd -d | md5sum > tmp2
|
||||
@diff tmp1 tmp2
|
||||
@./datagen -g17MB -P80 | md5sum > tmp1
|
||||
./datagen -g17MB -P80 | ./zstd -v13 | ./zstd -d | md5sum > tmp2
|
||||
@diff tmp1 tmp2
|
||||
./datagen -g17MB -P80 | ./zstd -v14 | ./zstd -d | md5sum > tmp2
|
||||
@diff tmp1 tmp2
|
||||
./datagen -g17MB -P80 | ./zstd -v15 | ./zstd -d | md5sum > tmp2
|
||||
@diff tmp1 tmp2
|
||||
./datagen -g17MB -P80 | ./zstd -v16 | ./zstd -d | md5sum > tmp2
|
||||
@diff tmp1 tmp2
|
||||
./datagen -g17MB -P80 | ./zstd -v17 | ./zstd -d | md5sum > tmp2
|
||||
@diff tmp1 tmp2
|
||||
@./datagen -g49MB -P93 | md5sum > tmp1
|
||||
./datagen -g49MB -P93 | ./zstd -v18 | ./zstd -d | md5sum > tmp2
|
||||
@diff tmp1 tmp2
|
||||
./datagen -g49MB -P93 | ./zstd -v19 | ./zstd -d | md5sum > tmp2
|
||||
@diff tmp1 tmp2
|
||||
@./datagen -g97MB -P99 | md5sum > tmp1
|
||||
./datagen -g97MB -P99 | ./zstd -v20 | ./zstd -d | md5sum > tmp2
|
||||
@diff tmp1 tmp2
|
||||
./datagen -g6GB -P99 | md5sum > tmp1
|
||||
./datagen -g6GB -P99 | ./zstd -vq | ./zstd -d | md5sum > tmp2
|
||||
@diff tmp1 tmp2
|
||||
|
||||
test-zstd32: zstd32 datagen
|
||||
./datagen | ./zstd32 -v | ./zstd32 -d > $(VOID)
|
||||
@ -178,17 +241,25 @@ test-fuzzer: fuzzer
|
||||
test-fuzzer32: fuzzer32
|
||||
./fuzzer32
|
||||
|
||||
valgrindTest: zstd datagen fuzzer fullbench
|
||||
test-zbuff: zbufftest
|
||||
./zbufftest $(ZBUFFTEST)
|
||||
|
||||
test-zbuff32: zbufftest32
|
||||
./zbufftest32 $(ZBUFFTEST)
|
||||
|
||||
valgrindTest: zstd datagen fuzzer fullbench zbufftest
|
||||
@echo "\n ---- valgrind tests : memory analyzer ----"
|
||||
valgrind --leak-check=yes --error-exitcode=1 ./datagen -g50M > $(VOID)
|
||||
./datagen -g16KB > tmp
|
||||
valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp $(VOID)
|
||||
./datagen -g2930KB > tmp
|
||||
valgrind --leak-check=yes --error-exitcode=1 ./zstd -4 -vf tmp $(VOID)
|
||||
valgrind --leak-check=yes --error-exitcode=1 ./zstd -5 -vf tmp tmp2
|
||||
valgrind --leak-check=yes --error-exitcode=1 ./zstd -vdf tmp2 $(VOID)
|
||||
./datagen -g64MB > tmp
|
||||
valgrind --leak-check=yes --error-exitcode=1 ./zstd -vf tmp $(VOID)
|
||||
@rm tmp
|
||||
valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i1000 -t1
|
||||
valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1
|
||||
valgrind --leak-check=yes --error-exitcode=1 ./zbufftest -T1mn
|
||||
|
||||
endif
|
||||
|
||||
@ -61,7 +61,6 @@
|
||||
|
||||
#include "mem.h"
|
||||
#include "zstd.h"
|
||||
#include "zstdhc.h"
|
||||
#include "xxhash.h"
|
||||
|
||||
|
||||
@ -231,12 +230,6 @@ typedef struct
|
||||
|
||||
typedef size_t (*compressor_t) (void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel);
|
||||
|
||||
static size_t local_compress_fast (void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
|
||||
{
|
||||
(void)compressionLevel;
|
||||
return ZSTD_compress(dst, maxDstSize, src, srcSize);
|
||||
}
|
||||
|
||||
#define MIN(a,b) ((a)<(b) ? (a) : (b))
|
||||
|
||||
static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, int cLevel)
|
||||
@ -247,7 +240,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
|
||||
const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize);
|
||||
void* const compressedBuffer = malloc(maxCompressedSize);
|
||||
void* const resultBuffer = malloc(srcSize);
|
||||
const compressor_t compressor = (cLevel <= 1) ? local_compress_fast : ZSTD_HC_compress;
|
||||
const compressor_t compressor = ZSTD_compress;
|
||||
U64 crcOrig;
|
||||
|
||||
/* init */
|
||||
@ -413,7 +406,7 @@ static size_t BMK_findMaxMem(U64 requiredMem)
|
||||
return (size_t)(requiredMem - step);
|
||||
}
|
||||
|
||||
static int BMK_benchOneFile(char* inFileName, int cLevel)
|
||||
static int BMK_benchOneFile(const char* inFileName, int cLevel)
|
||||
{
|
||||
FILE* inFile;
|
||||
U64 inFileSize;
|
||||
@ -513,7 +506,7 @@ static int BMK_syntheticTest(int cLevel, double compressibility)
|
||||
}
|
||||
|
||||
|
||||
int BMK_benchFiles(char** fileNamesTable, unsigned nbFiles, unsigned cLevel)
|
||||
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, unsigned cLevel)
|
||||
{
|
||||
double compressibility = (double)g_compressibilityDefault / 100;
|
||||
|
||||
|
||||
@ -26,7 +26,7 @@
|
||||
|
||||
|
||||
/* Main function */
|
||||
int BMK_benchFiles(char** fileNamesTable, unsigned nbFiles, unsigned cLevel);
|
||||
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, unsigned cLevel);
|
||||
|
||||
/* Set Parameters */
|
||||
void BMK_SetNbIterations(int nbLoops);
|
||||
|
||||
@ -81,7 +81,6 @@ typedef BYTE litDistribTable[LTSIZE];
|
||||
|
||||
|
||||
|
||||
|
||||
/*********************************************************
|
||||
* Local Functions
|
||||
*********************************************************/
|
||||
@ -130,7 +129,7 @@ static BYTE RDG_genChar(U32* seed, const litDistribTable lt)
|
||||
}
|
||||
|
||||
|
||||
#define RDG_RAND15BITS ((RDG_rand(seed) >> 3) & 32767)
|
||||
#define RDG_RAND15BITS ((RDG_rand(seed) >> 3) & 0x7FFF)
|
||||
#define RDG_RANDLENGTH ( ((RDG_rand(seed) >> 7) & 7) ? (RDG_rand(seed) & 15) : (RDG_rand(seed) & 511) + 15)
|
||||
void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, litDistribTable lt, unsigned* seedPtr)
|
||||
{
|
||||
@ -138,6 +137,7 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match
|
||||
const U32 matchProba32 = (U32)(32768 * matchProba);
|
||||
size_t pos = prefixSize;
|
||||
U32* seed = seedPtr;
|
||||
U32 prevOffset = 1;
|
||||
|
||||
/* special case : sparse content */
|
||||
while (matchProba >= 1.0)
|
||||
@ -170,11 +170,13 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match
|
||||
size_t d;
|
||||
int length = RDG_RANDLENGTH + 4;
|
||||
U32 offset = RDG_RAND15BITS + 1;
|
||||
U32 repeatOffset = (RDG_rand(seed) & 15) == 2;
|
||||
if (repeatOffset) offset = prevOffset;
|
||||
if (offset > pos) offset = (U32)pos;
|
||||
match = pos - offset;
|
||||
d = pos + length;
|
||||
if (d > buffSize) d = buffSize;
|
||||
while (pos < d) buffPtr[pos++] = buffPtr[match++];
|
||||
while (pos < d) buffPtr[pos++] = buffPtr[match++]; /* correctly manages overlaps */
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
@ -49,8 +49,6 @@
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
#endif
|
||||
|
||||
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
||||
|
||||
#define _FILE_OFFSET_BITS 64 /* Large file support on 32-bits unix */
|
||||
#define _POSIX_SOURCE 1 /* enable fileno() within <stdio.h> on unix */
|
||||
|
||||
@ -67,11 +65,11 @@
|
||||
#include <sys/stat.h> /* stat64 */
|
||||
#include "mem.h"
|
||||
#include "fileio.h"
|
||||
#include "zstd_static.h"
|
||||
#include "zstdhc_static.h"
|
||||
#include "zstd_static.h" /* ZSTD_magicNumber */
|
||||
#include "zstd_buffered_static.h"
|
||||
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
# include "zstd_legacy.h" /* legacy */
|
||||
# include "zstd_legacy.h" /* legacy */
|
||||
# include "fileio_legacy.h" /* legacy */
|
||||
#endif
|
||||
|
||||
@ -83,7 +81,7 @@
|
||||
# include <fcntl.h> /* _O_BINARY */
|
||||
# include <io.h> /* _setmode, _isatty */
|
||||
# ifdef __MINGW32__
|
||||
/* int _fileno(FILE *stream); // seems no longer useful // MINGW somehow forgets to include this windows declaration into <stdio.h> */
|
||||
// int _fileno(FILE *stream); /* seems no longer useful /* MINGW somehow forgets to include this windows declaration into <stdio.h> */
|
||||
# endif
|
||||
# define SET_BINARY_MODE(file) { int unused = _setmode(_fileno(file), _O_BINARY); (void)unused; }
|
||||
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
|
||||
@ -115,8 +113,8 @@
|
||||
#define BIT6 0x40
|
||||
#define BIT7 0x80
|
||||
|
||||
//static const unsigned FIO_maxBlockSizeID = 0xB; /* => 2MB block */
|
||||
static const unsigned FIO_blockHeaderSize = 3;
|
||||
#define BLOCKSIZE (128 KB)
|
||||
#define ROLLBUFFERSIZE (BLOCKSIZE*8*64)
|
||||
|
||||
#define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */
|
||||
#define FSE_CHECKSUM_SEED 0
|
||||
@ -237,102 +235,30 @@ static U64 FIO_getFileSize(const char* infilename)
|
||||
}
|
||||
|
||||
|
||||
typedef void* (*FIO_createC) (void);
|
||||
static void* local_ZSTD_createCCtx(void) { return (void*) ZSTD_createCCtx(); }
|
||||
static void* local_ZSTD_HC_createCCtx(void) { return (void*) ZSTD_HC_createCCtx(); }
|
||||
|
||||
typedef size_t (*FIO_initC) (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint);
|
||||
static size_t local_ZSTD_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint)
|
||||
{
|
||||
(void)cLevel; (void)srcSizeHint;
|
||||
return ZSTD_compressBegin((ZSTD_CCtx*)ctx, dst, maxDstSize);
|
||||
}
|
||||
static size_t local_ZSTD_HC_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint)
|
||||
{
|
||||
return ZSTD_HC_compressBegin((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, cLevel, srcSizeHint);
|
||||
}
|
||||
|
||||
typedef size_t (*FIO_continueC) (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
|
||||
static size_t local_ZSTD_compressContinue (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
return ZSTD_compressContinue((ZSTD_CCtx*)ctx, dst, maxDstSize, src, srcSize);
|
||||
}
|
||||
static size_t local_ZSTD_HC_compressContinue (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
return ZSTD_HC_compressContinue((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, src, srcSize);
|
||||
}
|
||||
|
||||
typedef size_t (*FIO_endC) (void* ctx, void* dst, size_t maxDstSize);
|
||||
static size_t local_ZSTD_compressEnd (void* ctx, void* dst, size_t maxDstSize)
|
||||
{
|
||||
return ZSTD_compressEnd((ZSTD_CCtx*)ctx, dst, maxDstSize);
|
||||
}
|
||||
static size_t local_ZSTD_HC_compressEnd (void* ctx, void* dst, size_t maxDstSize)
|
||||
{
|
||||
return ZSTD_HC_compressEnd((ZSTD_HC_CCtx*)ctx, dst, maxDstSize);
|
||||
}
|
||||
|
||||
typedef void (*FIO_freeC) (void* ctx);
|
||||
static void local_ZSTD_freeCCtx(void* ctx) { ZSTD_freeCCtx((ZSTD_CCtx*)ctx); }
|
||||
static void local_ZSTD_HC_freeCCtx(void* ctx) { ZSTD_HC_freeCCtx((ZSTD_HC_CCtx*)ctx); }
|
||||
|
||||
|
||||
unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename, int cLevel)
|
||||
{
|
||||
U64 filesize = 0;
|
||||
U64 compressedfilesize = 0;
|
||||
BYTE* inBuff;
|
||||
BYTE* inSlot;
|
||||
BYTE* inEnd;
|
||||
BYTE* outBuff;
|
||||
size_t blockSize = 128 KB;
|
||||
size_t inBuffSize = 4 * blockSize;
|
||||
size_t outBuffSize = ZSTD_compressBound(blockSize);
|
||||
size_t inBuffSize = ZBUFF_recommendedCInSize();
|
||||
size_t outBuffSize = ZBUFF_recommendedCOutSize();
|
||||
FILE* finput;
|
||||
FILE* foutput;
|
||||
size_t sizeCheck, cSize;
|
||||
void* ctx;
|
||||
FIO_createC createC=NULL;
|
||||
FIO_initC initC=NULL;
|
||||
FIO_continueC continueC = NULL;
|
||||
FIO_endC endC = NULL;
|
||||
FIO_freeC freeC = NULL;
|
||||
|
||||
/* Init */
|
||||
if (cLevel <= 1)
|
||||
{
|
||||
createC = local_ZSTD_createCCtx;
|
||||
initC = local_ZSTD_compressBegin;
|
||||
continueC = local_ZSTD_compressContinue;
|
||||
endC = local_ZSTD_compressEnd;
|
||||
freeC = local_ZSTD_freeCCtx;
|
||||
}
|
||||
else
|
||||
{
|
||||
createC = local_ZSTD_HC_createCCtx;
|
||||
initC = local_ZSTD_HC_compressBegin;
|
||||
continueC = local_ZSTD_HC_compressContinue;
|
||||
endC = local_ZSTD_HC_compressEnd;
|
||||
freeC = local_ZSTD_HC_freeCCtx;
|
||||
}
|
||||
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
|
||||
filesize = FIO_getFileSize(input_filename);
|
||||
size_t sizeCheck, errorCode;
|
||||
ZBUFF_CCtx* ctx;
|
||||
|
||||
/* Allocate Memory */
|
||||
ctx = createC();
|
||||
ctx = ZBUFF_createCCtx();
|
||||
inBuff = (BYTE*)malloc(inBuffSize);
|
||||
outBuff = (BYTE*)malloc(outBuffSize);
|
||||
if (!inBuff || !outBuff || !ctx) EXM_THROW(21, "Allocation error : not enough memory");
|
||||
inSlot = inBuff;
|
||||
inEnd = inBuff + inBuffSize;
|
||||
|
||||
/* Write Frame Header */
|
||||
cSize = initC(ctx, outBuff, outBuffSize, cLevel, filesize);
|
||||
if (ZSTD_isError(cSize)) EXM_THROW(22, "Compression error : cannot create frame header");
|
||||
|
||||
sizeCheck = fwrite(outBuff, 1, cSize, foutput);
|
||||
if (sizeCheck!=cSize) EXM_THROW(23, "Write error : cannot write header into %s", output_filename);
|
||||
compressedfilesize += cSize;
|
||||
/* init */
|
||||
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
|
||||
filesize = FIO_getFileSize(input_filename);
|
||||
errorCode = ZBUFF_compressInit_advanced(ctx, ZSTD_getParams(cLevel, filesize));
|
||||
if (ZBUFF_isError(errorCode)) EXM_THROW(22, "Error initializing compression");
|
||||
filesize = 0;
|
||||
|
||||
/* Main compression loop */
|
||||
@ -341,33 +267,41 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
|
||||
size_t inSize;
|
||||
|
||||
/* Fill input Buffer */
|
||||
if (inSlot + blockSize > inEnd) inSlot = inBuff;
|
||||
inSize = fread(inSlot, (size_t)1, blockSize, finput);
|
||||
inSize = fread(inBuff, (size_t)1, inBuffSize, finput);
|
||||
if (inSize==0) break;
|
||||
filesize += inSize;
|
||||
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20));
|
||||
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20));
|
||||
|
||||
/* Compress Block */
|
||||
cSize = continueC(ctx, outBuff, outBuffSize, inSlot, inSize);
|
||||
if (ZSTD_isError(cSize))
|
||||
EXM_THROW(24, "Compression error : %s ", ZSTD_getErrorName(cSize));
|
||||
{
|
||||
/* Compress (buffered streaming ensures appropriate formatting) */
|
||||
size_t usedInSize = inSize;
|
||||
size_t cSize = outBuffSize;
|
||||
size_t result = ZBUFF_compressContinue(ctx, outBuff, &cSize, inBuff, &usedInSize);
|
||||
if (ZBUFF_isError(result))
|
||||
EXM_THROW(23, "Compression error : %s ", ZBUFF_getErrorName(result));
|
||||
if (inSize != usedInSize)
|
||||
/* inBuff should be entirely consumed since buffer sizes are recommended ones */
|
||||
EXM_THROW(24, "Compression error : input block not fully consumed");
|
||||
|
||||
/* Write cBlock */
|
||||
sizeCheck = fwrite(outBuff, 1, cSize, foutput);
|
||||
if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", output_filename);
|
||||
compressedfilesize += cSize;
|
||||
inSlot += inSize;
|
||||
/* Write cBlock */
|
||||
sizeCheck = fwrite(outBuff, 1, cSize, foutput);
|
||||
if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", output_filename);
|
||||
compressedfilesize += cSize;
|
||||
}
|
||||
|
||||
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(filesize>>20), (double)compressedfilesize/filesize*100);
|
||||
}
|
||||
|
||||
/* End of Frame */
|
||||
cSize = endC(ctx, outBuff, outBuffSize);
|
||||
if (ZSTD_isError(cSize)) EXM_THROW(26, "Compression error : cannot create frame end");
|
||||
{
|
||||
size_t cSize = outBuffSize;
|
||||
size_t result = ZBUFF_compressEnd(ctx, outBuff, &cSize);
|
||||
if (result!=0) EXM_THROW(26, "Compression error : cannot create frame end");
|
||||
|
||||
sizeCheck = fwrite(outBuff, 1, cSize, foutput);
|
||||
if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", output_filename);
|
||||
compressedfilesize += cSize;
|
||||
sizeCheck = fwrite(outBuff, 1, cSize, foutput);
|
||||
if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", output_filename);
|
||||
compressedfilesize += cSize;
|
||||
}
|
||||
|
||||
/* Status */
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
@ -377,7 +311,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
|
||||
/* clean */
|
||||
free(inBuff);
|
||||
free(outBuff);
|
||||
freeC(ctx);
|
||||
ZBUFF_freeCCtx(ctx);
|
||||
fclose(finput);
|
||||
if (fclose(foutput)) EXM_THROW(28, "Write error : cannot properly close %s", output_filename);
|
||||
|
||||
@ -386,124 +320,87 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
|
||||
|
||||
|
||||
unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
|
||||
BYTE* inBuff, size_t inBuffSize,
|
||||
BYTE* inBuff, size_t inBuffSize, size_t alreadyLoaded,
|
||||
BYTE* outBuff, size_t outBuffSize,
|
||||
ZSTD_DCtx* dctx)
|
||||
ZBUFF_DCtx* dctx)
|
||||
{
|
||||
BYTE* op = outBuff;
|
||||
BYTE* const oend = outBuff + outBuffSize;
|
||||
U64 filesize = 0;
|
||||
size_t toRead;
|
||||
size_t sizeCheck;
|
||||
|
||||
U64 frameSize = 0;
|
||||
size_t readSize=alreadyLoaded;
|
||||
|
||||
/* Main decompression Loop */
|
||||
toRead = ZSTD_nextSrcSizeToDecompress(dctx);
|
||||
while (toRead)
|
||||
ZBUFF_decompressInit(dctx);
|
||||
while (1)
|
||||
{
|
||||
size_t readSize, decodedSize;
|
||||
/* Decode */
|
||||
size_t sizeCheck;
|
||||
size_t inSize=readSize, decodedSize=outBuffSize;
|
||||
size_t inStart=0;
|
||||
size_t toRead = ZBUFF_decompressContinue(dctx, outBuff, &decodedSize, inBuff+inStart, &inSize);
|
||||
if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead));
|
||||
if (toRead==0) break; /* end of Frame */
|
||||
readSize -= inSize;
|
||||
inStart += inSize;
|
||||
|
||||
/* Write block */
|
||||
sizeCheck = fwrite(outBuff, 1, decodedSize, foutput);
|
||||
if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file");
|
||||
frameSize += decodedSize;
|
||||
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) );
|
||||
|
||||
if (readSize) continue; /* still some data left within inBuff */
|
||||
|
||||
/* Fill input buffer */
|
||||
if (toRead > inBuffSize)
|
||||
EXM_THROW(34, "too large block");
|
||||
if (toRead > inBuffSize) EXM_THROW(34, "too large block");
|
||||
readSize = fread(inBuff, 1, toRead, finput);
|
||||
if (readSize != toRead)
|
||||
EXM_THROW(35, "Read error");
|
||||
|
||||
/* Decode block */
|
||||
decodedSize = ZSTD_decompressContinue(dctx, op, oend-op, inBuff, readSize);
|
||||
if (ZSTD_isError(decodedSize)) EXM_THROW(36, "Decoding error : input corrupted");
|
||||
|
||||
if (decodedSize) /* not a header */
|
||||
{
|
||||
/* Write block */
|
||||
sizeCheck = fwrite(op, 1, decodedSize, foutput);
|
||||
if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file");
|
||||
filesize += decodedSize;
|
||||
op += decodedSize;
|
||||
if (op==oend) op = outBuff;
|
||||
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(filesize>>20) );
|
||||
}
|
||||
|
||||
/* prepare for next Block */
|
||||
toRead = ZSTD_nextSrcSizeToDecompress(dctx);
|
||||
if (readSize != toRead) EXM_THROW(35, "Read error");
|
||||
}
|
||||
|
||||
return filesize;
|
||||
return frameSize;
|
||||
}
|
||||
|
||||
|
||||
#define MAXHEADERSIZE (FIO_FRAMEHEADERSIZE+3)
|
||||
unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
|
||||
{
|
||||
FILE* finput, *foutput;
|
||||
BYTE* inBuff=NULL;
|
||||
size_t inBuffSize = 0;
|
||||
size_t inBuffSize = ZBUFF_recommendedDInSize();
|
||||
BYTE* outBuff=NULL;
|
||||
size_t outBuffSize = 0;
|
||||
U32 blockSize = 128 KB;
|
||||
U32 wNbBlocks = 4;
|
||||
size_t outBuffSize = ZBUFF_recommendedDOutSize();
|
||||
U64 filesize = 0;
|
||||
BYTE* header[MAXHEADERSIZE];
|
||||
size_t toRead;
|
||||
size_t sizeCheck;
|
||||
|
||||
|
||||
/* Init */
|
||||
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||
ZBUFF_DCtx* dctx = ZBUFF_createDCtx();
|
||||
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
|
||||
|
||||
/* Allocate Memory (if needed) */
|
||||
inBuff = (BYTE*)malloc(inBuffSize);
|
||||
outBuff = (BYTE*)malloc(outBuffSize);
|
||||
if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
|
||||
|
||||
/* for each frame */
|
||||
for ( ; ; )
|
||||
{
|
||||
/* check magic number -> version */
|
||||
U32 magicNumber;
|
||||
toRead = sizeof(ZSTD_magicNumber);;
|
||||
sizeCheck = fread(header, (size_t)1, toRead, finput);
|
||||
toRead = 0;
|
||||
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
/* check magic number -> version */
|
||||
toRead = 4;
|
||||
sizeCheck = fread(inBuff, (size_t)1, toRead, finput);
|
||||
if (sizeCheck==0) break; /* no more input */
|
||||
if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
|
||||
|
||||
magicNumber = MEM_readLE32(header);
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
magicNumber = MEM_readLE32(inBuff);
|
||||
if (ZSTD_isLegacy(magicNumber))
|
||||
{
|
||||
filesize += FIO_decompressLegacyFrame(foutput, finput, magicNumber);
|
||||
continue;
|
||||
}
|
||||
#endif /* ZSTD_LEGACY_SUPPORT */
|
||||
if (magicNumber != ZSTD_magicNumber) EXM_THROW(32, "Error : unknown frame prefix");
|
||||
|
||||
/* prepare frame decompression, by completing header */
|
||||
ZSTD_resetDCtx(dctx);
|
||||
toRead = ZSTD_nextSrcSizeToDecompress(dctx) - sizeof(ZSTD_magicNumber);
|
||||
if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
|
||||
sizeCheck = fread(&header[sizeof(ZSTD_magicNumber)], 1, toRead, finput);
|
||||
if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
|
||||
sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, sizeof(ZSTD_magicNumber)+toRead); // Decode frame header
|
||||
if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
|
||||
|
||||
/* Here later : blockSize determination */
|
||||
|
||||
/* Allocate Memory (if needed) */
|
||||
{
|
||||
size_t newInBuffSize = blockSize + FIO_blockHeaderSize;
|
||||
size_t newOutBuffSize = wNbBlocks * blockSize;
|
||||
if (newInBuffSize > inBuffSize)
|
||||
{
|
||||
free(inBuff);
|
||||
inBuffSize = newInBuffSize;
|
||||
inBuff = (BYTE*)malloc(inBuffSize);
|
||||
}
|
||||
if (newOutBuffSize > outBuffSize)
|
||||
{
|
||||
free(outBuff);
|
||||
outBuffSize = newOutBuffSize;
|
||||
outBuff = (BYTE*)malloc(outBuffSize);
|
||||
}
|
||||
}
|
||||
if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
|
||||
|
||||
filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, outBuff, outBuffSize, dctx);
|
||||
filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, toRead, outBuff, outBuffSize, dctx);
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
@ -512,7 +409,7 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
|
||||
/* clean */
|
||||
free(inBuff);
|
||||
free(outBuff);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
ZBUFF_freeDCtx(dctx);
|
||||
fclose(finput);
|
||||
if (fclose(foutput)) EXM_THROW(38, "Write error : cannot properly close %s", output_filename);
|
||||
|
||||
@ -520,74 +417,3 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
|
||||
}
|
||||
|
||||
|
||||
#if 0
|
||||
unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
|
||||
{
|
||||
FILE* finput, *foutput;
|
||||
BYTE* inBuff=NULL;
|
||||
size_t inBuffSize = 0;
|
||||
BYTE* outBuff=NULL;
|
||||
size_t outBuffSize = 0;
|
||||
U32 blockSize = 128 KB;
|
||||
U32 wNbBlocks = 4;
|
||||
U64 filesize = 0;
|
||||
BYTE* header[MAXHEADERSIZE];
|
||||
ZSTD_Dctx* dctx;
|
||||
size_t toRead;
|
||||
size_t sizeCheck;
|
||||
|
||||
|
||||
/* Init */
|
||||
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
|
||||
dctx = ZSTD_createDCtx();
|
||||
|
||||
/* for each frame */
|
||||
for ( ; ; )
|
||||
{
|
||||
/* check header */
|
||||
ZSTD_resetDCtx(dctx);
|
||||
toRead = ZSTD_nextSrcSizeToDecompress(dctx);
|
||||
if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
|
||||
sizeCheck = fread(header, (size_t)1, toRead, finput);
|
||||
if (sizeCheck==0) break; /* no more input */
|
||||
if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
|
||||
sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead); // Decode frame header
|
||||
if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
|
||||
|
||||
/* Here later : blockSize determination */
|
||||
|
||||
/* Allocate Memory (if needed) */
|
||||
{
|
||||
size_t newInBuffSize = blockSize + FIO_blockHeaderSize;
|
||||
size_t newOutBuffSize = wNbBlocks * blockSize;
|
||||
if (newInBuffSize > inBuffSize)
|
||||
{
|
||||
free(inBuff);
|
||||
inBuffSize = newInBuffSize;
|
||||
inBuff = (BYTE*)malloc(inBuffSize);
|
||||
}
|
||||
if (newOutBuffSize > outBuffSize)
|
||||
{
|
||||
free(outBuff);
|
||||
outBuffSize = newOutBuffSize;
|
||||
outBuff = (BYTE*)malloc(outBuffSize);
|
||||
}
|
||||
}
|
||||
if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
|
||||
|
||||
filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, outBuff, outBuffSize, dctx);
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
DISPLAYLEVEL(2, "Decoded %llu bytes \n", (long long unsigned)filesize);
|
||||
|
||||
/* clean */
|
||||
free(inBuff);
|
||||
free(outBuff);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
fclose(finput);
|
||||
if (fclose(foutput)) EXM_THROW(38, "Write error : cannot properly close %s", output_filename);
|
||||
|
||||
return filesize;
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -61,7 +61,7 @@
|
||||
#endif
|
||||
|
||||
#include "mem.h"
|
||||
#include "zstd.h"
|
||||
#include "zstd_static.h"
|
||||
#include "fse_static.h"
|
||||
#include "datagen.h"
|
||||
|
||||
@ -209,7 +209,7 @@ typedef struct
|
||||
} blockProperties_t;
|
||||
|
||||
static size_t g_cSize = 0;
|
||||
static U32 g_litCtx[40 * 1024];
|
||||
static ZSTD_DCtx* g_dctxPtr = NULL;
|
||||
|
||||
extern size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr);
|
||||
extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, const void* src, size_t srcSize);
|
||||
@ -217,7 +217,7 @@ extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
|
||||
size_t local_ZSTD_compress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
|
||||
{
|
||||
(void)buff2;
|
||||
return ZSTD_compress(dst, dstSize, src, srcSize);
|
||||
return ZSTD_compress(dst, dstSize, src, srcSize, 1);
|
||||
}
|
||||
|
||||
size_t local_ZSTD_decompress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
|
||||
@ -226,11 +226,11 @@ size_t local_ZSTD_decompress(void* dst, size_t dstSize, void* buff2, const void*
|
||||
return ZSTD_decompress(dst, dstSize, buff2, g_cSize);
|
||||
}
|
||||
|
||||
extern size_t ZSTD_decodeLiteralsBlock(void* ctx, const void* src, size_t srcSize);
|
||||
extern size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* ctx, const void* src, size_t srcSize);
|
||||
size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
|
||||
{
|
||||
(void)src; (void)srcSize; (void)dst; (void)dstSize;
|
||||
return ZSTD_decodeLiteralsBlock(g_litCtx, buff2, g_cSize);
|
||||
return ZSTD_decodeLiteralsBlock((ZSTD_DCtx*)g_dctxPtr, buff2, g_cSize);
|
||||
}
|
||||
|
||||
size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
|
||||
@ -243,31 +243,6 @@ size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const
|
||||
return ZSTD_decodeSeqHeaders(&nbSeq, &dumps, &length, DTableLL, DTableML, DTableOffb, buff2, g_cSize);
|
||||
}
|
||||
|
||||
size_t local_conditionalNull(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
|
||||
{
|
||||
U32 i;
|
||||
size_t total = 0;
|
||||
BYTE* data = (BYTE*)buff2;
|
||||
|
||||
(void)dst; (void)dstSize; (void)src;
|
||||
for (i=0; i < srcSize; i++)
|
||||
{
|
||||
U32 b = data[i];
|
||||
total += b;
|
||||
if (b==0) total = 0; // 825
|
||||
//if (!b) total = 0; // 825
|
||||
//total = b ? total : 0; // 622
|
||||
//total &= -!b; // 622
|
||||
//total *= !!b; // 465
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
size_t local_decodeLiteralsForward(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
|
||||
{
|
||||
(void)src; (void)srcSize;
|
||||
return FSE_decompress(dst, dstSize, buff2, g_cSize);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -300,12 +275,6 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
|
||||
case 32:
|
||||
benchFunction = local_ZSTD_decodeSeqHeaders; benchName = "ZSTD_decodeSeqHeaders";
|
||||
break;
|
||||
case 101:
|
||||
benchFunction = local_conditionalNull; benchName = "conditionalNull";
|
||||
break;
|
||||
case 102:
|
||||
benchFunction = local_decodeLiteralsForward; benchName = "ZSTD_decodeLiteralsForward";
|
||||
break;
|
||||
default :
|
||||
return 0;
|
||||
}
|
||||
@ -314,6 +283,7 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
|
||||
dstBuffSize = ZSTD_compressBound(srcSize);
|
||||
dstBuff = (BYTE*)malloc(dstBuffSize);
|
||||
buff2 = (BYTE*)malloc(dstBuffSize);
|
||||
g_dctxPtr = ZSTD_createDCtx();
|
||||
if ((!dstBuff) || (!buff2))
|
||||
{
|
||||
DISPLAY("\nError: not enough memory!\n");
|
||||
@ -325,22 +295,20 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
|
||||
switch(benchNb)
|
||||
{
|
||||
case 11:
|
||||
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize);
|
||||
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
|
||||
break;
|
||||
case 31: /* ZSTD_decodeLiteralsBlock */
|
||||
{
|
||||
blockProperties_t bp;
|
||||
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
|
||||
ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp); // Get first block type
|
||||
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
|
||||
ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp); /* Get 1st block type */
|
||||
if (bp.blockType != bt_compressed)
|
||||
{
|
||||
DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n");
|
||||
free(dstBuff);
|
||||
free(buff2);
|
||||
return 0;
|
||||
goto _cleanOut;
|
||||
}
|
||||
memcpy(buff2, dstBuff+7, g_cSize-7);
|
||||
srcSize = srcSize > 128 KB ? 128 KB : srcSize; // relative to block
|
||||
memcpy(buff2, dstBuff+8, g_cSize-8);
|
||||
srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */
|
||||
break;
|
||||
}
|
||||
case 32: /* ZSTD_decodeSeqHeaders */
|
||||
@ -349,44 +317,26 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
|
||||
const BYTE* ip = dstBuff;
|
||||
const BYTE* iend;
|
||||
size_t blockSize;
|
||||
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
|
||||
ip += 4; // Jump magic Number
|
||||
blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); // Get first block type
|
||||
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); /* it would be better to use direct block compression here */
|
||||
ip += 5; /* Skip frame Header */
|
||||
blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */
|
||||
if (bp.blockType != bt_compressed)
|
||||
{
|
||||
DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n");
|
||||
free(dstBuff);
|
||||
free(buff2);
|
||||
return 0;
|
||||
goto _cleanOut;
|
||||
}
|
||||
iend = ip + 3 + blockSize; // Get end of first block
|
||||
ip += 3; // jump first block header
|
||||
ip += ZSTD_decodeLiteralsBlock(g_litCtx, ip, iend-ip); // jump literal sub block and its header
|
||||
iend = ip + 3 + blockSize; /* End of first block */
|
||||
ip += 3; /* skip block header */
|
||||
ip += ZSTD_decodeLiteralsBlock(g_dctxPtr, ip, iend-ip); /* skip literal segment */
|
||||
g_cSize = iend-ip;
|
||||
memcpy(buff2, ip, g_cSize); // copy rest of block (starting with SeqHeader)
|
||||
srcSize = srcSize > 128 KB ? 128 KB : srcSize; // speed relative to block
|
||||
memcpy(buff2, ip, g_cSize); /* copy rest of block (it starts by SeqHeader) */
|
||||
srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */
|
||||
break;
|
||||
}
|
||||
|
||||
/* test functions */
|
||||
/* by convention, test functions can be added > 100 */
|
||||
|
||||
case 101: /* conditionalNull */
|
||||
{
|
||||
size_t i;
|
||||
for (i=0; i<srcSize; i++)
|
||||
buff2[i] = i & 15;
|
||||
break;
|
||||
}
|
||||
case 102: /* local_decodeLiteralsForward */
|
||||
{
|
||||
blockProperties_t bp;
|
||||
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
|
||||
g_cSize = ZSTD_getcBlockSize(dstBuff+7, dstBuffSize, &bp);
|
||||
memcpy(buff2, dstBuff+10, g_cSize);
|
||||
//srcSize = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize); // real speed
|
||||
srcSize = srcSize > 128 KB ? 128 KB : srcSize; // relative to block
|
||||
break;
|
||||
}
|
||||
default : ;
|
||||
}
|
||||
|
||||
@ -418,8 +368,10 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
|
||||
|
||||
DISPLAY("%2u- %-30.30s : %7.1f MB/s (%9u)\n", benchNb, benchName, (double)srcSize / bestTime / 1000., (U32)errorCode);
|
||||
|
||||
_cleanOut:
|
||||
free(dstBuff);
|
||||
free(buff2);
|
||||
ZSTD_freeDCtx(g_dctxPtr);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@ -47,7 +47,6 @@
|
||||
#include <sys/timeb.h> /* timeb */
|
||||
#include <string.h> /* strcmp */
|
||||
#include "zstd_static.h"
|
||||
#include "zstdhc_static.h"
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
#include "xxhash.h" /* XXH64 */
|
||||
#include "mem.h"
|
||||
@ -64,7 +63,7 @@
|
||||
#define MB *(1U<<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
static const U32 nbTestsDefault = 32 KB;
|
||||
static const U32 nbTestsDefault = 30000;
|
||||
#define COMPRESSIBLE_NOISE_LENGTH (10 MB)
|
||||
#define FUZ_COMPRESSIBILITY_DEFAULT 50
|
||||
static const U32 prime1 = 2654435761U;
|
||||
@ -90,6 +89,8 @@ static U32 g_time = 0;
|
||||
/*********************************************************
|
||||
* Fuzzer functions
|
||||
*********************************************************/
|
||||
#define MAX(a,b) ((a)>(b)?(a):(b))
|
||||
|
||||
static U32 FUZ_GetMilliStart(void)
|
||||
{
|
||||
struct timeb tb;
|
||||
@ -159,7 +160,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
|
||||
/* Basic tests */
|
||||
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||
result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), CNBuffer, COMPRESSIBLE_NOISE_LENGTH);
|
||||
result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
|
||||
if (ZSTD_isError(result)) goto _output_error;
|
||||
cSize = result;
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
|
||||
@ -213,7 +214,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
sampleSize += 256 KB - 1;
|
||||
RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., randState);
|
||||
sampleSize += 96 KB;
|
||||
cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize);
|
||||
cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
|
||||
if (ZSTD_isError(cSize)) goto _output_error;
|
||||
result = ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize);
|
||||
if (ZSTD_isError(result)) goto _output_error;
|
||||
@ -265,11 +266,9 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
|
||||
U32 testNb = 0;
|
||||
U32 coreSeed = seed, lseed = 0;
|
||||
ZSTD_CCtx* ctx;
|
||||
ZSTD_HC_CCtx* hcctx;
|
||||
|
||||
/* allocation */
|
||||
ctx = ZSTD_createCCtx();
|
||||
hcctx = ZSTD_HC_createCCtx();
|
||||
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
|
||||
@ -277,7 +276,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
|
||||
cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
|
||||
dstBuffer = (BYTE*)malloc (dstBufferSize);
|
||||
cBuffer = (BYTE*)malloc (cBufferSize);
|
||||
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !dstBuffer || !cBuffer || !ctx || !hcctx,
|
||||
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !dstBuffer || !cBuffer || !ctx,
|
||||
"Not enough memory, fuzzer tests cancelled");
|
||||
|
||||
/* Create initial samples */
|
||||
@ -300,6 +299,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
|
||||
U32 sampleSizeLog, buffNb, cLevelMod;
|
||||
U64 crcOrig, crcDest;
|
||||
int cLevel;
|
||||
BYTE* sampleBuffer;
|
||||
|
||||
/* init */
|
||||
DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests);
|
||||
@ -326,14 +326,18 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
|
||||
sampleSize = (size_t)1 << sampleSizeLog;
|
||||
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
|
||||
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
|
||||
crcOrig = XXH64(srcBuffer + sampleStart, sampleSize, 0);
|
||||
|
||||
/* HC compression test */
|
||||
#define MAX(a,b) ((a)>(b)?(a):(b))
|
||||
/* create sample buffer (to catch read error with valgrind & sanitizers) */
|
||||
sampleBuffer = (BYTE*)malloc(sampleSize);
|
||||
CHECK (sampleBuffer==NULL, "not enough memory for sample buffer");
|
||||
memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize);
|
||||
crcOrig = XXH64(sampleBuffer, sampleSize, 0);
|
||||
|
||||
/* compression test */
|
||||
cLevelMod = MAX(1, 38 - (int)(MAX(9, sampleSizeLog) * 2)); /* use high compression levels with small samples, for speed */
|
||||
cLevel = (FUZ_rand(&lseed) % cLevelMod) +1;
|
||||
cSize = ZSTD_HC_compressCCtx(hcctx, cBuffer, cBufferSize, srcBuffer + sampleStart, sampleSize, cLevel);
|
||||
CHECK(ZSTD_isError(cSize), "ZSTD_HC_compressCCtx failed");
|
||||
cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
|
||||
CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed");
|
||||
|
||||
/* compression failure test : too small dest buffer */
|
||||
if (cSize > 3)
|
||||
@ -344,10 +348,10 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
|
||||
static const U32 endMark = 0x4DC2B1A9;
|
||||
U32 endCheck;
|
||||
memcpy(dstBuffer+tooSmallSize, &endMark, 4);
|
||||
errorCode = ZSTD_HC_compressCCtx(hcctx, dstBuffer, tooSmallSize, srcBuffer + sampleStart, sampleSize, cLevel);
|
||||
CHECK(!ZSTD_isError(errorCode), "ZSTD_HC_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize);
|
||||
errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel);
|
||||
CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize);
|
||||
memcpy(&endCheck, dstBuffer+tooSmallSize, 4);
|
||||
CHECK(endCheck != endMark, "ZSTD_HC_compressCCtx : dst buffer overflow");
|
||||
CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow");
|
||||
}
|
||||
|
||||
/* successfull decompression tests*/
|
||||
@ -355,7 +359,9 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
|
||||
dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize);
|
||||
CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize);
|
||||
crcDest = XXH64(dstBuffer, sampleSize, 0);
|
||||
CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(srcBuffer+sampleStart, dstBuffer, sampleSize), (U32)sampleSize);
|
||||
CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(sampleBuffer, dstBuffer, sampleSize), (U32)sampleSize);
|
||||
|
||||
free(sampleBuffer); /* no longer useful after this point */
|
||||
|
||||
/* truncated src decompression test */
|
||||
{
|
||||
@ -434,7 +440,6 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
|
||||
|
||||
_cleanup:
|
||||
ZSTD_freeCCtx(ctx);
|
||||
ZSTD_HC_freeCCtx(hcctx);
|
||||
free(cNoiseBuffer[0]);
|
||||
free(cNoiseBuffer[1]);
|
||||
free(cNoiseBuffer[2]);
|
||||
|
||||
@ -43,6 +43,10 @@
|
||||
# define BMK_LEGACY_TIMER 1
|
||||
#endif
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
# define snprintf _snprintf /* snprintf unsupported by Visual <= 2012 */
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
* Includes
|
||||
@ -62,8 +66,7 @@
|
||||
#endif
|
||||
|
||||
#include "mem.h"
|
||||
#include "zstdhc_static.h"
|
||||
#include "zstd.h"
|
||||
#include "zstd_static.h"
|
||||
#include "datagen.h"
|
||||
#include "xxhash.h"
|
||||
|
||||
@ -122,8 +125,7 @@ static U32 g_rand = 1;
|
||||
static U32 g_singleRun = 0;
|
||||
static U32 g_target = 0;
|
||||
static U32 g_noSeed = 0;
|
||||
static const ZSTD_HC_parameters* g_seedParams = ZSTD_HC_defaultParameters[0];
|
||||
static ZSTD_HC_parameters g_params = { 0, 0, 0, 0, 0, ZSTD_HC_greedy };
|
||||
static ZSTD_parameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy };
|
||||
|
||||
void BMK_SetNbIterations(int nbLoops)
|
||||
{
|
||||
@ -136,28 +138,6 @@ void BMK_SetNbIterations(int nbLoops)
|
||||
* Private functions
|
||||
*********************************************************/
|
||||
|
||||
static unsigned BMK_highbit(U32 val)
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
unsigned long r;
|
||||
_BitScanReverse(&r, val);
|
||||
return (unsigned)r;
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
|
||||
return 31 - __builtin_clz(val);
|
||||
# else /* Software version */
|
||||
static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
||||
U32 v = val;
|
||||
int r;
|
||||
v |= v >> 1;
|
||||
v |= v >> 2;
|
||||
v |= v >> 4;
|
||||
v |= v >> 8;
|
||||
v |= v >> 16;
|
||||
r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
|
||||
return r;
|
||||
# endif
|
||||
}
|
||||
|
||||
#if defined(BMK_LEGACY_TIMER)
|
||||
|
||||
static int BMK_GetMilliStart(void)
|
||||
@ -270,8 +250,8 @@ typedef struct
|
||||
|
||||
static size_t BMK_benchParam(BMK_result_t* resultPtr,
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_HC_CCtx* ctx,
|
||||
const ZSTD_HC_parameters params)
|
||||
ZSTD_CCtx* ctx,
|
||||
const ZSTD_parameters params)
|
||||
{
|
||||
const size_t blockSize = g_blockSize ? g_blockSize : srcSize;
|
||||
const U32 nbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize);
|
||||
@ -284,7 +264,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
|
||||
U32 Hlog = params.hashLog;
|
||||
U32 Slog = params.searchLog;
|
||||
U32 Slength = params.searchLength;
|
||||
ZSTD_HC_strategy strat = params.strategy;
|
||||
ZSTD_strategy strat = params.strategy;
|
||||
char name[30] = { 0 };
|
||||
U64 crcOrig;
|
||||
|
||||
@ -358,7 +338,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
|
||||
while (BMK_GetMilliSpan(milliTime) < TIMELOOP)
|
||||
{
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++)
|
||||
blockTable[blockNb].cSize = ZSTD_HC_compress_advanced(ctx,
|
||||
blockTable[blockNb].cSize = ZSTD_compress_advanced(ctx,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
|
||||
blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize,
|
||||
params);
|
||||
@ -429,13 +409,13 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
|
||||
}
|
||||
|
||||
|
||||
const char* g_stratName[] = { "ZSTD_HC_fast ",
|
||||
"ZSTD_HC_greedy ",
|
||||
"ZSTD_HC_lazy ",
|
||||
"ZSTD_HC_lazy2 ",
|
||||
"ZSTD_HC_btlazy2" };
|
||||
const char* g_stratName[] = { "ZSTD_fast ",
|
||||
"ZSTD_greedy ",
|
||||
"ZSTD_lazy ",
|
||||
"ZSTD_lazy2 ",
|
||||
"ZSTD_btlazy2" };
|
||||
|
||||
static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_HC_parameters params, size_t srcSize)
|
||||
static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_parameters params, size_t srcSize)
|
||||
{
|
||||
DISPLAY("\r%79s\r", "");
|
||||
fprintf(f," {%3u,%3u,%3u,%3u,%3u, %s }, ",
|
||||
@ -447,11 +427,11 @@ static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_HC_pa
|
||||
}
|
||||
|
||||
|
||||
static U32 g_cSpeedTarget[ZSTD_HC_MAX_CLEVEL+1] = { 0 };
|
||||
static U32 g_cSpeedTarget[ZSTD_MAX_CLEVEL+1] = { 0 };
|
||||
|
||||
typedef struct {
|
||||
BMK_result_t result;
|
||||
ZSTD_HC_parameters params;
|
||||
ZSTD_parameters params;
|
||||
} winnerInfo_t;
|
||||
|
||||
static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize)
|
||||
@ -459,11 +439,11 @@ static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSi
|
||||
int cLevel;
|
||||
|
||||
fprintf(f, "\n /* Selected configurations : */ \n");
|
||||
fprintf(f, "#define ZSTD_HC_MAX_CLEVEL %2u \n", ZSTD_HC_MAX_CLEVEL);
|
||||
fprintf(f, "static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] = {\n");
|
||||
fprintf(f, "#define ZSTD_MAX_CLEVEL %2u \n", ZSTD_MAX_CLEVEL);
|
||||
fprintf(f, "static const ZSTD_parameters ZSTD_defaultParameters[ZSTD_MAX_CLEVEL+1] = {\n");
|
||||
fprintf(f, " /* W, C, H, S, L, strat */ \n");
|
||||
|
||||
for (cLevel=0; cLevel <= ZSTD_HC_MAX_CLEVEL; cLevel++)
|
||||
for (cLevel=0; cLevel <= ZSTD_MAX_CLEVEL; cLevel++)
|
||||
BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize);
|
||||
}
|
||||
|
||||
@ -477,9 +457,9 @@ static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSiz
|
||||
}
|
||||
|
||||
|
||||
static int BMK_seed(winnerInfo_t* winners, const ZSTD_HC_parameters params,
|
||||
static int BMK_seed(winnerInfo_t* winners, const ZSTD_parameters params,
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_HC_CCtx* ctx)
|
||||
ZSTD_CCtx* ctx)
|
||||
{
|
||||
BMK_result_t testResult;
|
||||
int better = 0;
|
||||
@ -487,7 +467,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_HC_parameters params,
|
||||
|
||||
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params);
|
||||
|
||||
for (cLevel = 1; cLevel <= ZSTD_HC_MAX_CLEVEL; cLevel++)
|
||||
for (cLevel = 1; cLevel <= ZSTD_MAX_CLEVEL; cLevel++)
|
||||
{
|
||||
if (testResult.cSpeed < g_cSpeedTarget[cLevel])
|
||||
continue; /* not fast enough for this level */
|
||||
@ -514,9 +494,9 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_HC_parameters params,
|
||||
double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed);
|
||||
|
||||
size_t W_CMemUsed = (1 << params.windowLog) + 4 * (1 << params.hashLog) +
|
||||
((params.strategy==ZSTD_HC_fast) ? 0 : 4 * (1 << params.contentLog));
|
||||
((params.strategy==ZSTD_fast) ? 0 : 4 * (1 << params.contentLog));
|
||||
size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + 4 * (1 << winners[cLevel].params.hashLog) +
|
||||
((winners[cLevel].params.strategy==ZSTD_HC_fast) ? 0 : 4 * (1 << winners[cLevel].params.contentLog));
|
||||
((winners[cLevel].params.strategy==ZSTD_fast) ? 0 : 4 * (1 << winners[cLevel].params.contentLog));
|
||||
double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed);
|
||||
double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);
|
||||
|
||||
@ -581,10 +561,10 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_HC_parameters params,
|
||||
|
||||
|
||||
/* nullified useless params, to ensure count stats */
|
||||
static ZSTD_HC_parameters* sanitizeParams(ZSTD_HC_parameters params)
|
||||
static ZSTD_parameters* sanitizeParams(ZSTD_parameters params)
|
||||
{
|
||||
g_params = params;
|
||||
if (params.strategy == ZSTD_HC_fast)
|
||||
if (params.strategy == ZSTD_fast)
|
||||
{
|
||||
g_params.contentLog = 0;
|
||||
g_params.searchLog = 0;
|
||||
@ -604,16 +584,16 @@ static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */
|
||||
#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
|
||||
|
||||
static void playAround(FILE* f, winnerInfo_t* winners,
|
||||
ZSTD_HC_parameters params,
|
||||
ZSTD_parameters params,
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_HC_CCtx* ctx)
|
||||
ZSTD_CCtx* ctx)
|
||||
{
|
||||
int nbVariations = 0;
|
||||
const int startTime = BMK_GetMilliStart();
|
||||
|
||||
while (BMK_GetMilliSpan(startTime) < g_maxVariationTime)
|
||||
{
|
||||
ZSTD_HC_parameters p = params;
|
||||
ZSTD_parameters p = params;
|
||||
U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1;
|
||||
if (nbVariations++ > g_maxNbVariations) break;
|
||||
|
||||
@ -643,16 +623,16 @@ static void playAround(FILE* f, winnerInfo_t* winners,
|
||||
case 9:
|
||||
p.searchLength--; break;
|
||||
case 10:
|
||||
p.strategy = (ZSTD_HC_strategy)(((U32)p.strategy)+1); break;
|
||||
p.strategy = (ZSTD_strategy)(((U32)p.strategy)+1); break;
|
||||
case 11:
|
||||
p.strategy = (ZSTD_HC_strategy)(((U32)p.strategy)-1); break;
|
||||
p.strategy = (ZSTD_strategy)(((U32)p.strategy)-1); break;
|
||||
}
|
||||
}
|
||||
|
||||
/* validate new conf */
|
||||
{
|
||||
ZSTD_HC_parameters saved = p;
|
||||
ZSTD_HC_validateParams(&p, g_blockSize ? g_blockSize : srcSize);
|
||||
ZSTD_parameters saved = p;
|
||||
ZSTD_validateParams(&p);
|
||||
if (memcmp(&p, &saved, sizeof(p))) continue; /* p was invalid */
|
||||
}
|
||||
|
||||
@ -675,19 +655,19 @@ static void playAround(FILE* f, winnerInfo_t* winners,
|
||||
static void BMK_selectRandomStart(
|
||||
FILE* f, winnerInfo_t* winners,
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_HC_CCtx* ctx)
|
||||
ZSTD_CCtx* ctx)
|
||||
{
|
||||
U32 id = (FUZ_rand(&g_rand) % (ZSTD_HC_MAX_CLEVEL+1));
|
||||
U32 id = (FUZ_rand(&g_rand) % (ZSTD_MAX_CLEVEL+1));
|
||||
if ((id==0) || (winners[id].params.windowLog==0))
|
||||
{
|
||||
/* totally random entry */
|
||||
ZSTD_HC_parameters p;
|
||||
p.contentLog = FUZ_rand(&g_rand) % (ZSTD_HC_CONTENTLOG_MAX+1 - ZSTD_HC_CONTENTLOG_MIN) + ZSTD_HC_CONTENTLOG_MIN;
|
||||
p.hashLog = FUZ_rand(&g_rand) % (ZSTD_HC_HASHLOG_MAX+1 - ZSTD_HC_HASHLOG_MIN) + ZSTD_HC_HASHLOG_MIN;
|
||||
p.searchLog = FUZ_rand(&g_rand) % (ZSTD_HC_SEARCHLOG_MAX+1 - ZSTD_HC_SEARCHLOG_MIN) + ZSTD_HC_SEARCHLOG_MIN;
|
||||
p.windowLog = FUZ_rand(&g_rand) % (ZSTD_HC_WINDOWLOG_MAX+1 - ZSTD_HC_WINDOWLOG_MIN) + ZSTD_HC_WINDOWLOG_MIN;
|
||||
p.searchLength=FUZ_rand(&g_rand) % (ZSTD_HC_SEARCHLENGTH_MAX+1 - ZSTD_HC_SEARCHLENGTH_MIN) + ZSTD_HC_SEARCHLENGTH_MIN;
|
||||
p.strategy = (ZSTD_HC_strategy) (FUZ_rand(&g_rand) % (ZSTD_HC_btlazy2+1));
|
||||
ZSTD_parameters p;
|
||||
p.contentLog = FUZ_rand(&g_rand) % (ZSTD_CONTENTLOG_MAX+1 - ZSTD_CONTENTLOG_MIN) + ZSTD_CONTENTLOG_MIN;
|
||||
p.hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN;
|
||||
p.searchLog = FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN) + ZSTD_SEARCHLOG_MIN;
|
||||
p.windowLog = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN;
|
||||
p.searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN;
|
||||
p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btlazy2+1));
|
||||
playAround(f, winners, p, srcBuffer, srcSize, ctx);
|
||||
}
|
||||
else
|
||||
@ -697,19 +677,19 @@ static void BMK_selectRandomStart(
|
||||
|
||||
static void BMK_benchMem(void* srcBuffer, size_t srcSize)
|
||||
{
|
||||
ZSTD_HC_CCtx* ctx = ZSTD_HC_createCCtx();
|
||||
ZSTD_HC_parameters params;
|
||||
winnerInfo_t winners[ZSTD_HC_MAX_CLEVEL+1];
|
||||
ZSTD_CCtx* ctx = ZSTD_createCCtx();
|
||||
ZSTD_parameters params;
|
||||
winnerInfo_t winners[ZSTD_MAX_CLEVEL+1];
|
||||
int i;
|
||||
const char* rfName = "grillResults.txt";
|
||||
FILE* f;
|
||||
const size_t blockSize = g_blockSize ? g_blockSize : srcSize;
|
||||
const U32 srcLog = BMK_highbit((U32)(blockSize-1))+1;
|
||||
|
||||
if (g_singleRun)
|
||||
{
|
||||
BMK_result_t testResult;
|
||||
ZSTD_HC_validateParams(&g_params, blockSize);
|
||||
g_params.srcSize = blockSize;
|
||||
ZSTD_validateParams(&g_params);
|
||||
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params);
|
||||
DISPLAY("\n");
|
||||
return;
|
||||
@ -731,28 +711,23 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
|
||||
params.contentLog = 1;
|
||||
params.searchLog = 1;
|
||||
params.searchLength = 7;
|
||||
params.strategy = ZSTD_HC_fast;
|
||||
ZSTD_HC_validateParams(¶ms, blockSize);
|
||||
params.strategy = ZSTD_fast;
|
||||
params.srcSize = blockSize;
|
||||
ZSTD_validateParams(¶ms);
|
||||
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params);
|
||||
g_cSpeedTarget[1] = (testResult.cSpeed * 15) >> 4;
|
||||
g_cSpeedTarget[1] = (testResult.cSpeed * 31) >> 5;
|
||||
}
|
||||
|
||||
/* establish speed objectives (relative to level 1) */
|
||||
for (i=2; i<=ZSTD_HC_MAX_CLEVEL; i++)
|
||||
for (i=2; i<=ZSTD_MAX_CLEVEL; i++)
|
||||
g_cSpeedTarget[i] = (g_cSpeedTarget[i-1] * 25) >> 5;
|
||||
|
||||
/* populate initial solution */
|
||||
{
|
||||
const int tableID = (blockSize > 128 KB);
|
||||
const int maxSeeds = g_noSeed ? 1 : ZSTD_HC_MAX_CLEVEL;
|
||||
g_seedParams = ZSTD_HC_defaultParameters[tableID];
|
||||
const int maxSeeds = g_noSeed ? 1 : ZSTD_MAX_CLEVEL;
|
||||
for (i=1; i<=maxSeeds; i++)
|
||||
{
|
||||
const U32 btPlus = (params.strategy == ZSTD_HC_btlazy2);
|
||||
params = g_seedParams[i];
|
||||
params.windowLog = MIN(srcLog, params.windowLog);
|
||||
params.contentLog = MIN(params.windowLog+btPlus, params.contentLog);
|
||||
params.searchLog = MIN(params.contentLog, params.searchLog);
|
||||
params = ZSTD_getParams(i, blockSize);
|
||||
BMK_seed(winners, params, srcBuffer, srcSize, ctx);
|
||||
}
|
||||
}
|
||||
@ -775,7 +750,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
|
||||
|
||||
/* clean up*/
|
||||
fclose(f);
|
||||
ZSTD_HC_freeCCtx(ctx);
|
||||
ZSTD_freeCCtx(ctx);
|
||||
}
|
||||
|
||||
|
||||
@ -874,6 +849,7 @@ int usage(char* exename)
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [arg] file\n", exename);
|
||||
DISPLAY( "Arguments :\n");
|
||||
DISPLAY( " file : path to the file used as reference (if none, generates a compressible sample)\n");
|
||||
DISPLAY( " -H/-h : Help (this text + advanced options)\n");
|
||||
return 0;
|
||||
}
|
||||
@ -882,7 +858,8 @@ int usage_advanced(void)
|
||||
{
|
||||
DISPLAY( "\nAdvanced options :\n");
|
||||
DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS);
|
||||
DISPLAY( " -P# : sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100);
|
||||
DISPLAY( " -B# : cut input into blocks of size # (default : single block)\n");
|
||||
DISPLAY( " -P# : generated sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -958,7 +935,7 @@ int main(int argc, char** argv)
|
||||
case 'S':
|
||||
g_singleRun = 1;
|
||||
argument++;
|
||||
g_params = g_seedParams[2];
|
||||
g_params = ZSTD_getParams(2, g_blockSize);
|
||||
for ( ; ; )
|
||||
{
|
||||
switch(*argument)
|
||||
@ -994,12 +971,12 @@ int main(int argc, char** argv)
|
||||
g_params.searchLength *= 10, g_params.searchLength += *argument++ - '0';
|
||||
continue;
|
||||
case 't': /* strategy */
|
||||
g_params.strategy = (ZSTD_HC_strategy)0;
|
||||
g_params.strategy = (ZSTD_strategy)0;
|
||||
argument++;
|
||||
while ((*argument>= '0') && (*argument<='9'))
|
||||
{
|
||||
g_params.strategy = (ZSTD_HC_strategy)((U32)g_params.strategy *10);
|
||||
g_params.strategy = (ZSTD_HC_strategy)((U32)g_params.strategy + *argument++ - '0');
|
||||
g_params.strategy = (ZSTD_strategy)((U32)g_params.strategy *10);
|
||||
g_params.strategy = (ZSTD_strategy)((U32)g_params.strategy + *argument++ - '0');
|
||||
}
|
||||
continue;
|
||||
case 'L':
|
||||
@ -1008,9 +985,7 @@ int main(int argc, char** argv)
|
||||
argument++;
|
||||
while ((*argument>= '0') && (*argument<='9'))
|
||||
cLevel *= 10, cLevel += *argument++ - '0';
|
||||
if (cLevel < 1) cLevel = 1;
|
||||
if (cLevel > ZSTD_HC_MAX_CLEVEL) cLevel = ZSTD_HC_MAX_CLEVEL;
|
||||
g_params = g_seedParams[cLevel];
|
||||
g_params = ZSTD_getParams(cLevel, g_blockSize);
|
||||
continue;
|
||||
}
|
||||
default : ;
|
||||
|
||||
587
programs/zbufftest.c
Normal file
587
programs/zbufftest.c
Normal file
@ -0,0 +1,587 @@
|
||||
/*
|
||||
Fuzzer test tool for zstd_buffered
|
||||
Copyright (C) Yann Collet 2105
|
||||
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- ZSTD source repository : https://github.com/Cyan4973/zstd
|
||||
- ZSTD public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
|
||||
/**************************************
|
||||
* Compiler specific
|
||||
**************************************/
|
||||
#ifdef _MSC_VER /* Visual Studio */
|
||||
# define _CRT_SECURE_NO_WARNINGS /* fgets */
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
|
||||
#endif
|
||||
|
||||
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
||||
#ifdef __GNUC__
|
||||
# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
|
||||
# pragma GCC diagnostic ignored "-Wmissing-field-initializers" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
* Includes
|
||||
**************************************/
|
||||
#include <stdlib.h> /* free */
|
||||
#include <stdio.h> /* fgets, sscanf */
|
||||
#include <sys/timeb.h> /* timeb */
|
||||
#include <string.h> /* strcmp */
|
||||
#include "mem.h"
|
||||
#include "zstd_buffered.h"
|
||||
#include "zstd.h" /* ZSTD_compressBound() */
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
#include "xxhash.h" /* XXH64 */
|
||||
|
||||
|
||||
/**************************************
|
||||
Constants
|
||||
**************************************/
|
||||
#ifndef ZSTD_VERSION
|
||||
# define ZSTD_VERSION ""
|
||||
#endif
|
||||
|
||||
#define KB *(1U<<10)
|
||||
#define MB *(1U<<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
static const U32 nbTestsDefault = 10000;
|
||||
#define COMPRESSIBLE_NOISE_LENGTH (10 MB)
|
||||
#define FUZ_COMPRESSIBILITY_DEFAULT 50
|
||||
static const U32 prime1 = 2654435761U;
|
||||
static const U32 prime2 = 2246822519U;
|
||||
|
||||
|
||||
|
||||
/**************************************
|
||||
* Display Macros
|
||||
**************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
static U32 g_displayLevel = 2;
|
||||
|
||||
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
|
||||
if ((FUZ_GetMilliSpan(g_displayTime) > g_refreshRate) || (g_displayLevel>=4)) \
|
||||
{ g_displayTime = FUZ_GetMilliStart(); DISPLAY(__VA_ARGS__); \
|
||||
if (g_displayLevel>=4) fflush(stdout); } }
|
||||
static const U32 g_refreshRate = 150;
|
||||
static U32 g_displayTime = 0;
|
||||
|
||||
static U32 g_testTime = 0;
|
||||
|
||||
|
||||
/*********************************************************
|
||||
* Fuzzer functions
|
||||
*********************************************************/
|
||||
#define MAX(a,b) ((a)>(b)?(a):(b))
|
||||
|
||||
static U32 FUZ_GetMilliStart(void)
|
||||
{
|
||||
struct timeb tb;
|
||||
U32 nCount;
|
||||
ftime( &tb );
|
||||
nCount = (U32) (((tb.time & 0xFFFFF) * 1000) + tb.millitm);
|
||||
return nCount;
|
||||
}
|
||||
|
||||
|
||||
static U32 FUZ_GetMilliSpan(U32 nTimeStart)
|
||||
{
|
||||
U32 nCurrent = FUZ_GetMilliStart();
|
||||
U32 nSpan = nCurrent - nTimeStart;
|
||||
if (nTimeStart > nCurrent)
|
||||
nSpan += 0x100000 * 1000;
|
||||
return nSpan;
|
||||
}
|
||||
|
||||
|
||||
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
|
||||
unsigned int FUZ_rand(unsigned int* src)
|
||||
{
|
||||
U32 rand32 = *src;
|
||||
rand32 *= prime1;
|
||||
rand32 += prime2;
|
||||
rand32 = FUZ_rotl32(rand32, 13);
|
||||
*src = rand32;
|
||||
return rand32 >> 5;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
static unsigned FUZ_highbit32(U32 v32)
|
||||
{
|
||||
unsigned nbBits = 0;
|
||||
if (v32==0) return 0;
|
||||
for ( ; v32 ; v32>>=1) nbBits++;
|
||||
return nbBits;
|
||||
}
|
||||
*/
|
||||
|
||||
static int basicUnitTests(U32 seed, double compressibility)
|
||||
{
|
||||
int testResult = 0;
|
||||
void* CNBuffer;
|
||||
size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
|
||||
void* compressedBuffer;
|
||||
size_t compressedBufferSize = ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
|
||||
void* decodedBuffer;
|
||||
size_t decodedBufferSize = CNBufferSize;
|
||||
U32 randState = seed;
|
||||
size_t result, cSize, readSize, genSize;
|
||||
U32 testNb=0;
|
||||
ZBUFF_CCtx* zc = ZBUFF_createCCtx();
|
||||
ZBUFF_DCtx* zd = ZBUFF_createDCtx();
|
||||
|
||||
/* Create compressible test buffer */
|
||||
CNBuffer = malloc(CNBufferSize);
|
||||
compressedBuffer = malloc(compressedBufferSize);
|
||||
decodedBuffer = malloc(decodedBufferSize);
|
||||
if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd)
|
||||
{
|
||||
DISPLAY("Not enough memory, aborting\n");
|
||||
goto _output_error;
|
||||
}
|
||||
RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., randState);
|
||||
|
||||
/* Basic compression test */
|
||||
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||
ZBUFF_compressInit(zc, 1);
|
||||
readSize = CNBufferSize;
|
||||
genSize = compressedBufferSize;
|
||||
result = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize);
|
||||
if (ZBUFF_isError(result)) goto _output_error;
|
||||
if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */
|
||||
cSize = genSize;
|
||||
genSize = compressedBufferSize - cSize;
|
||||
result = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize);
|
||||
if (result != 0) goto _output_error; /* error, or some data not flushed */
|
||||
cSize += genSize;
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
|
||||
|
||||
/* Basic decompression test */
|
||||
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||
ZBUFF_decompressInit(zd);
|
||||
readSize = cSize;
|
||||
genSize = CNBufferSize;
|
||||
result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSize);
|
||||
if (result != 0) goto _output_error; /* should reach end of frame == 0; otherwise, some data left, or an error */
|
||||
if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */
|
||||
if (readSize != cSize) goto _output_error; /* should have read the entire frame */
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* check regenerated data is byte exact */
|
||||
{
|
||||
size_t i;
|
||||
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
||||
for (i=0; i<CNBufferSize; i++)
|
||||
{
|
||||
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
}
|
||||
|
||||
_end:
|
||||
ZBUFF_freeCCtx(zc);
|
||||
ZBUFF_freeDCtx(zd);
|
||||
free(CNBuffer);
|
||||
free(compressedBuffer);
|
||||
free(decodedBuffer);
|
||||
return testResult;
|
||||
|
||||
_output_error:
|
||||
testResult = 1;
|
||||
DISPLAY("Error detected in Unit tests ! \n");
|
||||
goto _end;
|
||||
}
|
||||
|
||||
|
||||
static size_t findDiff(const void* buf1, const void* buf2, size_t max)
|
||||
{
|
||||
const BYTE* b1 = (const BYTE*)buf1;
|
||||
const BYTE* b2 = (const BYTE*)buf2;
|
||||
size_t i;
|
||||
for (i=0; i<max; i++)
|
||||
{
|
||||
if (b1[i] != b2[i]) break;
|
||||
}
|
||||
return i;
|
||||
}
|
||||
|
||||
# define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
|
||||
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
|
||||
|
||||
static const U32 maxSrcLog = 24;
|
||||
static const U32 maxSampleLog = 19;
|
||||
|
||||
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
|
||||
{
|
||||
BYTE* cNoiseBuffer[5];
|
||||
BYTE* srcBuffer;
|
||||
size_t srcBufferSize = (size_t)1<<maxSrcLog;
|
||||
BYTE* copyBuffer;
|
||||
size_t copyBufferSize = srcBufferSize + (1<<maxSampleLog);
|
||||
BYTE* cBuffer;
|
||||
size_t cBufferSize = ZSTD_compressBound(srcBufferSize);
|
||||
BYTE* dstBuffer;
|
||||
size_t dstBufferSize = srcBufferSize;
|
||||
U32 result = 0;
|
||||
U32 testNb = 0;
|
||||
U32 coreSeed = seed, lseed = 0;
|
||||
ZBUFF_CCtx* zc;
|
||||
ZBUFF_DCtx* zd;
|
||||
XXH64_state_t crc64;
|
||||
U32 startTime = FUZ_GetMilliStart();
|
||||
|
||||
/* allocation */
|
||||
zc = ZBUFF_createCCtx();
|
||||
zd = ZBUFF_createDCtx();
|
||||
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
|
||||
copyBuffer= (BYTE*)malloc (copyBufferSize);
|
||||
dstBuffer = (BYTE*)malloc (dstBufferSize);
|
||||
cBuffer = (BYTE*)malloc (cBufferSize);
|
||||
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
|
||||
!copyBuffer || !dstBuffer || !cBuffer || !zc || !zd,
|
||||
"Not enough memory, fuzzer tests cancelled");
|
||||
|
||||
/* Create initial samples */
|
||||
RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
|
||||
RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
|
||||
RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
|
||||
RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
|
||||
RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
|
||||
srcBuffer = cNoiseBuffer[2];
|
||||
|
||||
/* catch up testNb */
|
||||
for (testNb=1; testNb < startTest; testNb++)
|
||||
FUZ_rand(&coreSeed);
|
||||
|
||||
/* test loop */
|
||||
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ )
|
||||
{
|
||||
size_t sampleSize, sampleStart;
|
||||
size_t cSize;
|
||||
size_t maxTestSize, totalTestSize, readSize, totalCSize, genSize, totalGenSize;
|
||||
size_t errorCode;
|
||||
U32 sampleSizeLog, buffNb, n, nbChunks;
|
||||
U64 crcOrig, crcDest;
|
||||
|
||||
/* init */
|
||||
DISPLAYUPDATE(2, "\r%6u", testNb);
|
||||
if (nbTests >= testNb) DISPLAYUPDATE(2, "/%6u ", nbTests);
|
||||
FUZ_rand(&coreSeed);
|
||||
lseed = coreSeed ^ prime1;
|
||||
buffNb = FUZ_rand(&lseed) & 127;
|
||||
if (buffNb & 7) buffNb=2; /* select buffer */
|
||||
else
|
||||
{
|
||||
buffNb >>= 3;
|
||||
if (buffNb & 7)
|
||||
{
|
||||
const U32 tnb[2] = { 1, 3 };
|
||||
buffNb = tnb[buffNb >> 3];
|
||||
}
|
||||
else
|
||||
{
|
||||
const U32 tnb[2] = { 0, 4 };
|
||||
buffNb = tnb[buffNb >> 3];
|
||||
}
|
||||
}
|
||||
srcBuffer = cNoiseBuffer[buffNb];
|
||||
|
||||
/* Multi - segments compression test */
|
||||
XXH64_reset(&crc64, 0);
|
||||
nbChunks = (FUZ_rand(&lseed) & 127) + 2;
|
||||
sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog;
|
||||
maxTestSize = (size_t)1 << sampleSizeLog;
|
||||
maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1);
|
||||
ZBUFF_compressInit(zc, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1);
|
||||
totalTestSize = 0;
|
||||
cSize = 0;
|
||||
for (n=0; n<nbChunks; n++)
|
||||
{
|
||||
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
|
||||
sampleSize = (size_t)1 << sampleSizeLog;
|
||||
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
|
||||
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
|
||||
|
||||
XXH64_update(&crc64, srcBuffer+sampleStart, sampleSize);
|
||||
memcpy(copyBuffer+totalTestSize, srcBuffer+sampleStart, sampleSize);
|
||||
|
||||
readSize = sampleSize;
|
||||
genSize = cBufferSize - cSize;
|
||||
errorCode = ZBUFF_compressContinue(zc, cBuffer+cSize, &genSize, srcBuffer+sampleStart, &readSize);
|
||||
CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
|
||||
CHECK (readSize != sampleSize, "compression test condition not respected : input should be fully consumed")
|
||||
cSize += genSize;
|
||||
totalTestSize += sampleSize;
|
||||
|
||||
if ((FUZ_rand(&lseed) & 15) == 0)
|
||||
{
|
||||
/* add a few random flushes operations, to mess around */
|
||||
genSize = cBufferSize - cSize;
|
||||
errorCode = ZBUFF_compressFlush(zc, cBuffer+cSize, &genSize);
|
||||
CHECK (ZBUFF_isError(errorCode), "flush error : %s", ZBUFF_getErrorName(errorCode));
|
||||
cSize += genSize;
|
||||
}
|
||||
|
||||
if (totalTestSize > maxTestSize) break;
|
||||
}
|
||||
genSize = cBufferSize - cSize;
|
||||
errorCode = ZBUFF_compressEnd(zc, cBuffer+cSize, &genSize);
|
||||
CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
|
||||
CHECK (errorCode != 0, "frame epilogue not fully consumed");
|
||||
cSize += genSize;
|
||||
crcOrig = XXH64_digest(&crc64);
|
||||
|
||||
/* multi - fragments decompression test */
|
||||
ZBUFF_decompressInit(zd);
|
||||
totalCSize = 0;
|
||||
totalGenSize = 0;
|
||||
while (totalCSize < cSize)
|
||||
{
|
||||
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
|
||||
sampleSize = (size_t)1 << sampleSizeLog;
|
||||
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
|
||||
readSize = sampleSize;
|
||||
genSize = dstBufferSize - totalGenSize;
|
||||
errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize);
|
||||
CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode));
|
||||
totalGenSize += genSize;
|
||||
totalCSize += readSize;
|
||||
}
|
||||
CHECK (errorCode != 0, "frame not fully decoded");
|
||||
CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
|
||||
CHECK (totalCSize != cSize, "compressed data should be fully read")
|
||||
crcDest = XXH64(dstBuffer, totalTestSize, 0);
|
||||
if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
|
||||
CHECK (crcDest!=crcOrig, "decompressed data corrupted");
|
||||
|
||||
/* noisy/erroneous src decompression test */
|
||||
/* add some noise */
|
||||
nbChunks = (FUZ_rand(&lseed) & 7) + 2;
|
||||
for (n=0; n<nbChunks; n++)
|
||||
{
|
||||
size_t cStart;
|
||||
|
||||
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
|
||||
sampleSize = (size_t)1 << sampleSizeLog;
|
||||
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
|
||||
if (sampleSize > cSize/3) sampleSize = cSize/3;
|
||||
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
|
||||
cStart = FUZ_rand(&lseed) % (cSize - sampleSize);
|
||||
|
||||
memcpy(cBuffer+cStart, srcBuffer+sampleStart, sampleSize);
|
||||
}
|
||||
|
||||
/* try decompression on noisy data */
|
||||
ZBUFF_decompressInit(zd);
|
||||
totalCSize = 0;
|
||||
totalGenSize = 0;
|
||||
while (totalCSize < cSize)
|
||||
{
|
||||
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
|
||||
sampleSize = (size_t)1 << sampleSizeLog;
|
||||
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
|
||||
readSize = sampleSize;
|
||||
genSize = dstBufferSize - totalGenSize;
|
||||
errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize);
|
||||
if (ZBUFF_isError(errorCode)) break; /* error correctly detected */
|
||||
}
|
||||
}
|
||||
DISPLAY("\r%u fuzzer tests completed \n", testNb);
|
||||
|
||||
_cleanup:
|
||||
ZBUFF_freeCCtx(zc);
|
||||
ZBUFF_freeDCtx(zd);
|
||||
free(cNoiseBuffer[0]);
|
||||
free(cNoiseBuffer[1]);
|
||||
free(cNoiseBuffer[2]);
|
||||
free(cNoiseBuffer[3]);
|
||||
free(cNoiseBuffer[4]);
|
||||
free(copyBuffer);
|
||||
free(cBuffer);
|
||||
free(dstBuffer);
|
||||
return result;
|
||||
|
||||
_output_error:
|
||||
result = 1;
|
||||
goto _cleanup;
|
||||
}
|
||||
|
||||
|
||||
/*********************************************************
|
||||
* Command line
|
||||
*********************************************************/
|
||||
int FUZ_usage(char* programName)
|
||||
{
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [args]\n", programName);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Arguments :\n");
|
||||
DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault);
|
||||
DISPLAY( " -s# : Select seed (default:prompt user)\n");
|
||||
DISPLAY( " -t# : Select starting test number (default:0)\n");
|
||||
DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
|
||||
DISPLAY( " -v : verbose\n");
|
||||
DISPLAY( " -p : pause at the end\n");
|
||||
DISPLAY( " -h : display help and exit\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
U32 seed=0;
|
||||
int seedset=0;
|
||||
int argNb;
|
||||
int nbTests = nbTestsDefault;
|
||||
int testNb = 0;
|
||||
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
|
||||
int result=0;
|
||||
U32 mainPause = 0;
|
||||
char* programName;
|
||||
|
||||
/* Check command line */
|
||||
programName = argv[0];
|
||||
for(argNb=1; argNb<argc; argNb++)
|
||||
{
|
||||
char* argument = argv[argNb];
|
||||
|
||||
if(!argument) continue; /* Protection if argument empty */
|
||||
|
||||
/* Handle commands. Aggregated commands are allowed */
|
||||
if (argument[0]=='-')
|
||||
{
|
||||
argument++;
|
||||
|
||||
while (*argument!=0)
|
||||
{
|
||||
switch(*argument)
|
||||
{
|
||||
case 'h':
|
||||
return FUZ_usage(programName);
|
||||
case 'v':
|
||||
argument++;
|
||||
g_displayLevel=4;
|
||||
break;
|
||||
case 'q':
|
||||
argument++;
|
||||
g_displayLevel--;
|
||||
break;
|
||||
case 'p': /* pause at the end */
|
||||
argument++;
|
||||
mainPause = 1;
|
||||
break;
|
||||
|
||||
case 'i':
|
||||
argument++;
|
||||
nbTests=0; g_testTime=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
nbTests *= 10;
|
||||
nbTests += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
case 'T':
|
||||
argument++;
|
||||
nbTests=0; g_testTime=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
g_testTime *= 10;
|
||||
g_testTime += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (*argument=='m') g_testTime *=60, argument++;
|
||||
if (*argument=='n') argument++;
|
||||
g_testTime *= 1000;
|
||||
break;
|
||||
|
||||
case 's':
|
||||
argument++;
|
||||
seed=0;
|
||||
seedset=1;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
seed *= 10;
|
||||
seed += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
case 't':
|
||||
argument++;
|
||||
testNb=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
testNb *= 10;
|
||||
testNb += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
case 'P': /* compressibility % */
|
||||
argument++;
|
||||
proba=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
proba *= 10;
|
||||
proba += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (proba<0) proba=0;
|
||||
if (proba>100) proba=100;
|
||||
break;
|
||||
|
||||
default:
|
||||
return FUZ_usage(programName);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Get Seed */
|
||||
DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION);
|
||||
|
||||
if (!seedset) seed = FUZ_GetMilliStart() % 10000;
|
||||
DISPLAY("Seed = %u\n", seed);
|
||||
if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
|
||||
|
||||
if (nbTests<=0) nbTests=1;
|
||||
|
||||
if (testNb==0) result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */
|
||||
if (!result)
|
||||
result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
|
||||
if (mainPause)
|
||||
{
|
||||
int unused;
|
||||
DISPLAY("Press Enter \n");
|
||||
unused = getchar();
|
||||
(void)unused;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@ -70,7 +70,7 @@
|
||||
**************************************/
|
||||
#define COMPRESSOR_NAME "zstd command line interface"
|
||||
#ifndef ZSTD_VERSION
|
||||
# define ZSTD_VERSION "v0.3.6"
|
||||
# define ZSTD_VERSION "v0.4.0"
|
||||
#endif
|
||||
#define AUTHOR "Yann Collet"
|
||||
#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), ZSTD_VERSION, AUTHOR, __DATE__
|
||||
@ -163,7 +163,7 @@ static void waitEnter(void)
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char** argv)
|
||||
int main(int argCount, const char** argv)
|
||||
{
|
||||
int i,
|
||||
bench=0,
|
||||
@ -196,9 +196,9 @@ int main(int argc, char** argv)
|
||||
decode=1;
|
||||
|
||||
/* command switches */
|
||||
for(i=1; i<argc; i++)
|
||||
for(i=1; i<argCount; i++)
|
||||
{
|
||||
char* argument = argv[i];
|
||||
const char* argument = argv[i];
|
||||
|
||||
if(!argument) continue; /* Protection if argument empty */
|
||||
|
||||
@ -311,7 +311,7 @@ int main(int argc, char** argv)
|
||||
}
|
||||
|
||||
/* first provided filename is input */
|
||||
if (!inFileName) { inFileName = argument; fileNameStart = i; nbFiles = argc-i; continue; }
|
||||
if (!inFileName) { inFileName = argument; fileNameStart = i; nbFiles = argCount-i; continue; }
|
||||
|
||||
/* second provided filename is output */
|
||||
if (!outFileName)
|
||||
|
||||
@ -73,7 +73,7 @@
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
|
||||
<IncludePath>$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>true</RunCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
@ -95,7 +95,7 @@
|
||||
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<EnablePREfast>true</EnablePREfast>
|
||||
<AdditionalOptions>/analyze:stacksize19000 %(AdditionalOptions)</AdditionalOptions>
|
||||
<AdditionalOptions>/analyze:stacksize25000 %(AdditionalOptions)</AdditionalOptions>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
@ -163,7 +163,8 @@
|
||||
<ClCompile Include="..\..\..\lib\huff0.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v02.c" />
|
||||
<ClCompile Include="..\..\..\lib\zstd.c" />
|
||||
<ClCompile Include="..\..\..\lib\zstd_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||
<ClCompile Include="..\..\..\programs\fullbench.c" />
|
||||
</ItemGroup>
|
||||
|
||||
@ -18,9 +18,6 @@
|
||||
<ClCompile Include="..\..\..\lib\fse.c">
|
||||
<Filter>Fichiers sources</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\zstd.c">
|
||||
<Filter>Fichiers sources</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\programs\fullbench.c">
|
||||
<Filter>Fichiers sources</Filter>
|
||||
</ClCompile>
|
||||
@ -36,6 +33,12 @@
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v02.c">
|
||||
<Filter>Fichiers sources</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\zstd_compress.c">
|
||||
<Filter>Fichiers sources</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\zstd_decompress.c">
|
||||
<Filter>Fichiers sources</Filter>
|
||||
</ClCompile>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="..\..\..\lib\fse.h">
|
||||
|
||||
@ -73,7 +73,7 @@
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
|
||||
<IncludePath>$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>true</RunCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
@ -95,7 +95,7 @@
|
||||
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<EnablePREfast>true</EnablePREfast>
|
||||
<AdditionalOptions>/analyze:stacksize19000 %(AdditionalOptions)</AdditionalOptions>
|
||||
<AdditionalOptions>/analyze:stacksize25000 %(AdditionalOptions)</AdditionalOptions>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
@ -163,8 +163,8 @@
|
||||
<ClCompile Include="..\..\..\lib\huff0.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v02.c" />
|
||||
<ClCompile Include="..\..\..\lib\zstd.c" />
|
||||
<ClCompile Include="..\..\..\lib\zstdhc.c" />
|
||||
<ClCompile Include="..\..\..\lib\zstd_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\programs\datagen.c" />
|
||||
<ClCompile Include="..\..\..\programs\fuzzer.c" />
|
||||
<ClCompile Include="..\..\..\programs\xxhash.c" />
|
||||
@ -178,8 +178,6 @@
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v01.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v02.h" />
|
||||
<ClInclude Include="..\..\..\lib\zstd.h" />
|
||||
<ClInclude Include="..\..\..\lib\zstdhc.h" />
|
||||
<ClInclude Include="..\..\..\lib\zstdhc_static.h" />
|
||||
<ClInclude Include="..\..\..\lib\zstd_internal.h" />
|
||||
<ClInclude Include="..\..\..\lib\zstd_static.h" />
|
||||
<ClInclude Include="..\..\..\programs\datagen.h" />
|
||||
|
||||
@ -18,9 +18,6 @@
|
||||
<ClCompile Include="..\..\..\lib\fse.c">
|
||||
<Filter>Fichiers sources</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\zstd.c">
|
||||
<Filter>Fichiers sources</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\programs\fuzzer.c">
|
||||
<Filter>Fichiers sources</Filter>
|
||||
</ClCompile>
|
||||
@ -36,10 +33,13 @@
|
||||
<ClCompile Include="..\..\..\lib\huff0.c">
|
||||
<Filter>Fichiers sources</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\zstdhc.c">
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v02.c">
|
||||
<Filter>Fichiers sources</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v02.c">
|
||||
<ClCompile Include="..\..\..\lib\zstd_compress.c">
|
||||
<Filter>Fichiers sources</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\zstd_decompress.c">
|
||||
<Filter>Fichiers sources</Filter>
|
||||
</ClCompile>
|
||||
</ItemGroup>
|
||||
@ -74,12 +74,6 @@
|
||||
<ClInclude Include="..\..\..\lib\zstd_internal.h">
|
||||
<Filter>Fichiers d%27en-tête</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\zstdhc.h">
|
||||
<Filter>Fichiers d%27en-tête</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\zstdhc_static.h">
|
||||
<Filter>Fichiers d%27en-tête</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h">
|
||||
<Filter>Fichiers d%27en-tête</Filter>
|
||||
</ClInclude>
|
||||
|
||||
@ -23,8 +23,9 @@
|
||||
<ClCompile Include="..\..\..\lib\huff0.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v02.c" />
|
||||
<ClCompile Include="..\..\..\lib\zstd.c" />
|
||||
<ClCompile Include="..\..\..\lib\zstdhc.c" />
|
||||
<ClCompile Include="..\..\..\lib\zstd_buffered.c" />
|
||||
<ClCompile Include="..\..\..\lib\zstd_compress.c" />
|
||||
<ClCompile Include="..\..\..\lib\zstd_decompress.c" />
|
||||
<ClCompile Include="..\..\..\programs\bench.c" />
|
||||
<ClCompile Include="..\..\..\programs\fileio.c" />
|
||||
<ClCompile Include="..\..\..\programs\legacy\fileio_legacy.c" />
|
||||
@ -40,8 +41,8 @@
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v01.h" />
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_v02.h" />
|
||||
<ClInclude Include="..\..\..\lib\zstd.h" />
|
||||
<ClInclude Include="..\..\..\lib\zstdhc.h" />
|
||||
<ClInclude Include="..\..\..\lib\zstdhc_static.h" />
|
||||
<ClInclude Include="..\..\..\lib\zstd_buffered.h" />
|
||||
<ClInclude Include="..\..\..\lib\zstd_buffered_static.h" />
|
||||
<ClInclude Include="..\..\..\lib\zstd_internal.h" />
|
||||
<ClInclude Include="..\..\..\lib\zstd_static.h" />
|
||||
<ClInclude Include="..\..\..\programs\bench.h" />
|
||||
@ -104,7 +105,7 @@
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<LinkIncremental>true</LinkIncremental>
|
||||
<IncludePath>$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
|
||||
<IncludePath>$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>true</RunCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||
@ -115,7 +116,7 @@
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<LinkIncremental>false</LinkIncremental>
|
||||
<IncludePath>$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib;$(VCInstallDir)include;$(VCInstallDir)atlmfc\include;$(WindowsSDK_IncludePath);</IncludePath>
|
||||
<RunCodeAnalysis>true</RunCodeAnalysis>
|
||||
<RunCodeAnalysis>false</RunCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
|
||||
<ClCompile>
|
||||
@ -126,7 +127,7 @@
|
||||
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<EnablePREfast>true</EnablePREfast>
|
||||
<AdditionalOptions>/analyze:stacksize19000 %(AdditionalOptions)</AdditionalOptions>
|
||||
<AdditionalOptions>/analyze:stacksize25000 %(AdditionalOptions)</AdditionalOptions>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
@ -140,7 +141,7 @@
|
||||
<WarningLevel>Level4</WarningLevel>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<EnablePREfast>true</EnablePREfast>
|
||||
<AdditionalOptions>/analyze:stacksize19000 %(AdditionalOptions)</AdditionalOptions>
|
||||
</ClCompile>
|
||||
@ -178,9 +179,8 @@
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<TreatWarningAsError>true</TreatWarningAsError>
|
||||
<EnablePREfast>true</EnablePREfast>
|
||||
<AdditionalOptions>/analyze:stacksize19000 %(AdditionalOptions)</AdditionalOptions>
|
||||
<TreatWarningAsError>false</TreatWarningAsError>
|
||||
<EnablePREfast>false</EnablePREfast>
|
||||
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
|
||||
@ -18,9 +18,6 @@
|
||||
<ClCompile Include="..\..\..\lib\fse.c">
|
||||
<Filter>Fichiers sources</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\zstd.c">
|
||||
<Filter>Fichiers sources</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\programs\bench.c">
|
||||
<Filter>Fichiers sources</Filter>
|
||||
</ClCompile>
|
||||
@ -39,15 +36,21 @@
|
||||
<ClCompile Include="..\..\..\lib\huff0.c">
|
||||
<Filter>Fichiers sources</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\zstdhc.c">
|
||||
<Filter>Fichiers sources</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\legacy\zstd_v02.c">
|
||||
<Filter>Fichiers sources</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\programs\legacy\fileio_legacy.c">
|
||||
<Filter>Fichiers sources</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\zstd_compress.c">
|
||||
<Filter>Fichiers sources</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\zstd_decompress.c">
|
||||
<Filter>Fichiers sources</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\lib\zstd_buffered.c">
|
||||
<Filter>Fichiers sources</Filter>
|
||||
</ClCompile>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="..\..\..\lib\fse.h">
|
||||
@ -83,12 +86,6 @@
|
||||
<ClInclude Include="..\..\..\lib\zstd_internal.h">
|
||||
<Filter>Fichiers d%27en-tête</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\zstdhc.h">
|
||||
<Filter>Fichiers d%27en-tête</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\zstdhc_static.h">
|
||||
<Filter>Fichiers d%27en-tête</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h">
|
||||
<Filter>Fichiers d%27en-tête</Filter>
|
||||
</ClInclude>
|
||||
@ -98,5 +95,11 @@
|
||||
<ClInclude Include="..\..\..\programs\legacy\fileio_legacy.h">
|
||||
<Filter>Fichiers d%27en-tête</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\zstd_buffered.h">
|
||||
<Filter>Fichiers d%27en-tête</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\lib\zstd_buffered_static.h">
|
||||
<Filter>Fichiers d%27en-tête</Filter>
|
||||
</ClInclude>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
Loading…
Reference in New Issue
Block a user