AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/Cyan4973/dev' into dev

Browse Source
This commit is contained in:
inikep 2016-04-11 16:49:27 +02:00
commit 8844e21dff
22 changed files with 5366 additions and 398 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/Makefile
View File

@ -52,7 +52,7 @@ LIBDIR ?= $(PREFIX)/lib
INCLUDEDIR=$(PREFIX)/include
ZSTD_FILES := zstd_compress.c zstd_decompress.c fse.c huff0.c zdict.c divsufsort.c
ZSTD_LEGACY:= legacy/zstd_v01.c legacy/zstd_v02.c legacy/zstd_v03.c legacy/zstd_v04.c
ZSTD_LEGACY:= legacy/zstd_v01.c legacy/zstd_v02.c legacy/zstd_v03.c legacy/zstd_v04.c legacy/zstd_v05.c
ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0

19
lib/legacy/zstd_legacy.h
View File

@ -46,6 +46,7 @@ extern "C" {
#include "zstd_v02.h"
#include "zstd_v03.h"
#include "zstd_v04.h"
#include "zstd_v05.h"
MEM_STATIC unsigned ZSTD_isLegacy (U32 magicNumberLE)
{
@ -53,28 +54,32 @@ MEM_STATIC unsigned ZSTD_isLegacy (U32 magicNumberLE)
{
case ZSTDv01_magicNumberLE :
case ZSTDv02_magicNumber :
case ZSTDv03_magicNumber :
case ZSTDv04_magicNumber : return 1;
case ZSTDv03_magicNumber :
case ZSTDv04_magicNumber :
case ZSTDv05_MAGICNUMBER :
return 1;
default : return 0;
}
}
MEM_STATIC size_t ZSTD_decompressLegacy(
void* dst, size_t maxOriginalSize,
void* dst, size_t dstCapacity,
const void* src, size_t compressedSize,
U32 magicNumberLE)
{
switch(magicNumberLE)
{
case ZSTDv01_magicNumberLE :
return ZSTDv01_decompress(dst, maxOriginalSize, src, compressedSize);
return ZSTDv01_decompress(dst, dstCapacity, src, compressedSize);
case ZSTDv02_magicNumber :
return ZSTDv02_decompress(dst, maxOriginalSize, src, compressedSize);
return ZSTDv02_decompress(dst, dstCapacity, src, compressedSize);
case ZSTDv03_magicNumber :
return ZSTDv03_decompress(dst, maxOriginalSize, src, compressedSize);
return ZSTDv03_decompress(dst, dstCapacity, src, compressedSize);
case ZSTDv04_magicNumber :
return ZSTDv04_decompress(dst, maxOriginalSize, src, compressedSize);
return ZSTDv04_decompress(dst, dstCapacity, src, compressedSize);
case ZSTDv05_MAGICNUMBER :
return ZSTDv05_decompress(dst, dstCapacity, src, compressedSize);
default :
return ERROR(prefix_unknown);
}

2
lib/legacy/zstd_v04.h
View File

@ -95,7 +95,7 @@ size_t ZSTDv04_decompressContinue(ZSTDv04_Dctx* dctx, void* dst, size_t maxDstSi
***************************************/
typedef struct ZBUFFv04_DCtx_s ZBUFFv04_DCtx;
ZBUFFv04_DCtx* ZBUFFv04_createDCtx(void);
size_t ZBUFFv04_freeDCtx(ZBUFFv04_DCtx* dctx);
size_t ZBUFFv04_freeDCtx(ZBUFFv04_DCtx* dctx);
size_t ZBUFFv04_decompressInit(ZBUFFv04_DCtx* dctx);
size_t ZBUFFv04_decompressWithDictionary(ZBUFFv04_DCtx* dctx, const void* dict, size_t dictSize);

4726
lib/legacy/zstd_v05.c Normal file
View File

File diff suppressed because it is too large Load Diff

156
lib/legacy/zstd_v05.h Normal file
View File

@ -0,0 +1,156 @@
/*
zstd_v05 - decoder for 0.5 format
Header File
Copyright (C) 2014-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- zstd source repository : https://github.com/Cyan4973/zstd
*/
#ifndef ZSTDv05_H
#define ZSTDv05_H
#if defined (__cplusplus)
extern "C" {
#endif
/*-*************************************
* Dependencies
***************************************/
#include <stddef.h> /* size_t */
/* *************************************
* Simple functions
***************************************/
/*! ZSTDv05_decompress() :
`compressedSize` : is the _exact_ size of the compressed blob, otherwise decompression will fail.
`dstCapacity` must be large enough, equal or larger than originalSize.
@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
or an errorCode if it fails (which can be tested using ZSTDv05_isError()) */
size_t ZSTDv05_decompress( void* dst, size_t dstCapacity,
const void* src, size_t compressedSize);
/* *************************************
* Helper functions
***************************************/
/* Error Management */
unsigned ZSTDv05_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
const char* ZSTDv05_getErrorName(size_t code); /*!< provides readable string for an error code */
/* *************************************
* Explicit memory management
***************************************/
/** Decompression context */
typedef struct ZSTDv05_DCtx_s ZSTDv05_DCtx;
ZSTDv05_DCtx* ZSTDv05_createDCtx(void);
size_t ZSTDv05_freeDCtx(ZSTDv05_DCtx* dctx); /*!< @return : errorCode */
/** ZSTDv05_decompressDCtx() :
* Same as ZSTDv05_decompress(), but requires an already allocated ZSTDv05_DCtx (see ZSTDv05_createDCtx()) */
size_t ZSTDv05_decompressDCtx(ZSTDv05_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/*-***********************
* Dictionary API
*************************/
/*! ZSTDv05_decompress_usingDict() :
* Decompression using a pre-defined Dictionary content (see dictBuilder).
* Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted.
* Note : dict can be NULL, in which case, it's equivalent to ZSTDv05_decompressDCtx() */
size_t ZSTDv05_decompress_usingDict(ZSTDv05_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const void* dict,size_t dictSize);
typedef struct ZBUFFv05_DCtx_s ZBUFFv05_DCtx;
ZBUFFv05_DCtx* ZBUFFv05_createDCtx(void);
size_t ZBUFFv05_freeDCtx(ZBUFFv05_DCtx* dctx);
size_t ZBUFFv05_decompressInit(ZBUFFv05_DCtx* dctx);
size_t ZBUFFv05_decompressInitDictionary(ZBUFFv05_DCtx* dctx, const void* dict, size_t dictSize);
size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* dctx,
void* dst, size_t* dstCapacityPtr,
const void* src, size_t* srcSizePtr);
/*-***************************************************************************
* Streaming decompression
*
* A ZBUFFv05_DCtx object is required to track streaming operations.
* Use ZBUFFv05_createDCtx() and ZBUFFv05_freeDCtx() to create/release resources.
* Use ZBUFFv05_decompressInit() to start a new decompression operation,
* or ZBUFFv05_decompressInitDictionary() if decompression requires a dictionary.
* Note that ZBUFFv05_DCtx objects can be reused multiple times.
*
* Use ZBUFFv05_decompressContinue() repetitively to consume your input.
* *srcSizePtr and *dstCapacityPtr can be any size.
* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change @dst.
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency)
* or 0 when a frame is completely decoded
* or an error code, which can be tested using ZBUFFv05_isError().
*
* Hint : recommended buffer sizes (not compulsory) : ZBUFFv05_recommendedDInSize() / ZBUFFv05_recommendedDOutSize()
* output : ZBUFFv05_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
* input : ZBUFFv05_recommendedDInSize==128Kb+3; just follow indications from ZBUFFv05_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
* *******************************************************************************/
/* *************************************
* Tool functions
***************************************/
unsigned ZBUFFv05_isError(size_t errorCode);
const char* ZBUFFv05_getErrorName(size_t errorCode);
/** Functions below provide recommended buffer sizes for Compression or Decompression operations.
* These sizes are just hints, and tend to offer better latency */
size_t ZBUFFv05_recommendedDInSize(void);
size_t ZBUFFv05_recommendedDOutSize(void);
/*-*************************************
* Constants
***************************************/
#define ZSTDv05_MAGICNUMBER 0xFD2FB525 /* v0.5 */
#if defined (__cplusplus)
}
#endif
#endif /* ZSTDv0505_H */

130
lib/zbuff.c
View File

@ -319,7 +319,7 @@ typedef enum { ZBUFFds_init, ZBUFFds_readHeader,
/* *** Resource management *** */
struct ZBUFF_DCtx_s {
ZSTD_DCtx* zc;
ZSTD_DCtx* zd;
ZSTD_frameParams fParams;
size_t blockSize;
char* inBuff;
@ -335,63 +335,63 @@ struct ZBUFF_DCtx_s {
ZBUFF_DCtx* ZBUFF_createDCtx(void)
{
ZBUFF_DCtx* zbc = (ZBUFF_DCtx*)malloc(sizeof(ZBUFF_DCtx));
if (zbc==NULL) return NULL;
memset(zbc, 0, sizeof(*zbc));
zbc->zc = ZSTD_createDCtx();
zbc->stage = ZBUFFds_init;
return zbc;
ZBUFF_DCtx* zbd = (ZBUFF_DCtx*)malloc(sizeof(ZBUFF_DCtx));
if (zbd==NULL) return NULL;
memset(zbd, 0, sizeof(*zbd));
zbd->zd = ZSTD_createDCtx();
zbd->stage = ZBUFFds_init;
return zbd;
}
size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbc)
size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd)
{
if (zbc==NULL) return 0; /* support free on null */
ZSTD_freeDCtx(zbc->zc);
free(zbc->inBuff);
free(zbc->outBuff);
free(zbc);
if (zbd==NULL) return 0; /* support free on null */
ZSTD_freeDCtx(zbd->zd);
free(zbd->inBuff);
free(zbd->outBuff);
free(zbd);
return 0;
}
/* *** Initialization *** */
size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbc, const void* dict, size_t dictSize)
size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize)
{
zbc->stage = ZBUFFds_readHeader;
zbc->inPos = zbc->outStart = zbc->outEnd = 0;
return ZSTD_decompressBegin_usingDict(zbc->zc, dict, dictSize);
zbd->stage = ZBUFFds_readHeader;
zbd->inPos = zbd->outStart = zbd->outEnd = 0;
return ZSTD_decompressBegin_usingDict(zbd->zd, dict, dictSize);
}
size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbc)
size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbd)
{
return ZBUFF_decompressInitDictionary(zbc, NULL, 0);
return ZBUFF_decompressInitDictionary(zbd, NULL, 0);
}
/* *** Decompression *** */
size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
void* dst, size_t* dstCapacityPtr,
const void* src, size_t* srcSizePtr)
{
const char* const istart = (const char*)src;
const char* ip = istart;
const char* const iend = istart + *srcSizePtr;
const char* ip = istart;
char* const ostart = (char*)dst;
char* op = ostart;
char* const oend = ostart + *dstCapacityPtr;
char* op = ostart;
U32 notDone = 1;
while (notDone) {
switch(zbc->stage)
switch(zbd->stage)
{
case ZBUFFds_init :
return ERROR(init_missing);
case ZBUFFds_readHeader :
/* read header from src */
{ size_t const headerSize = ZSTD_getFrameParams(&(zbc->fParams), src, *srcSizePtr);
{ size_t const headerSize = ZSTD_getFrameParams(&(zbd->fParams), src, *srcSizePtr);
if (ZSTD_isError(headerSize)) return headerSize;
if (headerSize) {
/* not enough input to decode header : needs headerSize > *srcSizePtr */
@ -401,76 +401,76 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
} }
/* Frame header instruct buffer sizes */
{ size_t const blockSize = MIN(1 << zbc->fParams.windowLog, ZSTD_BLOCKSIZE_MAX);
zbc->blockSize = blockSize;
if (zbc->inBuffSize < blockSize) {
free(zbc->inBuff);
zbc->inBuffSize = blockSize;
zbc->inBuff = (char*)malloc(blockSize);
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
{ size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTD_BLOCKSIZE_MAX);
zbd->blockSize = blockSize;
if (zbd->inBuffSize < blockSize) {
free(zbd->inBuff);
zbd->inBuffSize = blockSize;
zbd->inBuff = (char*)malloc(blockSize);
if (zbd->inBuff == NULL) return ERROR(memory_allocation);
}
{ size_t const neededOutSize = ((size_t)1 << zbc->fParams.windowLog) + blockSize;
if (zbc->outBuffSize < neededOutSize) {
free(zbc->outBuff);
zbc->outBuffSize = neededOutSize;
zbc->outBuff = (char*)malloc(neededOutSize);
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
{ size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize;
if (zbd->outBuffSize < neededOutSize) {
free(zbd->outBuff);
zbd->outBuffSize = neededOutSize;
zbd->outBuff = (char*)malloc(neededOutSize);
if (zbd->outBuff == NULL) return ERROR(memory_allocation);
} } }
zbc->stage = ZBUFFds_read;
zbd->stage = ZBUFFds_read;
case ZBUFFds_read:
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd);
if (neededInSize==0) { /* end of frame */
zbc->stage = ZBUFFds_init;
zbd->stage = ZBUFFds_init;
notDone = 0;
break;
}
if ((size_t)(iend-ip) >= neededInSize) {
/* directly decode from src */
size_t const decodedSize = ZSTD_decompressContinue(zbc->zc,
zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart,
size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
ip, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize;
ip += neededInSize;
if (!decodedSize) break; /* this was just a header */
zbc->outEnd = zbc->outStart + decodedSize;
zbc->stage = ZBUFFds_flush;
zbd->outEnd = zbd->outStart + decodedSize;
zbd->stage = ZBUFFds_flush;
break;
}
if (ip==iend) { notDone = 0; break; } /* no more input */
zbc->stage = ZBUFFds_load;
zbd->stage = ZBUFFds_load;
}
case ZBUFFds_load:
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
size_t const toLoad = neededInSize - zbc->inPos; /* should always be <= remaining space within inBuff */
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd);
size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */
size_t loadedSize;
if (toLoad > zbc->inBuffSize - zbc->inPos) return ERROR(corruption_detected); /* should never happen */
loadedSize = ZBUFF_limitCopy(zbc->inBuff + zbc->inPos, toLoad, ip, iend-ip);
if (toLoad > zbd->inBuffSize - zbd->inPos) return ERROR(corruption_detected); /* should never happen */
loadedSize = ZBUFF_limitCopy(zbd->inBuff + zbd->inPos, toLoad, ip, iend-ip);
ip += loadedSize;
zbc->inPos += loadedSize;
zbd->inPos += loadedSize;
if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */
/* decode loaded input */
{ size_t const decodedSize = ZSTD_decompressContinue(zbc->zc,
zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart,
zbc->inBuff, neededInSize);
{ size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
zbd->inBuff, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize;
zbc->inPos = 0; /* input is consumed */
if (!decodedSize) { zbc->stage = ZBUFFds_read; break; } /* this was just a header */
zbc->outEnd = zbc->outStart + decodedSize;
zbc->stage = ZBUFFds_flush;
zbd->inPos = 0; /* input is consumed */
if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
zbd->outEnd = zbd->outStart + decodedSize;
zbd->stage = ZBUFFds_flush;
// break; /* ZBUFFds_flush follows */
} }
case ZBUFFds_flush:
{ size_t const toFlushSize = zbc->outEnd - zbc->outStart;
size_t const flushedSize = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outStart, toFlushSize);
{ size_t const toFlushSize = zbd->outEnd - zbd->outStart;
size_t const flushedSize = ZBUFF_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize);
op += flushedSize;
zbc->outStart += flushedSize;
zbd->outStart += flushedSize;
if (flushedSize == toFlushSize) {
zbc->stage = ZBUFFds_read;
if (zbc->outStart + zbc->blockSize > zbc->outBuffSize)
zbc->outStart = zbc->outEnd = 0;
zbd->stage = ZBUFFds_read;
if (zbd->outStart + zbd->blockSize > zbd->outBuffSize)
zbd->outStart = zbd->outEnd = 0;
break;
}
/* cannot flush everything */
@ -483,9 +483,9 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
/* result */
*srcSizePtr = ip-istart;
*dstCapacityPtr = op-ostart;
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbc->zc);
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbd->zd);
if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */
nextSrcSizeHint -= zbc->inPos; /* already loaded*/
nextSrcSizeHint -= zbd->inPos; /* already loaded*/
return nextSrcSizeHint;
}
}

67
lib/zstd_compress.c
View File

@ -818,7 +818,7 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 24 */ /* 24 */
if (MEM_32bits()) BIT_flushBits(&blockStream); /* (7)*/
FSE_encodeSymbol(&blockStream, &stateLitLength, llCode); /* 16 */ /* 33 */
if (MEM_32bits() || (ofBits+mlBits+llBits > 64-7-(LLFSELog+MLFSELog+OffFSELog)))
if (MEM_32bits() || (ofBits+mlBits+llBits >= 64-7-(LLFSELog+MLFSELog+OffFSELog)))
BIT_flushBits(&blockStream); /* (7)*/
BIT_addBits(&blockStream, llTable[n], llBits);
if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream);
@ -858,7 +858,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B
static const BYTE* g_start = NULL;
const U32 pos = (U32)(literals - g_start);
if (g_start==NULL) g_start = literals;
if ((pos > 200000000) && (pos < 200900000))
if ((pos > 5810300) && (pos < 5810500))
printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n",
pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
#endif
@ -1291,7 +1291,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
while (nbCompares-- && (matchIndex > windowLow)) {
U32* nextPtr = bt + 2*(matchIndex & btMask);
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
#if 1 /* note : can create issues when hlog small <= 11 */
#if 0 /* note : can create issues when hlog small <= 11 */
const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */
if (matchIndex == predictedSmall) {
/* no need to check length, result known */
@ -1645,8 +1645,8 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
const BYTE* const ilimit = iend - 8;
const BYTE* const base = ctx->base + ctx->dictLimit;
const U32 maxSearches = 1 << ctx->params.cParams.searchLog;
const U32 mls = ctx->params.cParams.searchLength;
U32 const maxSearches = 1 << ctx->params.cParams.searchLog;
U32 const mls = ctx->params.cParams.searchLength;
typedef size_t (*searchMax_f)(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iLimit,
size_t* offsetPtr,
@ -1655,8 +1655,7 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
/* init */
U32 rep[ZSTD_REP_INIT];
for (U32 i=0; i<ZSTD_REP_INIT; i++)
rep[i]=REPCODE_STARTVALUE;
{ U32 i ; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=REPCODE_STARTVALUE; }
ctx->nextToUpdate3 = ctx->nextToUpdate;
ZSTD_resetSeqStore(seqStorePtr);
@ -1818,8 +1817,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
/* init */
U32 rep[ZSTD_REP_INIT];
for (U32 i=0; i<ZSTD_REP_INIT; i++)
rep[i]=REPCODE_STARTVALUE;
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=REPCODE_STARTVALUE; }
ctx->nextToUpdate3 = ctx->nextToUpdate;
ZSTD_resetSeqStore(seqStorePtr);
@ -2366,6 +2364,7 @@ size_t ZSTD_compress_usingPreparedCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* prepare
}
{ size_t const cSize = ZSTD_compressContinue(cctx, dst, dstCapacity, src, srcSize);
if (ZSTD_isError(cSize)) return cSize;
{ size_t const endSize = ZSTD_compressEnd(cctx, (char*)dst+cSize, dstCapacity-cSize);
if (ZSTD_isError(endSize)) return endSize;
return cSize + endSize;
@ -2445,7 +2444,7 @@ unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = {
{ /* "default" */
/* W, C, H, S, L, SL, strat */
/* W, C, H, S, L, TL, strat */
{ 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 - never used */
{ 19, 13, 14, 1, 7, 4, ZSTD_fast }, /* level 1 */
{ 19, 15, 16, 1, 6, 4, ZSTD_fast }, /* level 2 */
@ -2463,38 +2462,38 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 22, 21, 22, 6, 5, 4, ZSTD_lazy2 }, /* level 14 */
{ 22, 21, 21, 5, 5, 4, ZSTD_btlazy2 }, /* level 15 */
{ 23, 22, 22, 5, 5, 4, ZSTD_btlazy2 }, /* level 16 */
{ 23, 22, 22, 6, 5, 22, ZSTD_btopt }, /* level 17 */
{ 22, 22, 22, 5, 3, 44, ZSTD_btopt }, /* level 18 */
{ 23, 24, 22, 7, 3, 44, ZSTD_btopt }, /* level 19 */
{ 25, 26, 22, 7, 3, 71, ZSTD_btopt }, /* level 20 */
{ 26, 26, 24, 7, 3,256, ZSTD_btopt }, /* level 21 */
{ 27, 28, 26, 9, 3,256, ZSTD_btopt }, /* level 22 */
{ 23, 23, 22, 5, 5, 4, ZSTD_btlazy2 }, /* level 17.*/
{ 23, 23, 22, 6, 5, 24, ZSTD_btopt }, /* level 18.*/
{ 23, 23, 22, 6, 3, 48, ZSTD_btopt }, /* level 19.*/
{ 25, 26, 23, 7, 3, 64, ZSTD_btopt }, /* level 20.*/
{ 26, 26, 23, 7, 3,256, ZSTD_btopt }, /* level 21.*/
{ 27, 27, 25, 9, 3,512, ZSTD_btopt }, /* level 22.*/
},
{ /* for srcSize <= 256 KB */
/* W, C, H, S, L, T, strat */
{ 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 */
{ 18, 14, 15, 1, 6, 4, ZSTD_fast }, /* level 1 */
{ 18, 14, 16, 1, 5, 4, ZSTD_fast }, /* level 2 */
{ 18, 14, 17, 1, 5, 4, ZSTD_fast }, /* level 3.*/
{ 18, 14, 15, 4, 4, 4, ZSTD_greedy }, /* level 4 */
{ 18, 16, 17, 4, 4, 4, ZSTD_greedy }, /* level 5 */
{ 18, 17, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */
{ 18, 13, 14, 1, 6, 4, ZSTD_fast }, /* level 1 */
{ 18, 15, 17, 1, 5, 4, ZSTD_fast }, /* level 2 */
{ 18, 13, 15, 1, 5, 4, ZSTD_greedy }, /* level 3.*/
{ 18, 15, 17, 1, 5, 4, ZSTD_greedy }, /* level 4.*/
{ 18, 16, 17, 4, 5, 4, ZSTD_greedy }, /* level 5 */
{ 18, 17, 17, 5, 5, 4, ZSTD_greedy }, /* level 6 */
{ 18, 17, 17, 4, 4, 4, ZSTD_lazy }, /* level 7 */
{ 18, 17, 17, 4, 4, 4, ZSTD_lazy2 }, /* level 8 */
{ 18, 17, 17, 5, 4, 4, ZSTD_lazy2 }, /* level 9 */
{ 18, 17, 17, 6, 4, 4, ZSTD_lazy2 }, /* level 10 */
{ 18, 17, 17, 7, 4, 4, ZSTD_lazy2 }, /* level 11 */
{ 18, 18, 17, 4, 4, 4, ZSTD_btlazy2 }, /* level 12 */
{ 18, 19, 17, 7, 4, 4, ZSTD_btlazy2 }, /* level 13.*/
{ 18, 17, 19, 8, 4, 24, ZSTD_btopt }, /* level 14.*/
{ 18, 19, 19, 8, 4, 48, ZSTD_btopt }, /* level 15.*/
{ 18, 19, 18, 9, 4,128, ZSTD_btopt }, /* level 16.*/
{ 18, 19, 18, 9, 4,192, ZSTD_btopt }, /* level 17.*/
{ 18, 19, 18, 9, 4,256, ZSTD_btopt }, /* level 18.*/
{ 18, 19, 18, 10, 4,256, ZSTD_btopt }, /* level 19.*/
{ 18, 19, 18, 11, 4,256, ZSTD_btopt }, /* level 20.*/
{ 18, 19, 18, 12, 4,256, ZSTD_btopt }, /* level 21.*/
{ 18, 19, 18, 12, 4,256, ZSTD_btopt }, /* level 22*/
{ 18, 18, 17, 6, 4, 4, ZSTD_lazy2 }, /* level 11.*/
{ 18, 18, 17, 7, 4, 4, ZSTD_lazy2 }, /* level 12.*/
{ 18, 19, 17, 7, 4, 4, ZSTD_btlazy2 }, /* level 13 */
{ 18, 18, 18, 4, 4, 16, ZSTD_btopt }, /* level 14.*/
{ 18, 18, 18, 8, 4, 24, ZSTD_btopt }, /* level 15.*/
{ 18, 19, 18, 8, 3, 48, ZSTD_btopt }, /* level 16.*/
{ 18, 19, 18, 8, 3, 96, ZSTD_btopt }, /* level 17.*/
{ 18, 19, 18, 9, 3,128, ZSTD_btopt }, /* level 18.*/
{ 18, 19, 18, 10, 3,256, ZSTD_btopt }, /* level 19.*/
{ 18, 19, 18, 11, 3,512, ZSTD_btopt }, /* level 20.*/
{ 18, 19, 18, 12, 3,512, ZSTD_btopt }, /* level 21.*/
{ 18, 19, 18, 13, 3,512, ZSTD_btopt }, /* level 22.*/
},
{ /* for srcSize <= 128 KB */
/* W, C, H, S, L, T, strat */

93
lib/zstd_decompress.c
View File

@ -128,7 +128,7 @@ struct ZSTD_DCtx_s
ZSTD_frameParams fParams;
blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
ZSTD_dStage stage;
U32 flagStaticTables;
U32 flagRepeatTable;
const BYTE* litPtr;
size_t litBufSize;
size_t litSize;
@ -136,7 +136,7 @@ struct ZSTD_DCtx_s
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
}; /* typedef'd to ZSTD_DCtx within "zstd_static.h" */
size_t sizeofDCtx (void) { return sizeof(ZSTD_DCtx); }
size_t ZSTD_sizeofDCtx (void) { return sizeof(ZSTD_DCtx); } /* non published interface */
size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
{
@ -147,7 +147,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
dctx->vBase = NULL;
dctx->dictEnd = NULL;
dctx->hufTableX4[0] = HufLog;
dctx->flagStaticTables = 0;
dctx->flagRepeatTable = 0;
return 0;
}
@ -374,9 +374,9 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
switch(istart[0]>> 6)
{
case IS_HUF:
{
size_t litSize, litCSize, singleStream=0;
{ size_t litSize, litCSize, singleStream=0;
U32 lhSize = ((istart[0]) >> 4) & 3;
if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for lhSize, + cSize (+nbSeq) */
switch(lhSize)
{
case 0: case 1: default: /* note : default is impossible, since lhSize into [0..3] */
@ -400,6 +400,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
break;
}
if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected);
if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
if (HUF_isError(singleStream ?
HUF_decompress1X2(dctx->litBuffer, litSize, istart+lhSize, litCSize) :
@ -412,13 +413,11 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
return litCSize + lhSize;
}
case IS_PCH:
{
size_t errorCode;
size_t litSize, litCSize;
{ size_t litSize, litCSize;
U32 lhSize = ((istart[0]) >> 4) & 3;
if (lhSize != 1) /* only case supported for now : small litSize, single stream */
return ERROR(corruption_detected);
if (!dctx->flagStaticTables)
if (!dctx->flagRepeatTable)
return ERROR(dictionary_corrupted);
/* 2 - 2 - 10 - 10 */
@ -426,17 +425,16 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2);
litCSize = ((istart[1] & 3) << 8) + istart[2];
errorCode = HUF_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4);
if (HUF_isError(errorCode)) return ERROR(corruption_detected);
{ size_t const errorCode = HUF_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4);
if (HUF_isError(errorCode)) return ERROR(corruption_detected);
}
dctx->litPtr = dctx->litBuffer;
dctx->litBufSize = ZSTD_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize;
return litCSize + lhSize;
}
case IS_RAW:
{
size_t litSize;
{ size_t litSize;
U32 lhSize = ((istart[0]) >> 4) & 3;
switch(lhSize)
{
@ -467,8 +465,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
return lhSize+litSize;
}
case IS_RLE:
{
size_t litSize;
{ size_t litSize;
U32 lhSize = ((istart[0]) >> 4) & 3;
switch(lhSize)
{
@ -481,6 +478,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
break;
case 3:
litSize = ((istart[0] & 15) << 16) + (istart[1] << 8) + istart[2];
if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */
break;
}
if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected);
@ -502,7 +500,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
*/
FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, U32 type, U32 max, U32 maxLog,
const void* src, size_t srcSize,
const S16* defaultNorm, U32 defaultLog)
const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable)
{
switch(type)
{
@ -515,6 +513,7 @@ FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, U32 type, U32 max, U3
FSE_buildDTable(DTable, defaultNorm, max, defaultLog);
return 0;
case FSE_ENCODING_STATIC:
if (!flagRepeatTable) return ERROR(corruption_detected);
return 0;
default : /* impossible */
case FSE_ENCODING_DYNAMIC :
@ -530,7 +529,7 @@ FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, U32 type, U32 max, U3
size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr,
FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb,
FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, U32 flagRepeatTable,
const void* src, size_t srcSize)
{
const BYTE* const istart = (const BYTE* const)src;
@ -562,15 +561,15 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr,
if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
/* Build DTables */
{ size_t const bhSize = ZSTD_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog);
{ size_t const bhSize = ZSTD_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
if (ZSTD_isError(bhSize)) return ERROR(corruption_detected);
ip += bhSize;
}
{ size_t const bhSize = ZSTD_buildSeqTable(DTableOffb, Offtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog);
{ size_t const bhSize = ZSTD_buildSeqTable(DTableOffb, Offtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, flagRepeatTable);
if (ZSTD_isError(bhSize)) return ERROR(corruption_detected);
ip += bhSize;
}
{ size_t const bhSize = ZSTD_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog);
{ size_t const bhSize = ZSTD_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, flagRepeatTable);
if (ZSTD_isError(bhSize)) return ERROR(corruption_detected);
ip += bhSize;
} }
@ -750,25 +749,24 @@ static size_t ZSTD_decompressSequences(
const BYTE* ip = (const BYTE*)seqStart;
const BYTE* const iend = ip + seqSize;
BYTE* const ostart = (BYTE* const)dst;
BYTE* op = ostart;
BYTE* const oend = ostart + maxDstSize;
BYTE* op = ostart;
const BYTE* litPtr = dctx->litPtr;
const BYTE* const litLimit_8 = litPtr + dctx->litBufSize - 8;
const BYTE* const litEnd = litPtr + dctx->litSize;
U32* DTableLL = dctx->LLTable;
U32* DTableML = dctx->MLTable;
U32* DTableOffb = dctx->OffTable;
FSE_DTable* DTableLL = dctx->LLTable;
FSE_DTable* DTableML = dctx->MLTable;
FSE_DTable* DTableOffb = dctx->OffTable;
const BYTE* const base = (const BYTE*) (dctx->base);
const BYTE* const vBase = (const BYTE*) (dctx->vBase);
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
int nbSeq;
/* Build Decoding Tables */
{ size_t const seqHSize = ZSTD_decodeSeqHeaders(&nbSeq,
DTableLL, DTableML, DTableOffb,
ip, seqSize);
{ size_t const seqHSize = ZSTD_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, dctx->flagRepeatTable, ip, seqSize);
if (ZSTD_isError(seqHSize)) return seqHSize;
ip += seqHSize;
dctx->flagRepeatTable = 0;
}
/* Regen sequences */
@ -778,8 +776,7 @@ static size_t ZSTD_decompressSequences(
memset(&sequence, 0, sizeof(sequence));
sequence.offset = REPCODE_STARTVALUE;
for (U32 i=0; i<ZSTD_REP_INIT; i++)
seqState.prevOffset[i] = REPCODE_STARTVALUE;
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) seqState.prevOffset[i] = REPCODE_STARTVALUE; }
{ size_t const errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip);
if (ERR_isError(errorCode)) return ERROR(corruption_detected); }
FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL);
@ -787,20 +784,22 @@ static size_t ZSTD_decompressSequences(
FSE_initDState(&(seqState.stateML), &(seqState.DStream), DTableML);
for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) {
size_t oneSeqSize;
nbSeq--;
ZSTD_decodeSequence(&sequence, &seqState);
#if 0 /* for debug */
{ U32 pos = (U32)(op-base);
// if ((pos > 200802300) && (pos < 200802400))
printf("Dpos %6u :%5u literals & match %3u bytes at distance %6u \n",
pos, (U32)sequence.litLength, (U32)sequence.matchLength, (U32)sequence.offset);
}
#if 0 /* debug */
static BYTE* start = NULL;
if (start==NULL) start = op;
size_t pos = (size_t)(op-start);
if ((pos >= 5810037) && (pos < 5810400))
printf("Dpos %6u :%5u literals & match %3u bytes at distance %6u \n",
pos, (U32)sequence.litLength, (U32)sequence.matchLength, (U32)sequence.offset);
#endif
oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
}
{ size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litLimit_8, base, vBase, dictEnd);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
} }
/* check if reached exact end */
if (nbSeq) return ERROR(corruption_detected);
@ -838,11 +837,11 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong);
/* Decode literals sub-block */
{ size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
if (ZSTD_isError(litCSize)) return litCSize;
ip += litCSize;
srcSize -= litCSize; }
{ size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
if (ZSTD_isError(litCSize)) return litCSize;
ip += litCSize;
srcSize -= litCSize;
}
return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize);
}
@ -1099,7 +1098,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSiz
errorCode = FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted);
dctx->flagStaticTables = 1;
dctx->flagRepeatTable = 1;
return hSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize;
}

30
lib/zstd_opt.h
View File

@ -250,7 +250,7 @@ U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* zc, const BYTE* ip)
U32 idx = zc->nextToUpdate3;
const U32 target = zc->nextToUpdate3 = (U32)(ip - base);
const size_t hash3 = ZSTD_hash3Ptr(ip, hashLog3);
while(idx < target) {
hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx;
idx++;
@ -472,8 +472,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
/* init */
U32 rep[ZSTD_REP_INIT];
for (U32 i=0; i<ZSTD_REP_INIT; i++)
rep[i]=REPCODE_STARTVALUE;
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=REPCODE_STARTVALUE; }
ctx->nextToUpdate3 = ctx->nextToUpdate;
ZSTD_resetSeqStore(seqStorePtr);
@ -499,7 +498,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
opt[0].litlen = (U32)(ip - litstart);
/* check repCode */
for (U32 i=0; i<ZSTD_REP_NUM; i++)
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++)
if (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - rep[i], minMatch)) {
/* repcode : we take it */
mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-rep[i], iend) + minMatch;
@ -518,15 +517,14 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
mlen--;
} while (mlen >= minMatch);
}
} }
match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches); /* first search (depth 0) */
ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos);
if (!last_pos && !match_num) { ip++; continue; }
for (U32 i=0; i<ZSTD_REP_INIT; i++)
opt[0].rep[i] = rep[i];
{ U32 i ; for (i=0; i<ZSTD_REP_INIT; i++) opt[0].rep[i] = rep[i]; }
opt[0].mlen = 1;
if (match_num && matches[match_num-1].len > sufficient_len) {
@ -593,7 +591,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
ZSTD_LOG_PARSER("%d: CURRENT_NoExt price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]);
best_mlen = 0;
for (U32 i=0; i<ZSTD_REP_NUM; i++)
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++)
if (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - opt[cur].rep[i], minMatch)) { /* check rep */
mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - opt[cur].rep[i], iend) + minMatch;
ZSTD_LOG_PARSER("%d: Found REP %d/%d mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), i, ZSTD_REP_NUM, mlen, i, opt[cur].rep[i], cur, opt[cur].off);
@ -624,7 +622,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
SET_PRICE(cur + mlen, mlen, i, litlen, price);
mlen--;
} while (mlen >= minMatch);
}
} }
match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches);
@ -776,8 +774,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
/* init */
U32 rep[ZSTD_REP_INIT];
for (U32 i=0; i<ZSTD_REP_INIT; i++)
rep[i]=REPCODE_STARTVALUE;
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=REPCODE_STARTVALUE; }
ctx->nextToUpdate3 = ctx->nextToUpdate;
ZSTD_resetSeqStore(seqStorePtr);
@ -800,7 +797,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
opt[0].litlen = (U32)(ip - litstart);
/* check repCode */
for (U32 i=0; i<ZSTD_REP_NUM; i++) {
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) {
const U32 repIndex = (U32)(current - rep[i]);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
@ -824,15 +821,14 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
mlen--;
} while (mlen >= minMatch);
} }
} } }
match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches); /* first search (depth 0) */
ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos);
if (!last_pos && !match_num) { ip++; continue; }
for (U32 i=0; i<ZSTD_REP_INIT; i++)
opt[0].rep[i] = rep[i];
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) opt[0].rep[i] = rep[i]; }
opt[0].mlen = 1;
if (match_num && matches[match_num-1].len > sufficient_len) {
@ -902,7 +898,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
ZSTD_LOG_PARSER("%d: CURRENT_Ext price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]);
best_mlen = 0;
for (U32 i=0; i<ZSTD_REP_NUM; i++) {
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) {
const U32 repIndex = (U32)(current+cur - opt[cur].rep[i]);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
@ -939,7 +935,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
SET_PRICE(cur + mlen, mlen, i, litlen, price);
mlen--;
} while (mlen >= minMatch);
} }
} } }
match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches);
ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num);

3
programs/.gitignore vendored
View File

@ -38,6 +38,9 @@ dictionary
grillResults.txt
_*
# fuzzer
afl
# Misc files
*.bat
fileTests.sh

3
programs/Makefile
View File

@ -61,7 +61,8 @@ ZSTD_FILES_LEGACY:=
else
ZSTD_LEGACY_SUPPORT:=1
CPPFLAGS += -I../lib/legacy -I./legacy
ZSTD_FILES_LEGACY:= $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c $(ZSTDDIR)/legacy/zstd_v03.c $(ZSTDDIR)/legacy/zstd_v04.c legacy/fileio_legacy.c
ZSTD_FILES_LEGACY:= $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c $(ZSTDDIR)/legacy/zstd_v03.c \
$(ZSTDDIR)/legacy/zstd_v04.c $(ZSTDDIR)/legacy/zstd_v05.c legacy/fileio_legacy.c
endif

35
programs/bench.c
View File

@ -1,6 +1,6 @@
/*
bench.c - Demo module to benchmark open-source compression algorithms
Copyright (C) Yann Collet 2012-2015
bench.c - open-source compression benchmark module
Copyright (C) Yann Collet 2012-2016
GPL v2 License
@ -19,8 +19,8 @@
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- zstd homepage : http://www.zstd.net
- zstd source repository : https://github.com/Cyan4973/zstd
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/* **************************************
@ -44,13 +44,13 @@
/* *************************************
* Includes
***************************************/
#define _POSIX_C_SOURCE 199309L /* before <time.h> - needed for nanosleep() */
#define _POSIX_C_SOURCE 199309L /* before <time.h> - needed for nanosleep() */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* memset */
#include <stdio.h> /* fprintf, fopen, ftello64 */
#include <sys/types.h> /* stat64 */
#include <sys/stat.h> /* stat64 */
#include <time.h> /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */
#include <time.h> /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */
/* sleep : posix - windows - others */
#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)))
@ -91,7 +91,6 @@
#include "xxhash.h"
/* *************************************
* Compiler specifics
***************************************/
@ -99,8 +98,12 @@
# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
#endif
#ifdef _MSC_VER
#define snprintf sprintf_s
#if defined(_MSC_VER)
# define snprintf sprintf_s
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L))
/* part of <stdio.h> */
#else
extern int snprintf (char* s, size_t maxlen, const char* format, ...); /* not declared in <stdio.h> when C version < c99 */
#endif
@ -181,7 +184,7 @@ void BMK_SetBlockSize(size_t blockSize)
static U64 BMK_clockSpan( BMK_time_t clockStart, BMK_time_t ticksPerSecond )
{
BMK_time_t clockEnd;
(void)ticksPerSecond;
BMK_getTime(clockEnd);
return BMK_getSpanTimeMicro(ticksPerSecond, clockStart, clockEnd);
@ -224,7 +227,7 @@ typedef struct
double dSpeed;
} benchResult_t;
#define MIN(a,b) ((a)<(b) ? (a) : (b))
#define MAX(a,b) ((a)>(b) ? (a) : (b))
@ -400,7 +403,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
if (crcOrig == crcCheck) {
result->ratio = ratio;
result->cSize = cSize;
result->cSpeed = (double)srcSize / fastestC;
result->cSpeed = (double)srcSize / fastestC;
result->dSpeed = (double)srcSize / fastestD;
}
DISPLAYLEVEL(2, "%2i#\n", cLevel);
@ -457,7 +460,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
if (cLevelLast < cLevel) cLevelLast = cLevel;
for (l=cLevel; l <= cLevelLast; l++) {
for (l=cLevel; l <= cLevelLast; l++) {
BMK_benchMem(srcBuffer, benchedSize,
displayName, l,
fileSizes, nbFiles,
@ -471,15 +474,13 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
total.cSpeed += result.cSpeed;
total.dSpeed += result.dSpeed;
total.ratio += result.ratio;
}
}
if (g_displayLevel == 1 && cLevelLast > cLevel)
{
} }
if (g_displayLevel == 1 && cLevelLast > cLevel) {
total.cSize /= 1+cLevelLast-cLevel;
total.cSpeed /= 1+cLevelLast-cLevel;
total.dSpeed /= 1+cLevelLast-cLevel;
total.ratio /= 1+cLevelLast-cLevel;
DISPLAY("avg%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", (int)total.cSize, total.ratio, total.cSpeed, total.dSpeed, displayName);
DISPLAY("avg%11i (%5.3f) %6.1f MB/s %6.1f MB/s %s\n", (int)total.cSize, total.ratio, total.cSpeed, total.dSpeed, displayName);
}
}

35
programs/datagen.c
View File

@ -1,6 +1,6 @@
/*
datagen.c - compressible data generator test tool
Copyright (C) Yann Collet 2012-2015
Copyright (C) Yann Collet 2012-2016
GPL v2 License
@ -19,8 +19,8 @@
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- ZSTD source repository : https://github.com/Cyan4973/zstd
- Public forum : https://groups.google.com/forum/#!forum/lz4c
- zstd homepage : http://www.zstd.net/
- source repository : https://github.com/Cyan4973/zstd
*/
/*-************************************
@ -67,9 +67,6 @@
**************************************/
#define KB *(1 <<10)
#define PRIME1 2654435761U
#define PRIME2 2246822519U
/*-************************************
* Local types
@ -87,9 +84,11 @@ typedef BYTE litDistribTable[LTSIZE];
#define RDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
static unsigned int RDG_rand(U32* src)
{
static const U32 prime1 = 2654435761U;
static const U32 prime2 = 2246822519U;
U32 rand32 = *src;
rand32 *= PRIME1;
rand32 ^= PRIME2;
rand32 *= prime1;
rand32 ^= prime2;
rand32 = RDG_rotl32(rand32, 13);
*src = rand32;
return rand32;
@ -103,7 +102,7 @@ static void RDG_fillLiteralDistrib(litDistribTable lt, double ld)
BYTE firstChar = '(';
BYTE lastChar = '}';
if (ld==0.0) {
if (ld<=0.0) {
character = 0;
firstChar = 0;
lastChar =255;
@ -122,7 +121,7 @@ static void RDG_fillLiteralDistrib(litDistribTable lt, double ld)
static BYTE RDG_genChar(U32* seed, const litDistribTable lt)
{
U32 id = RDG_rand(seed) & LTMASK;
U32 const id = RDG_rand(seed) & LTMASK;
return (lt[id]);
}
@ -194,28 +193,28 @@ void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba
#define RDG_DICTSIZE (32 KB)
#define RDG_BLOCKSIZE (128 KB)
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed)
{
BYTE* buff = (BYTE*)malloc(RDG_DICTSIZE + RDG_BLOCKSIZE);
U64 total = 0;
size_t genBlockSize = RDG_BLOCKSIZE;
litDistribTable lt;
litDistribTable ldt;
/* init */
if (buff==NULL) { fprintf(stdout, "not enough memory\n"); exit(1); }
if (litProba==0.0) litProba = matchProba / 4.5;
RDG_fillLiteralDistrib(lt, litProba);
if (litProba<=0.0) litProba = matchProba / 4.5;
RDG_fillLiteralDistrib(ldt, litProba);
SET_BINARY_MODE(stdout);
/* Generate initial dict */
RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, lt, &seed);
RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, ldt, &seed);
/* Generate compressible data */
while (total < size) {
RDG_genBlock(buff, RDG_DICTSIZE+RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, lt, &seed);
if (size-total < RDG_BLOCKSIZE) genBlockSize = (size_t)(size-total);
size_t const genBlockSize = (size_t) (MIN (RDG_BLOCKSIZE, size-total));
RDG_genBlock(buff, RDG_DICTSIZE+RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, ldt, &seed);
total += genBlockSize;
fwrite(buff, 1, genBlockSize, stdout);
{ size_t const unused = fwrite(buff, 1, genBlockSize, stdout); (void)unused; }
/* update dict */
memcpy(buff, buff + RDG_BLOCKSIZE, RDG_DICTSIZE);
}

9
programs/fileio.c
View File

@ -567,7 +567,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
/* Complete Header loading */
{ size_t const toLoad = ZSTD_frameHeaderSize_max - alreadyLoaded; /* assumption : alreadyLoaded <= ZSTD_frameHeaderSize_max */
size_t const checkSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput);
if (checkSize != toLoad) EXM_THROW(32, "Read error");
if (checkSize != toLoad) EXM_THROW(32, "Read error"); /* assumption : srcSize >= ZSTD_frameHeaderSize_max */
}
readSize = ZSTD_frameHeaderSize_max;
@ -581,7 +581,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
/* Write block */
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block into destination"); }
if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block into destination"); }
frameSize += decodedSize;
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) );
@ -613,10 +613,9 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
/* for each frame */
for ( ; ; ) {
size_t sizeCheck;
/* check magic number -> version */
size_t toRead = 4;
sizeCheck = fread(ress.srcBuffer, (size_t)1, toRead, srcFile);
size_t const toRead = 4;
size_t const sizeCheck = fread(ress.srcBuffer, (size_t)1, toRead, srcFile);
if (sizeCheck==0) break; /* no more input */
if (sizeCheck != toRead) EXM_THROW(31, "zstd: %s read error : cannot read header", srcFileName);
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)

4
programs/fullbench.c
View File

@ -182,13 +182,13 @@ size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, co
}
extern size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr);
extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, const void* src, size_t srcSize);
extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, U32 tableRepeatFlag, const void* src, size_t srcSize);
size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
{
U32 DTableML[FSE_DTABLE_SIZE_U32(10)], DTableLL[FSE_DTABLE_SIZE_U32(10)], DTableOffb[FSE_DTABLE_SIZE_U32(9)]; /* MLFSELog, LLFSELog and OffFSELog are not public values */
int nbSeq;
(void)src; (void)srcSize; (void)dst; (void)dstSize;
return ZSTD_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, buff2, g_cSize);
return ZSTD_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, 0, buff2, g_cSize);
}

244
programs/fuzzer.c
View File

@ -39,6 +39,7 @@
#include <stdio.h> /* fgets, sscanf */
#include <sys/timeb.h> /* timeb */
#include <string.h> /* strcmp */
#include <time.h> /* clock_t */
#include "zstd_static.h"
#include "datagen.h" /* RDG_genBuffer */
#include "xxhash.h" /* XXH64 */
@ -69,11 +70,11 @@ static const U32 nbTestsDefault = 30000;
static U32 g_displayLevel = 2;
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
if ((FUZ_GetMilliSpan(g_displayTime) > g_refreshRate) || (g_displayLevel>=4)) \
{ g_displayTime = FUZ_GetMilliStart(); DISPLAY(__VA_ARGS__); \
if ((FUZ_clockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
{ g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
if (g_displayLevel>=4) fflush(stdout); } }
static const U32 g_refreshRate = 150;
static U32 g_displayTime = 0;
static const clock_t g_refreshRate = CLOCKS_PER_SEC * 150 / 1000;
static clock_t g_displayClock = 0;
/*-*******************************************************
@ -82,23 +83,9 @@ static U32 g_displayTime = 0;
#define MIN(a,b) ((a)<(b)?(a):(b))
#define MAX(a,b) ((a)>(b)?(a):(b))
static U32 FUZ_GetMilliStart(void)
static clock_t FUZ_clockSpan(clock_t cStart)
{
struct timeb tb;
U32 nCount;
ftime( &tb );
nCount = (U32) (((tb.time & 0xFFFFF) * 1000) + tb.millitm);
return nCount;
}
static U32 FUZ_GetMilliSpan(U32 nTimeStart)
{
U32 nCurrent = FUZ_GetMilliStart();
U32 nSpan = nCurrent - nTimeStart;
if (nTimeStart > nCurrent)
nSpan += 0x100000 * 1000;
return nSpan;
return clock() - cStart; /* works even when overflow; max span ~ 30mn */
}
@ -386,14 +373,26 @@ static size_t findDiff(const void* buf1, const void* buf2, size_t max)
return i;
}
static size_t FUZ_rLogLength(U32* seed, U32 logLength)
{
size_t const lengthMask = ((size_t)1 << logLength) - 1;
return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
}
static size_t FUZ_randomLength(U32* seed, U32 maxLog)
{
U32 const logLength = FUZ_rand(seed) % maxLog;
return FUZ_rLogLength(seed, logLength);
}
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
static const U32 maxSrcLog = 23;
static const U32 maxSampleLog = 22;
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 maxDuration, double compressibility)
static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxDurationS, double compressibility)
{
static const U32 maxSrcLog = 23;
static const U32 maxSampleLog = 22;
BYTE* cNoiseBuffer[5];
BYTE* srcBuffer;
BYTE* cBuffer;
@ -408,7 +407,8 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 maxDuration, doub
ZSTD_CCtx* refCtx;
ZSTD_CCtx* ctx;
ZSTD_DCtx* dctx;
U32 startTime = FUZ_GetMilliStart();
clock_t startClock = clock();
clock_t const maxClockSpan = maxDurationS * CLOCKS_PER_SEC;
/* allocation */
refCtx = ZSTD_createCCtx();
@ -438,13 +438,12 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 maxDuration, doub
for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed);
/* main test loop */
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < maxDuration); testNb++ ) {
for ( ; (testNb <= nbTests) || (FUZ_clockSpan(startClock) < maxClockSpan); testNb++ ) {
size_t sampleSize, sampleStart, maxTestSize, totalTestSize;
size_t cSize, dSize, errorCode, totalCSize, totalGenSize;
U32 sampleSizeLog, buffNb, cLevelMod, nbChunks, n;
size_t cSize, dSize, totalCSize, totalGenSize;
U32 sampleSizeLog, nbChunks, n;
XXH64_CREATESTATE_STATIC(xxh64);
U64 crcOrig, crcDest;
int cLevel;
U64 crcOrig;
BYTE* sampleBuffer;
const BYTE* dict;
size_t dictSize;
@ -455,21 +454,25 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 maxDuration, doub
FUZ_rand(&coreSeed);
{ U32 const prime1 = 2654435761U; lseed = coreSeed ^ prime1; }
buffNb = FUZ_rand(&lseed) & 127;
if (buffNb & 7) buffNb=2;
else {
buffNb >>= 3;
if (buffNb & 7) {
const U32 tnb[2] = { 1, 3 };
buffNb = tnb[buffNb >> 3];
} else {
const U32 tnb[2] = { 0, 4 };
buffNb = tnb[buffNb >> 3];
} }
srcBuffer = cNoiseBuffer[buffNb];
/* srcBuffer selection [0-4] */
{ U32 buffNb = FUZ_rand(&lseed) & 0x7F;
if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
else {
buffNb >>= 3;
if (buffNb & 7) {
const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
buffNb = tnb[buffNb >> 3];
} else {
const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
buffNb = tnb[buffNb >> 3];
} }
srcBuffer = cNoiseBuffer[buffNb];
}
/* select src segment */
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleSize = FUZ_rLogLength(&lseed, sampleSizeLog);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
/* create sample buffer (to catch read error with valgrind & sanitizers) */
@ -478,24 +481,25 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 maxDuration, doub
memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize);
crcOrig = XXH64(sampleBuffer, sampleSize, 0);
/* compression test */
cLevelMod = MIN( ZSTD_maxCLevel(), (U32)MAX(1, 55 - 3*(int)sampleSizeLog) ); /* high levels only for small samples, for manageable speed */
cLevel = (FUZ_rand(&lseed) % cLevelMod) +1;
cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed");
/* compression tests */
{ int const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (sampleSizeLog/3))) + 1;
cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed");
/* compression failure test : too small dest buffer */
if (cSize > 3) {
const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
const size_t tooSmallSize = cSize - missing;
const U32 endMark = 0x4DC2B1A9;
memcpy(dstBuffer+tooSmallSize, &endMark, 4);
errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel);
CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize);
{ U32 endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, 4);
CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); }
/* compression failure test : too small dest buffer */
if (cSize > 3) {
const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
const size_t tooSmallSize = cSize - missing;
const U32 endMark = 0x4DC2B1A9;
memcpy(dstBuffer+tooSmallSize, &endMark, 4);
{ size_t const errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel);
CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize); }
{ U32 endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, 4);
CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); }
}
}
/* frame header decompression test */
{ ZSTD_frameParams dParams;
size_t const check = ZSTD_getFrameParams(&dParams, cBuffer, cSize);
@ -504,34 +508,34 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 maxDuration, doub
}
/* successful decompression test */
{ size_t margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
{ size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize);
CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize);
crcDest = XXH64(dstBuffer, sampleSize, 0);
CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(sampleBuffer, dstBuffer, sampleSize), (U32)sampleSize);
}
{ U64 const crcDest = XXH64(dstBuffer, sampleSize, 0);
CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(sampleBuffer, dstBuffer, sampleSize), (U32)sampleSize);
} }
free(sampleBuffer); /* no longer useful after this point */
/* truncated src decompression test */
{ const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
const size_t tooSmallSize = cSize - missing;
{ size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
size_t const tooSmallSize = cSize - missing;
void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch overflows */
CHECK(cBufferTooSmall == NULL, "not enough memory !");
memcpy(cBufferTooSmall, cBuffer, tooSmallSize);
errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize);
CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)");
{ size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize);
CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)"); }
free(cBufferTooSmall);
}
/* too small dst decompression test */
if (sampleSize > 3) {
const size_t missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
const size_t tooSmallSize = sampleSize - missing;
size_t const missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
size_t const tooSmallSize = sampleSize - missing;
static const BYTE token = 0xA9;
dstBuffer[tooSmallSize] = token;
errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize);
CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (U32)errorCode, (U32)tooSmallSize);
{ size_t const errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize);
CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (U32)errorCode, (U32)tooSmallSize); }
CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow");
}
@ -563,66 +567,65 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 maxDuration, doub
/* decompress noisy source */
{ U32 const endMark = 0xA9B1C3D6;
memcpy(dstBuffer+sampleSize, &endMark, 4);
errorCode = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize);
/* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */
CHECK((!ZSTD_isError(errorCode)) && (errorCode>sampleSize),
"ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (U32)errorCode, (U32)sampleSize);
{ U32 endCheck; memcpy(&endCheck, dstBuffer+sampleSize, 4);
CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow"); }
} } /* noisy src decompression test */
{ size_t const decompressResult = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize);
/* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */
CHECK((!ZSTD_isError(decompressResult)) && (decompressResult>sampleSize),
"ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (U32)decompressResult, (U32)sampleSize);
}
{ U32 endCheck; memcpy(&endCheck, dstBuffer+sampleSize, 4);
CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow");
} } } /* noisy src decompression test */
/* Streaming compression of scattered segments test */
/*===== Streaming compression test, scattered segments and dictionary =====*/
{ U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
int const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
maxTestSize = FUZ_rLogLength(&lseed, testLog);
if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
sampleSize = FUZ_randomLength(&lseed, maxSampleLog);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
dict = srcBuffer + sampleStart;
dictSize = sampleSize;
{ size_t const errorCode = ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel);
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode)); }
{ size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx);
CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode)); }
}
XXH64_reset(xxh64, 0);
nbChunks = (FUZ_rand(&lseed) & 127) + 2;
sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog;
maxTestSize = (size_t)1 << sampleSizeLog;
maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1);
if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
dict = srcBuffer + sampleStart;
dictSize = sampleSize;
errorCode = ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1);
CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode));
errorCode = ZSTD_copyCCtx(ctx, refCtx);
CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode));
totalTestSize = 0; cSize = 0;
for (n=0; n<nbChunks; n++) {
for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) {
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
if (cBufferSize-cSize < ZSTD_compressBound(sampleSize))
/* avoid invalid dstBufferTooSmall */
break;
if (cBufferSize-cSize < ZSTD_compressBound(sampleSize)) break; /* avoid invalid dstBufferTooSmall */
if (totalTestSize+sampleSize > maxTestSize) break;
errorCode = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+sampleStart, sampleSize);
CHECK (ZSTD_isError(errorCode), "multi-segments compression error : %s", ZSTD_getErrorName(errorCode));
cSize += errorCode;
{ size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+sampleStart, sampleSize);
CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult));
cSize += compressResult;
}
XXH64_update(xxh64, srcBuffer+sampleStart, sampleSize);
memcpy(mirrorBuffer + totalTestSize, srcBuffer+sampleStart, sampleSize);
totalTestSize += sampleSize;
}
errorCode = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize);
CHECK (ZSTD_isError(errorCode), "multi-segments epilogue error : %s", ZSTD_getErrorName(errorCode));
cSize += errorCode;
{ size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize);
CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
cSize += flushResult;
}
crcOrig = XXH64_digest(xxh64);
/* streaming decompression test */
errorCode = ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
CHECK (ZSTD_isError(errorCode), "cannot init DCtx : %s", ZSTD_getErrorName(errorCode));
{ size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
CHECK (ZSTD_isError(errorCode), "cannot init DCtx : %s", ZSTD_getErrorName(errorCode)); }
totalCSize = 0;
totalGenSize = 0;
while (totalCSize < cSize) {
size_t inSize = ZSTD_nextSrcSizeToDecompress(dctx);
size_t genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize);
size_t const inSize = ZSTD_nextSrcSizeToDecompress(dctx);
size_t const genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize);
CHECK (ZSTD_isError(genSize), "streaming decompression error : %s", ZSTD_getErrorName(genSize));
totalGenSize += genSize;
totalCSize += inSize;
@ -630,12 +633,13 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 maxDuration, doub
CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded");
CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
CHECK (totalCSize != cSize, "compressed data should be fully read")
crcDest = XXH64(dstBuffer, totalTestSize, 0);
if (crcDest!=crcOrig)
errorCode = findDiff(mirrorBuffer, dstBuffer, totalTestSize);
CHECK (crcDest!=crcOrig, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)",
(U32)errorCode, (U32)totalTestSize, dstBuffer[errorCode], mirrorBuffer[errorCode]);
}
{ U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
if (crcDest!=crcOrig) {
size_t const errorPos = findDiff(mirrorBuffer, dstBuffer, totalTestSize);
CHECK (crcDest!=crcOrig, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)",
(U32)errorPos, (U32)totalTestSize, dstBuffer[errorPos], mirrorBuffer[errorPos]);
} }
} /* for ( ; (testNb <= nbTests) */
DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
_cleanup:
@ -689,10 +693,9 @@ int main(int argc, const char** argv)
int result=0;
U32 mainPause = 0;
U32 maxDuration = 0;
const char* programName;
const char* programName = argv[0];
/* Check command line */
programName = argv[0];
for (argNb=1; argNb<argc; argNb++) {
const char* argument = argv[argNb];
if(!argument) continue; /* Protection if argument empty */
@ -738,7 +741,6 @@ int main(int argc, const char** argv)
}
if (*argument=='m') maxDuration *=60, argument++;
if (*argument=='n') argument++;
maxDuration *= 1000;
break;
case 's':
@ -780,7 +782,7 @@ int main(int argc, const char** argv)
/* Get Seed */
DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION);
if (!seedset) seed = FUZ_GetMilliStart() % 10000;
if (!seedset) seed = (U32)(clock() % 10000);
DISPLAY("Seed = %u\n", seed);
if (proba!=FUZ_compressibility_default) DISPLAY("Compressibility : %u%%\n", proba);

124
programs/legacy/fileio_legacy.c
View File

@ -1,6 +1,6 @@
/*
fileio.c - File i/o handler
Copyright (C) Yann Collet 2013-2015
fileio_legacy.c - File i/o handler for legacy format
Copyright (C) Yann Collet 2015-2016
GPL v2 License
@ -19,12 +19,11 @@
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- zstd homepage : http://www.zstd.net
- zstd source repository : https://github.com/Cyan4973/zstd
- Public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/*
Note : this is stand-alone program.
It is not part of ZSTD compression library, it is a user program of ZSTD library.
Note : this file is not part of ZSTD compression library.
The license of ZSTD library is BSD.
The license of this file is GPLv2.
*/
@ -172,8 +171,7 @@ unsigned long long FIOv01_decompressFrame(FILE* foutput, FILE* finput)
/* Main decompression Loop */
toRead = ZSTDv01_nextSrcSizeToDecompress(dctx);
while (toRead)
{
while (toRead){
size_t readSize, decodedSize;
/* Fill input buffer */
@ -187,8 +185,7 @@ unsigned long long FIOv01_decompressFrame(FILE* foutput, FILE* finput)
decodedSize = ZSTDv01_decompressContinue(dctx, op, oend-op, inBuff, readSize);
if (ZSTDv01_isError(decodedSize)) EXM_THROW(45, "Decoding error : input corrupted");
if (decodedSize) /* not a header */
{
if (decodedSize) { /* not a header */
/* Write block */
sizeCheck = fwrite(op, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(46, "Write error : unable to write data block to destination file");
@ -232,8 +229,7 @@ unsigned long long FIOv02_decompressFrame(FILE* foutput, FILE* finput)
/* Main decompression Loop */
toRead = ZSTDv02_nextSrcSizeToDecompress(dctx);
while (toRead)
{
while (toRead) {
size_t readSize, decodedSize;
/* Fill input buffer */
@ -247,8 +243,7 @@ unsigned long long FIOv02_decompressFrame(FILE* foutput, FILE* finput)
decodedSize = ZSTDv02_decompressContinue(dctx, op, oend-op, inBuff, readSize);
if (ZSTDv02_isError(decodedSize)) EXM_THROW(45, "Decoding error : input corrupted");
if (decodedSize) /* not a header */
{
if (decodedSize) { /* not a header */
/* Write block */
sizeCheck = fwrite(op, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(46, "Write error : unable to write data block to destination file");
@ -292,8 +287,7 @@ unsigned long long FIOv03_decompressFrame(FILE* foutput, FILE* finput)
/* Main decompression Loop */
toRead = ZSTDv03_nextSrcSizeToDecompress(dctx);
while (toRead)
{
while (toRead) {
size_t readSize, decodedSize;
/* Fill input buffer */
@ -307,8 +301,7 @@ unsigned long long FIOv03_decompressFrame(FILE* foutput, FILE* finput)
decodedSize = ZSTDv03_decompressContinue(dctx, op, oend-op, inBuff, readSize);
if (ZSTDv03_isError(decodedSize)) EXM_THROW(45, "Decoding error : input corrupted");
if (decodedSize) /* not a header */
{
if (decodedSize) { /* not a header */
/* Write block */
sizeCheck = fwrite(op, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(46, "Write error : unable to write data block to destination file");
@ -329,7 +322,7 @@ unsigned long long FIOv03_decompressFrame(FILE* foutput, FILE* finput)
}
/*- v0.4.x -*/
/*===== v0.4.x =====*/
typedef struct {
void* srcBuffer;
@ -380,8 +373,7 @@ unsigned long long FIOv04_decompressFrame(dRessv04_t ress,
ZBUFFv04_decompressInit(ress.dctx);
ZBUFFv04_decompressWithDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
while (1)
{
while (1) {
/* Decode */
size_t sizeCheck;
size_t inSize=readSize, decodedSize=ress.dstBufferSize;
@ -404,11 +396,89 @@ unsigned long long FIOv04_decompressFrame(dRessv04_t ress,
if (readSize != toRead) EXM_THROW(35, "Read error");
}
FIOv04_freeDResources(ress);
return frameSize;
}
/*===== v0.5.x =====*/
typedef struct {
void* srcBuffer;
size_t srcBufferSize;
void* dstBuffer;
size_t dstBufferSize;
void* dictBuffer;
size_t dictBufferSize;
ZBUFFv05_DCtx* dctx;
} dRessv05_t;
static dRessv05_t FIOv05_createDResources(void)
{
dRessv05_t ress;
/* init */
ress.dctx = ZBUFFv05_createDCtx();
if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZBUFF decompression context");
ress.dictBuffer = NULL; ress.dictBufferSize=0;
/* Allocate Memory */
ress.srcBufferSize = ZBUFFv05_recommendedDInSize();
ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = ZBUFFv05_recommendedDOutSize();
ress.dstBuffer = malloc(ress.dstBufferSize);
if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
return ress;
}
static void FIOv05_freeDResources(dRessv05_t ress)
{
size_t const errorCode = ZBUFFv05_freeDCtx(ress.dctx);
if (ZBUFFv05_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFFv05_getErrorName(errorCode));
free(ress.srcBuffer);
free(ress.dstBuffer);
free(ress.dictBuffer);
}
unsigned long long FIOv05_decompressFrame(dRessv05_t ress,
FILE* foutput, FILE* finput)
{
U64 frameSize = 0;
size_t readSize = 4;
MEM_writeLE32(ress.srcBuffer, ZSTDv05_MAGICNUMBER);
ZBUFFv05_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
while (1) {
/* Decode */
size_t sizeCheck;
size_t inSize=readSize, decodedSize=ress.dstBufferSize;
size_t toRead = ZBUFFv05_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
if (ZBUFFv05_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFFv05_getErrorName(toRead));
readSize -= inSize;
/* Write block */
sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file");
frameSize += decodedSize;
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) );
if (toRead == 0) break;
if (readSize) EXM_THROW(38, "Decoding error : should consume entire input");
/* Fill input buffer */
if (toRead > ress.srcBufferSize) EXM_THROW(34, "too large block");
readSize = fread(ress.srcBuffer, 1, toRead, finput);
if (readSize != toRead) EXM_THROW(35, "Read error");
}
return frameSize;
}
/*===== General legacy dispatcher =====*/
unsigned long long FIO_decompressLegacyFrame(FILE* foutput, FILE* finput, U32 magicNumberLE)
{
switch(magicNumberLE)
@ -420,7 +490,17 @@ unsigned long long FIO_decompressLegacyFrame(FILE* foutput, FILE* finput, U32 ma
case ZSTDv03_magicNumber :
return FIOv03_decompressFrame(foutput, finput);
case ZSTDv04_magicNumber :
return FIOv04_decompressFrame(FIOv04_createDResources(), foutput, finput);
{ dRessv04_t r = FIOv04_createDResources();
unsigned long long const s = FIOv04_decompressFrame(r, foutput, finput);
FIOv04_freeDResources(r);
return s;
}
case ZSTDv05_MAGICNUMBER :
{ dRessv05_t r = FIOv05_createDResources();
unsigned long long const s = FIOv05_decompressFrame(r, foutput, finput);
FIOv05_freeDResources(r);
return s;
}
default :
return ERROR(prefix_unknown);
}

43
programs/paramgrill.c
View File

@ -983,8 +983,7 @@ int main(int argc, char** argv)
if (argc<1) { badusage(exename); return 1; }
for(i=1; i<argc; i++)
{
for(i=1; i<argc; i++) {
char* argument = argv[i];
if(!argument) continue; /* Protection if argument empty */
@ -1016,13 +1015,9 @@ int main(int argc, char** argv)
/* Sample compressibility (when no file provided) */
case 'P':
argument++;
{
U32 proba32 = 0;
while ((argument[0]>= '0') && (argument[0]<= '9')) {
proba32 *= 10;
proba32 += argument[0] - '0';
argument++;
}
{ U32 proba32 = 0;
while ((argument[0]>= '0') && (argument[0]<= '9'))
proba32 = (proba32*10) + (*argument++ - '0');
g_compressibility = (double)proba32 / 100.;
}
break;
@ -1082,8 +1077,7 @@ int main(int argc, char** argv)
g_params.strategy = (ZSTD_strategy)(*argument++ - '0');
continue;
case 'L':
{
int cLevel = 0;
{ int cLevel = 0;
argument++;
while ((*argument>= '0') && (*argument<='9'))
cLevel *= 10, cLevel += *argument++ - '0';
@ -1100,25 +1094,20 @@ int main(int argc, char** argv)
case 'T':
argument++;
g_target = 0;
while ((*argument >= '0') && (*argument <= '9')) {
g_target *= 10;
g_target += *argument - '0';
argument++;
}
while ((*argument >= '0') && (*argument <= '9'))
g_target = (g_target*10) + (*argument++ - '0');
break;
/* cut input into blocks */
case 'B':
{
g_blockSize = 0;
argument++;
while ((*argument >='0') && (*argument <='9'))
g_blockSize *= 10, g_blockSize += *argument++ - '0';
if (*argument=='K') g_blockSize<<=10, argument++; /* allows using KB notation */
if (*argument=='M') g_blockSize<<=20, argument++;
if (*argument=='B') argument++;
DISPLAY("using %u KB block size \n", g_blockSize>>10);
}
g_blockSize = 0;
argument++;
while ((*argument >='0') && (*argument <='9'))
g_blockSize = (g_blockSize*10) + (*argument++ - '0');
if (*argument=='K') g_blockSize<<=10, argument++; /* allows using KB notation */
if (*argument=='M') g_blockSize<<=20, argument++;
if (*argument=='B') argument++;
DISPLAY("using %u KB block size \n", g_blockSize>>10);
break;
/* Unknown command */
@ -1126,7 +1115,7 @@ int main(int argc, char** argv)
}
}
continue;
}
} /* if (argument[0]=='-') */
/* first provided filename is input */
if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }

17
programs/playTests.sh
View File

@ -25,16 +25,21 @@ roundTripTest() {
echo "\n**** simple tests **** "
./datagen > tmp
echo -n "trivial compression : "
$ZSTD -f tmp
echo "OK"
$ZSTD -f tmp # trivial compression case, creates tmp.zst
$ZSTD -df tmp.zst # trivial decompression case (overwrites tmp)
echo "test : too large compression level (must fail)"
$ZSTD -99 tmp && die "too large compression level undetected"
$ZSTD tmp -c > tmpCompressed
$ZSTD tmp --stdout > tmpCompressed
$ZSTD tmp -c > tmpCompressed # compression using stdout
$ZSTD tmp --stdout > tmpCompressed # compressoin using stdout, long format
echo "test : decompress file with wrong suffix (must fail)"
$ZSTD -d tmpCompressed && die "wrong suffix error not detected!"
$ZSTD -d tmpCompressed -c > tmpResult
$ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout
$ZSTD --decompress tmpCompressed -c > tmpResult
$ZSTD --decompress tmpCompressed --stdout > tmpResult
$ZSTD -d < tmp.zst > /dev/null # combine decompression, stdin & stdout
$ZSTD -d - < tmp.zst > /dev/null
$ZSTD -dc < tmp.zst > /dev/null
$ZSTD -dc - < tmp.zst > /dev/null
$ZSTD -q tmp && die "overwrite check failed!"
$ZSTD -q -f tmp
$ZSTD -q --force tmp

14
programs/zstdcli.c
View File

@ -22,9 +22,9 @@
- zstd homepage : http://www.zstd.net/
*/
/*
Note : this is user program, not part of libzstd.
The license of this command line program is GPLv2.
Note : this is a user program, not part of libzstd.
The license of libzstd is BSD.
The license of this command line program is GPLv2.
*/
@ -71,9 +71,10 @@
**************************************/
#define COMPRESSOR_NAME "zstd command line interface"
#ifndef ZSTD_VERSION
# define LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
# define QUOTE(str) #str
# define EXPAND_AND_QUOTE(str) QUOTE(str)
# define ZSTD_VERSION "v" EXPAND_AND_QUOTE(ZSTD_VERSION_MAJOR) "." EXPAND_AND_QUOTE(ZSTD_VERSION_MINOR) "." EXPAND_AND_QUOTE(ZSTD_VERSION_RELEASE)
# define ZSTD_VERSION "v" EXPAND_AND_QUOTE(LIB_VERSION)
#endif
#define AUTHOR "Yann Collet"
#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), ZSTD_VERSION, AUTHOR
@ -226,8 +227,7 @@ int main(int argCount, const char** argv)
/* '-' means stdin/stdout */
if (!strcmp(argument, "-")){
if (!filenameIdx) { filenameIdx=1, filenameTable[0]=stdinmark; continue; }
outFileName=stdoutmark; continue;
if (!filenameIdx) { filenameIdx=1, filenameTable[0]=stdinmark; outFileName=stdoutmark; continue; }
}
/* Decode commands (note : aggregated commands are allowed) */
@ -332,14 +332,14 @@ int main(int argCount, const char** argv)
break;
/* Pause at the end (-p) or set an additional param (-p#) (hidden option) */
case 'p': argument++;
case 'p': argument++;
#ifndef ZSTD_NOBENCH
if ((*argument>='0') && (*argument<='9')) {
int additionalParam = 0;
while ((*argument >= '0') && (*argument <= '9'))
additionalParam *= 10, additionalParam += *argument++ - '0';
BMK_setAdditionalParam(additionalParam);
} else
} else
#endif
main_pause=1;
break;

2
visual/2013/zstd/zstd.vcxproj Normal file → Executable file
View File

@ -26,6 +26,7 @@
<ClCompile Include="..\..\..\lib\legacy\zstd_v02.c" />
<ClCompile Include="..\..\..\lib\legacy\zstd_v03.c" />
<ClCompile Include="..\..\..\lib\legacy\zstd_v04.c" />
<ClCompile Include="..\..\..\lib\legacy\zstd_v05.c" />
<ClCompile Include="..\..\..\lib\zbuff.c" />
<ClCompile Include="..\..\..\lib\zdict.c" />
<ClCompile Include="..\..\..\lib\zstd_compress.c" />
@ -49,6 +50,7 @@
<ClInclude Include="..\..\..\lib\legacy\zstd_v02.h" />
<ClInclude Include="..\..\..\lib\legacy\zstd_v03.h" />
<ClInclude Include="..\..\..\lib\legacy\zstd_v04.h" />
<ClInclude Include="..\..\..\lib\legacy\zstd_v05.h" />
<ClInclude Include="..\..\..\lib\zbuff.h" />
<ClInclude Include="..\..\..\lib\zbuff_static.h" />
<ClInclude Include="..\..\..\lib\zdict.h" />

6
visual/2013/zstd/zstd.vcxproj.filters Normal file → Executable file
View File

@ -69,6 +69,9 @@
<ClCompile Include="..\..\..\programs\dibio.c">
<Filter>Fichiers sources</Filter>
</ClCompile>
<ClCompile Include="..\..\..\lib\legacy\zstd_v05.c">
<Filter>Fichiers sources</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\..\lib\fse.h">
@ -146,5 +149,8 @@
<ClInclude Include="..\..\..\programs\dibio.h">
<Filter>Fichiers d%27en-tête</Filter>
</ClInclude>
<ClInclude Include="..\..\..\lib\legacy\zstd_v05.h">
<Filter>Fichiers d%27en-tête</Filter>
</ClInclude>
</ItemGroup>
</Project>