AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add cctx param tests

Browse Source
This commit is contained in:
Stella Lau 2017-08-20 22:55:07 -07:00
parent 6cee6e07e5
commit 023b24e6d4
11 changed files with 639 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lib/common/zstd_internal.h
View File

@ -246,11 +246,14 @@ struct ZSTD_CCtx_params_s {
/* Dictionary */ /* Dictionary */
ZSTD_dictMode_e dictMode; /* select restricting dictionary to "rawContent" or "fullDict" only */ ZSTD_dictMode_e dictMode; /* select restricting dictionary to "rawContent" or "fullDict" only */
U32 dictContentByRef; U32 dictContentByRef;
U32 nbThreads;
/* Multithreading: used only to set mtctx parameters */ /* Multithreading: used only to set mtctx parameters */
U32 nbThreads;
unsigned jobSize; unsigned jobSize;
unsigned overlapSizeLog; unsigned overlapSizeLog;
/* Test parameter */
U32 testParam;
}; };
typedef struct { typedef struct {

230
lib/compress/zstd_compress.c
View File

@ -261,6 +261,7 @@ size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned
#define ZSTD_CLEVEL_CUSTOM 999 #define ZSTD_CLEVEL_CUSTOM 999
#if 0
static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx) static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx)
{ {
if (cctx->requestedParams.compressionLevel==ZSTD_CLEVEL_CUSTOM) return; if (cctx->requestedParams.compressionLevel==ZSTD_CLEVEL_CUSTOM) return;
@ -269,6 +270,7 @@ static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx)
cctx->pledgedSrcSizePlusOne-1, 0); cctx->pledgedSrcSizePlusOne-1, 0);
cctx->requestedParams.compressionLevel = ZSTD_CLEVEL_CUSTOM; cctx->requestedParams.compressionLevel = ZSTD_CLEVEL_CUSTOM;
} }
#endif
ZSTD_CCtx_params* ZSTD_createCCtxParams(void) ZSTD_CCtx_params* ZSTD_createCCtxParams(void)
{ {
@ -280,6 +282,14 @@ ZSTD_CCtx_params* ZSTD_createCCtxParams(void)
return params; return params;
} }
size_t ZSTD_resetCCtxParams(ZSTD_CCtx_params* params)
{
if (!params) { return ERROR(GENERIC); }
memset(params, 0, sizeof(ZSTD_CCtx_params));
params->compressionLevel = ZSTD_CLEVEL_DEFAULT;
return 0;
}
size_t ZSTD_initCCtxParams(ZSTD_CCtx_params* params, size_t ZSTD_initCCtxParams(ZSTD_CCtx_params* params,
ZSTD_compressionParameters cParams) ZSTD_compressionParameters cParams)
{ {
@ -330,102 +340,27 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
switch(param) switch(param)
{ {
case ZSTD_p_compressionLevel : case ZSTD_p_compressionLevel:
if ((int)value > ZSTD_maxCLevel()) value = ZSTD_maxCLevel(); /* cap max compression level */ case ZSTD_p_windowLog:
case ZSTD_p_hashLog:
case ZSTD_p_chainLog:
case ZSTD_p_searchLog:
case ZSTD_p_minMatch:
case ZSTD_p_targetLength:
case ZSTD_p_compressionStrategy:
if (value == 0) return 0; /* special value : 0 means "don't change anything" */ if (value == 0) return 0; /* special value : 0 means "don't change anything" */
if (cctx->cdict) return ERROR(stage_wrong); if (cctx->cdict) return ERROR(stage_wrong);
cctx->requestedParams.compressionLevel = value; return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
return 0;
case ZSTD_p_windowLog : case ZSTD_p_contentSizeFlag:
DEBUGLOG(5, "setting ZSTD_p_windowLog = %u (cdict:%u)", case ZSTD_p_checksumFlag:
value, (cctx->cdict!=NULL)); case ZSTD_p_dictIDFlag:
if (value == 0) return 0; /* special value : 0 means "don't change anything" */ return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
if (cctx->cdict) return ERROR(stage_wrong);
CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
ZSTD_cLevelToCParams(cctx);
cctx->requestedParams.cParams.windowLog = value;
return 0;
case ZSTD_p_hashLog : case ZSTD_p_dictMode:
if (value == 0) return 0; /* special value : 0 means "don't change anything" */ case ZSTD_p_refDictContent:
if (cctx->cdict) return ERROR(stage_wrong);
CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
ZSTD_cLevelToCParams(cctx);
cctx->requestedParams.cParams.hashLog = value;
return 0;
case ZSTD_p_chainLog :
if (value == 0) return 0; /* special value : 0 means "don't change anything" */
if (cctx->cdict) return ERROR(stage_wrong);
CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
ZSTD_cLevelToCParams(cctx);
cctx->requestedParams.cParams.chainLog = value;
return 0;
case ZSTD_p_searchLog :
if (value == 0) return 0; /* special value : 0 means "don't change anything" */
if (cctx->cdict) return ERROR(stage_wrong);
CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
ZSTD_cLevelToCParams(cctx);
cctx->requestedParams.cParams.searchLog = value;
return 0;
case ZSTD_p_minMatch :
if (value == 0) return 0; /* special value : 0 means "don't change anything" */
if (cctx->cdict) return ERROR(stage_wrong);
CLAMPCHECK(value, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX);
ZSTD_cLevelToCParams(cctx);
cctx->requestedParams.cParams.searchLength = value;
return 0;
case ZSTD_p_targetLength :
if (value == 0) return 0; /* special value : 0 means "don't change anything" */
if (cctx->cdict) return ERROR(stage_wrong);
CLAMPCHECK(value, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
ZSTD_cLevelToCParams(cctx);
cctx->requestedParams.cParams.targetLength = value;
return 0;
case ZSTD_p_compressionStrategy :
if (value == 0) return 0; /* special value : 0 means "don't change anything" */
if (cctx->cdict) return ERROR(stage_wrong);
CLAMPCHECK(value, (unsigned)ZSTD_fast, (unsigned)ZSTD_btultra);
ZSTD_cLevelToCParams(cctx);
cctx->requestedParams.cParams.strategy = (ZSTD_strategy)value;
return 0;
case ZSTD_p_contentSizeFlag :
DEBUGLOG(5, "set content size flag = %u", (value>0));
/* Content size written in frame header _when known_ (default:1) */
cctx->requestedParams.fParams.contentSizeFlag = value>0;
return 0;
case ZSTD_p_checksumFlag :
/* A 32-bits content checksum will be calculated and written at end of frame (default:0) */
cctx->requestedParams.fParams.checksumFlag = value>0;
return 0;
case ZSTD_p_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */
DEBUGLOG(5, "set dictIDFlag = %u", (value>0));
cctx->requestedParams.fParams.noDictIDFlag = (value==0);
return 0;
/* Dictionary parameters */
case ZSTD_p_dictMode :
if (cctx->cdict) return ERROR(stage_wrong); /* must be set before loading */ if (cctx->cdict) return ERROR(stage_wrong); /* must be set before loading */
/* restrict dictionary mode, to "rawContent" or "fullDict" only */ return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
ZSTD_STATIC_ASSERT((U32)ZSTD_dm_fullDict > (U32)ZSTD_dm_rawContent);
if (value > (unsigned)ZSTD_dm_fullDict)
return ERROR(parameter_outOfBound);
cctx->requestedParams.dictMode = (ZSTD_dictMode_e)value;
return 0;
case ZSTD_p_refDictContent :
if (cctx->cdict) return ERROR(stage_wrong); /* must be set before loading */
/* dictionary content will be referenced, instead of copied */
cctx->requestedParams.dictContentByRef = value>0;
return 0;
case ZSTD_p_forceMaxWindow : /* Force back-references to remain < windowSize, case ZSTD_p_forceMaxWindow : /* Force back-references to remain < windowSize,
* even when referencing into Dictionary content * even when referencing into Dictionary content
@ -462,6 +397,11 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
assert(cctx->mtctx != NULL); assert(cctx->mtctx != NULL);
return ZSTDMT_setMTCtxParameter(cctx->mtctx, ZSTDMT_p_overlapSectionLog, value); return ZSTDMT_setMTCtxParameter(cctx->mtctx, ZSTDMT_p_overlapSectionLog, value);
case ZSTD_p_test :
DEBUGLOG(2, "Setting test parameter = %u", value);
cctx->requestedParams.testParam = (value > 0);
return 0;
default: return ERROR(parameter_unsupported); default: return ERROR(parameter_unsupported);
} }
} }
@ -527,14 +467,18 @@ size_t ZSTD_CCtxParam_setParameter(
return 0; return 0;
case ZSTD_p_contentSizeFlag : case ZSTD_p_contentSizeFlag :
/* Content size written in frame header _when known_ (default:1) */
DEBUGLOG(5, "set content size flag = %u", (value>0));
params->fParams.contentSizeFlag = value > 0; params->fParams.contentSizeFlag = value > 0;
return 0; return 0;
case ZSTD_p_checksumFlag : case ZSTD_p_checksumFlag :
/* A 32-bits content checksum will be calculated and written at end of frame (default:0) */
params->fParams.checksumFlag = value > 0; params->fParams.checksumFlag = value > 0;
return 0; return 0;
case ZSTD_p_dictIDFlag : case ZSTD_p_dictIDFlag :
DEBUGLOG(5, "set dictIDFlag = %u", (value>0));
params->fParams.noDictIDFlag = (value == 0); params->fParams.noDictIDFlag = (value == 0);
return 0; return 0;
@ -559,7 +503,7 @@ size_t ZSTD_CCtxParam_setParameter(
#ifndef ZSTD_MULTITHREAD #ifndef ZSTD_MULTITHREAD
if (value > 1) return ERROR(parameter_unsupported); if (value > 1) return ERROR(parameter_unsupported);
#endif #endif
// Do checks when applying parameters to cctx. /* Do checks when applying params to cctx */
params->nbThreads = value; params->nbThreads = value;
return 0; return 0;
@ -569,18 +513,57 @@ size_t ZSTD_CCtxParam_setParameter(
return 0; return 0;
case ZSTD_p_overlapSizeLog : case ZSTD_p_overlapSizeLog :
if (params->nbThreads <= 1) { return ERROR(parameter_unsupported); }
params->overlapSizeLog = value; params->overlapSizeLog = value;
return 0; return 0;
case ZSTD_p_test :
DEBUGLOG(2, "setting opaque: ZSTD_p_test: %u", value);
params->testParam = (value > 0);
return 0;
default: return ERROR(parameter_unsupported); default: return ERROR(parameter_unsupported);
} }
} }
static void ZSTD_debugPrintCCtxParams(ZSTD_CCtx_params* params)
{
DEBUGLOG(2, "======CCtxParams======");
DEBUGLOG(2, "cParams: %u %u %u %u %u %u %u",
params->cParams.windowLog,
params->cParams.chainLog,
params->cParams.hashLog,
params->cParams.searchLog,
params->cParams.searchLength,
params->cParams.targetLength,
params->cParams.strategy);
DEBUGLOG(2, "fParams: %u %u %u",
params->fParams.contentSizeFlag,
params->fParams.checksumFlag,
params->fParams.noDictIDFlag);
DEBUGLOG(2, "cLevel, forceWindow: %u %u",
params->compressionLevel,
params->forceWindow);
DEBUGLOG(2, "dictionary: %u %u",
params->dictMode,
params->dictContentByRef);
DEBUGLOG(2, "multithreading: %u %u %u",
params->nbThreads,
params->jobSize,
params->overlapSizeLog);
DEBUGLOG(2, "testParam: %u",
params->testParam);
}
// This function should probably be updated whenever ZSTD_CCtx_params is updated. // This function should probably be updated whenever ZSTD_CCtx_params is updated.
ZSTDLIB_API size_t ZSTD_CCtx_applyCCtxParams(ZSTD_CCtx* cctx, ZSTD_CCtx_params* params) ZSTDLIB_API size_t ZSTD_CCtx_applyCCtxParams(ZSTD_CCtx* cctx, ZSTD_CCtx_params* params)
{ {
if (params == NULL) { return ERROR(GENERIC); }
if (cctx->cdict) { return ERROR(stage_wrong); } if (cctx->cdict) { return ERROR(stage_wrong); }
DEBUGLOG(2, "Applying cctx params\n");
ZSTD_debugPrintCCtxParams(params);
/* Assume the compression and frame parameters are validated */ /* Assume the compression and frame parameters are validated */
cctx->requestedParams.cParams = params->cParams; cctx->requestedParams.cParams = params->cParams;
cctx->requestedParams.fParams = params->fParams; cctx->requestedParams.fParams = params->fParams;
@ -596,9 +579,14 @@ ZSTDLIB_API size_t ZSTD_CCtx_applyCCtxParams(ZSTD_CCtx* cctx, ZSTD_CCtx_params*
/* Set multithreading parameters explicitly */ /* Set multithreading parameters explicitly */
CHECK_F( ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbThreads, params->nbThreads) ); CHECK_F( ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbThreads, params->nbThreads) );
CHECK_F( ZSTD_CCtx_setParameter(cctx, ZSTD_p_jobSize, params->jobSize) ); if (params->nbThreads > 1) {
CHECK_F( ZSTD_CCtx_setParameter( CHECK_F( ZSTD_CCtx_setParameter(cctx, ZSTD_p_jobSize, params->jobSize) );
CHECK_F( ZSTD_CCtx_setParameter(
cctx, ZSTD_p_overlapSizeLog, params->overlapSizeLog) ); cctx, ZSTD_p_overlapSizeLog, params->overlapSizeLog) );
}
/* Copy test parameter */
cctx->requestedParams.testParam = params->testParam;
return 0; return 0;
} }
@ -790,8 +778,7 @@ size_t ZSTD_estimateCCtxSize(int compressionLevel)
size_t ZSTD_estimateCStreamSize_advanced_opaque(ZSTD_CCtx_params* params) size_t ZSTD_estimateCStreamSize_advanced_opaque(ZSTD_CCtx_params* params)
{ {
if (params == NULL) { return 0; } if (params == NULL) { return 0; }
{ { size_t const CCtxSize = ZSTD_estimateCCtxSize_advanced_opaque(params);
size_t const CCtxSize = ZSTD_estimateCCtxSize_advanced_opaque(params);
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params->cParams.windowLog); size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params->cParams.windowLog);
size_t const inBuffSize = ((size_t)1 << params->cParams.windowLog) + blockSize; size_t const inBuffSize = ((size_t)1 << params->cParams.windowLog) + blockSize;
size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1; size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
@ -3265,7 +3252,6 @@ size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
ZSTD_compressionParameters cParams = (cLevel == ZSTD_CLEVEL_CUSTOM) ? ZSTD_compressionParameters cParams = (cLevel == ZSTD_CLEVEL_CUSTOM) ?
cctx->appliedParams.cParams : cctx->appliedParams.cParams :
ZSTD_getCParams(cLevel, 0, 0); ZSTD_getCParams(cLevel, 0, 0);
DEBUGLOG(2, "ZSTD_getBlockSize: cLevel %u\n", cLevel);
return MIN (ZSTD_BLOCKSIZE_MAX, 1 << cParams.windowLog); return MIN (ZSTD_BLOCKSIZE_MAX, 1 << cParams.windowLog);
} }
@ -3485,7 +3471,8 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
return ZSTD_compress_insertDictionary(cctx, dict, dictSize, params.dictMode); return ZSTD_compress_insertDictionary(cctx, dict, dictSize, params.dictMode);
} }
size_t ZSTD_compressBegin_advanced_opaque(ZSTD_CCtx* cctx, size_t ZSTD_compressBegin_advanced_opaque(
ZSTD_CCtx* cctx,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_CCtx_params params, ZSTD_CCtx_params params,
unsigned long long pledgedSrcSize) unsigned long long pledgedSrcSize)
@ -3619,11 +3606,12 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
} }
/* Internal */ /* Internal */
size_t ZSTD_compress_advanced_opaque(ZSTD_CCtx* cctx, size_t ZSTD_compress_advanced_opaque(
void* dst, size_t dstCapacity, ZSTD_CCtx* cctx,
const void* src, size_t srcSize, void* dst, size_t dstCapacity,
const void* dict,size_t dictSize, const void* src, size_t srcSize,
ZSTD_CCtx_params params) const void* dict,size_t dictSize,
ZSTD_CCtx_params params)
{ {
CHECK_F( ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL, CHECK_F( ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL,
params, srcSize, ZSTDb_not_buffered) ); params, srcSize, ZSTDb_not_buffered) );
@ -3751,11 +3739,11 @@ static size_t ZSTD_initCDict_internal(
} }
#endif #endif
ZSTD_CDict* ZSTD_createCDict_advanced_opaque( static ZSTD_CDict* ZSTD_createCDict_advanced_opaque(
const void* dictBuffer, size_t dictSize, const void* dictBuffer, size_t dictSize,
ZSTD_CCtx_params params, ZSTD_customMem customMem) ZSTD_CCtx_params params, ZSTD_customMem customMem)
{ {
DEBUGLOG(5, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode); DEBUGLOG(5, "ZSTD_createCDict_advanced_opaque, mode %u", (U32)params.dictMode);
if (!customMem.customAlloc ^ !customMem.customFree) return NULL; if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
{ ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); { ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
@ -3794,31 +3782,6 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize,
cctxParams.dictMode = dictMode; cctxParams.dictMode = dictMode;
cctxParams.dictContentByRef = byReference; cctxParams.dictContentByRef = byReference;
return ZSTD_createCDict_advanced_opaque(dictBuffer, dictSize, cctxParams, customMem); return ZSTD_createCDict_advanced_opaque(dictBuffer, dictSize, cctxParams, customMem);
#if 0
DEBUGLOG(5, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode);
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
{ ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem);
if (!cdict || !cctx) {
ZSTD_free(cdict, customMem);
ZSTD_freeCCtx(cctx);
return NULL;
}
cdict->refContext = cctx;
if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
dictBuffer, dictSize,
byReference, dictMode,
cParams) )) {
ZSTD_freeCDict(cdict);
return NULL;
}
return cdict;
}
#endif
} }
ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel)
@ -4064,7 +4027,6 @@ size_t ZSTD_initCStream_internal_opaque(ZSTD_CStream* zcs,
ZSTD_CCtx_params params, ZSTD_CCtx_params params,
unsigned long long pledgedSrcSize) unsigned long long pledgedSrcSize)
{ {
DEBUGLOG(5, "ZSTD_initCStream_internal");
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
assert(!((dict) && (cdict))); /* either dict or cdict, not both */ assert(!((dict) && (cdict))); /* either dict or cdict, not both */
@ -4145,9 +4107,9 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
{ {
ZSTD_CCtx_params cctxParams = zcs->requestedParams; ZSTD_CCtx_params cctxParams = zcs->requestedParams;
CHECK_F( ZSTD_checkCParams(params.cParams) ); CHECK_F( ZSTD_checkCParams(params.cParams) );
zcs->requestedParams.cParams = params.cParams; cctxParams.cParams = params.cParams;
zcs->requestedParams.fParams = params.fParams; cctxParams.fParams = params.fParams;
zcs->requestedParams.compressionLevel = ZSTD_CLEVEL_CUSTOM; cctxParams.compressionLevel = ZSTD_CLEVEL_CUSTOM;
return ZSTD_initCStream_internal_opaque(zcs, dict, dictSize, NULL, cctxParams, pledgedSrcSize); return ZSTD_initCStream_internal_opaque(zcs, dict, dictSize, NULL, cctxParams, pledgedSrcSize);
} }

4
lib/compress/zstdmt_compress.c
View File

@ -62,7 +62,7 @@ static unsigned long long GetCurrentClockTimeMicroseconds(void)
DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \ DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \
elapsedTime, #mutex); \ elapsedTime, #mutex); \
} } \ } } \
} else pthread_mutex_lock(mutex); \ } else { pthread_mutex_lock(mutex); } \
} }
#else #else
@ -646,7 +646,7 @@ static size_t ZSTDMT_compress_advanced_opaque(
} }
} /* for (chunkID=0; chunkID<nbChunks; chunkID++) */ } /* for (chunkID=0; chunkID<nbChunks; chunkID++) */
DEBUGLOG(4, "checksumFlag : %u ", params.fParams.checksumFlag); DEBUGLOG(4, "checksumFlag : %u ", cctxParams.fParams.checksumFlag);
if (cctxParams.fParams.checksumFlag) { if (cctxParams.fParams.checksumFlag) {
U32 const checksum = (U32)XXH64_digest(&xxh64); U32 const checksum = (U32)XXH64_digest(&xxh64);
if (dstPos + 4 > dstCapacity) { if (dstPos + 4 > dstCapacity) {

10
lib/zstd.h
View File

@ -498,6 +498,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
* It will also consider src size to be arbitrarily "large", which is worst case. * It will also consider src size to be arbitrarily "large", which is worst case.
* If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation. * If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
* ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. * ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
* TODO: ZSTD_estimateCCtxSize_advanced_opaque()
* Note : CCtx estimation is only correct for single-threaded compression */ * Note : CCtx estimation is only correct for single-threaded compression */
ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
ZSTDLIB_API size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams); ZSTDLIB_API size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams);
@ -509,6 +510,7 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
* It will also consider src size to be arbitrarily "large", which is worst case. * It will also consider src size to be arbitrarily "large", which is worst case.
* If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation. * If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
* ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. * ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
* TODO: ZSTD_estimateCStreamSize_advanced_opaque
* Note : CStream estimation is only correct for single-threaded compression. * Note : CStream estimation is only correct for single-threaded compression.
* ZSTD_DStream memory budget depends on window Size. * ZSTD_DStream memory budget depends on window Size.
* This information can be passed manually, using ZSTD_estimateDStreamSize, * This information can be passed manually, using ZSTD_estimateDStreamSize,
@ -525,11 +527,10 @@ ZSTDLIB_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t sr
/*! ZSTD_estimate?DictSize() : /*! ZSTD_estimate?DictSize() :
* ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict().
* ZSTD_estimateCStreamSize_advanced() makes it possible to control precisely compression parameters, like ZSTD_createCDict_advanced(). * ZSTD_estimateCStreamSize_advanced() makes it possible to control precisely compression parameters, like ZSTD_createCDict_advanced().
* TODO: ZSTD_estimateCDictSize_advanced_opaque(), can set by reference
* Note : dictionary created "byReference" are smaller */ * Note : dictionary created "byReference" are smaller */
ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, unsigned byReference); ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, unsigned byReference);
// By reference
ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced_opaque(size_t dictSize, ZSTD_CCtx_params* params); ZSTDLIB_API size_t ZSTD_estimateCDictSize_advanced_opaque(size_t dictSize, ZSTD_CCtx_params* params);
ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference); ZSTDLIB_API size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
@ -613,12 +614,11 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_initStaticCDict_advanced_opaque(
ZSTD_CCtx_params* params); ZSTD_CCtx_params* params);
ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void);
ZSTDLIB_API size_t ZSTD_resetCCtxParams(ZSTD_CCtx_params* params);
ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createAndInitCCtxParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createAndInitCCtxParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
ZSTDLIB_API size_t ZSTD_initCCtxParams(ZSTD_CCtx_params* params, ZSTD_compressionParameters cParams); ZSTDLIB_API size_t ZSTD_initCCtxParams(ZSTD_CCtx_params* params, ZSTD_compressionParameters cParams);
ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params);
ZSTDLIB_API
/*! ZSTD_getCParams() : /*! ZSTD_getCParams() :
* @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
@ -647,6 +647,7 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx,
const void* dict,size_t dictSize, const void* dict,size_t dictSize,
ZSTD_parameters params); ZSTD_parameters params);
/*! ZSTD_compress_usingCDict_advanced() : /*! ZSTD_compress_usingCDict_advanced() :
* Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */ * Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */
ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
@ -1002,6 +1003,7 @@ typedef enum {
/* advanced parameters - may not remain available after API update */ /* advanced parameters - may not remain available after API update */
ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize, ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize,
* even when referencing into Dictionary content (default:0) */ * even when referencing into Dictionary content (default:0) */
ZSTD_p_test,
} ZSTD_cParameter; } ZSTD_cParameter;

3
programs/Makefile
View File

@ -47,7 +47,8 @@ DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \ -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
-Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-Wredundant-decls -Wredundant-decls
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) ZSTD_DEBUG_FLAGS = -g -DZSTD_DEBUG=2
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) $(ZSTD_DEBUG_FLAGS)
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)

5
programs/fileio.c
View File

@ -213,6 +213,8 @@ void FIO_setOverlapLog(unsigned overlapLog){
DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n"); DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n");
g_overlapLog = overlapLog; g_overlapLog = overlapLog;
} }
static U32 g_testParamFlag = 0;
void FIO_setTestParamFlag(unsigned testParamFlag) { g_testParamFlag = testParamFlag; }
/*-************************************* /*-*************************************
@ -411,6 +413,9 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
/* dictionary */ /* dictionary */
CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) ); CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
/* Test */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_test, g_testParamFlag) );
} }
#elif defined(ZSTD_MULTITHREAD) #elif defined(ZSTD_MULTITHREAD)
{ ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize); { ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);

1
programs/zstdcli.c
View File

@ -430,6 +430,7 @@ int main(int argCount, const char* argv[])
if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
if (!strcmp(argument, "--testParam")) { FIO_setTestParamFlag(1); continue; }
#ifdef ZSTD_GZCOMPRESS #ifdef ZSTD_GZCOMPRESS
if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; } if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; }
#endif #endif

8
tests/Makefile
View File

@ -25,7 +25,7 @@ PRGDIR = ../programs
PYTHON ?= python3 PYTHON ?= python3
TESTARTEFACT := versionsTest namespaceTest TESTARTEFACT := versionsTest namespaceTest
DEBUGLEVEL= 1 DEBUGLEVEL= 2
DEBUGFLAGS= -g -DZSTD_DEBUG=$(DEBUGLEVEL) DEBUGFLAGS= -g -DZSTD_DEBUG=$(DEBUGLEVEL)
CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
@ -169,6 +169,12 @@ datagen : $(PRGDIR)/datagen.c datagencli.c
roundTripCrash : $(ZSTD_FILES) roundTripCrash.c roundTripCrash : $(ZSTD_FILES) roundTripCrash.c
$(CC) $(FLAGS) $^ -o $@$(EXT) $(CC) $(FLAGS) $^ -o $@$(EXT)
OPAQUEFILES := $(ZSTD_FILES) $(ZDICT_FILES) roundTripCrashOpaque.c
roundTripCrashOpaque : LDFLAGS += $(MULTITHREAD_CPP)
roundTripCrashOpaque : LDFLAGS += $(MULTITHREAD_LD)
roundTripCrashOpaque : $(OPAQUEFILES)
$(CC) $(FLAGS) $^ -o $@$(EXT)
longmatch : $(ZSTD_FILES) longmatch.c longmatch : $(ZSTD_FILES) longmatch.c
$(CC) $(FLAGS) $^ -o $@$(EXT) $(CC) $(FLAGS) $^ -o $@$(EXT)

5
tests/fullbench.c
View File

@ -136,8 +136,10 @@ size_t local_ZSTD_compressStream(void* dst, size_t dstCapacity, void* buff2, con
buffIn.src = src; buffIn.src = src;
buffIn.size = srcSize; buffIn.size = srcSize;
buffIn.pos = 0; buffIn.pos = 0;
ZSTD_compressStream(g_cstream, &buffOut, &buffIn); ZSTD_compressStream(g_cstream, &buffOut, &buffIn);
ZSTD_endStream(g_cstream, &buffOut); ZSTD_endStream(g_cstream, &buffOut);
return buffOut.pos; return buffOut.pos;
} }
@ -383,11 +385,13 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
const BYTE* ip = dstBuff; const BYTE* ip = dstBuff;
const BYTE* iend; const BYTE* iend;
size_t frameHeaderSize, cBlockSize; size_t frameHeaderSize, cBlockSize;
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); /* it would be better to use direct block compression here */ ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); /* it would be better to use direct block compression here */
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min); frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min; if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
ip += frameHeaderSize; /* Skip frame Header */ ip += frameHeaderSize; /* Skip frame Header */
cBlockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */ cBlockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed) { if (bp.blockType != bt_compressed) {
DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n"); DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n");
@ -395,6 +399,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
} }
iend = ip + ZSTD_blockHeaderSize + cBlockSize; /* End of first block */ iend = ip + ZSTD_blockHeaderSize + cBlockSize; /* End of first block */
ip += ZSTD_blockHeaderSize; /* skip block header */ ip += ZSTD_blockHeaderSize; /* skip block header */
ZSTD_decompressBegin(g_zdc); ZSTD_decompressBegin(g_zdc);
ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, iend-ip); /* skip literal segment */ ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, iend-ip); /* skip literal segment */
g_cSize = iend-ip; g_cSize = iend-ip;

203
tests/roundTripCrashOpaque.c Normal file
View File

@ -0,0 +1,203 @@
/**
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
/*
This program takes a file in input,
performs a zstd round-trip test (compression - decompress)
compares the result with original
and generates a crash (double free) on corruption detection.
*/
/*===========================================
* Dependencies
*==========================================*/
#include <stddef.h> /* size_t */
#include <stdlib.h> /* malloc, free, exit */
#include <stdio.h> /* fprintf */
#include <sys/types.h> /* stat */
#include <sys/stat.h> /* stat */
#include "xxhash.h"
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
/*===========================================
* Macros
*==========================================*/
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
#define CHECK_Z(f) { \
size_t const err = f; \
if (ZSTD_isError(err)) { \
fprintf(stderr, \
"Error=> %s: %s", \
#f, ZSTD_getErrorName(err)); \
exit(1); \
} }
/** roundTripTest() :
* Compresses `srcBuff` into `compressedBuff`,
* then decompresses `compressedBuff` into `resultBuff`.
* @return : result of decompression, which should be == `srcSize`
* or an error code if either compression or decompression fails.
* Note : `compressedBuffCapacity` should be `>= ZSTD_compressBound(srcSize)`
* for compression to be guaranteed to work */
static size_t roundTripTest(void* resultBuff, size_t resultBuffCapacity,
void* compressedBuff, size_t compressedBuffCapacity,
const void* srcBuff, size_t srcBuffSize)
{
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_CCtx_params* const cctxParams = ZSTD_createCCtxParams();
ZSTD_inBuffer inBuffer = { srcBuff, srcBuffSize, 0 };
ZSTD_outBuffer outBuffer = {compressedBuff, compressedBuffCapacity, 0 };
ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_compressionLevel, 1);
ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_test, 1);
ZSTD_CCtx_applyCCtxParams(cctx, cctxParams);
CHECK_Z (ZSTD_compress_generic(cctx, &outBuffer, &inBuffer, ZSTD_e_end) );
ZSTD_freeCCtxParams(cctxParams);
ZSTD_freeCCtx(cctx);
return ZSTD_decompress(resultBuff, resultBuffCapacity, compressedBuff, outBuffer.pos);
}
static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize)
{
const char* ip1 = (const char*)buff1;
const char* ip2 = (const char*)buff2;
size_t pos;
for (pos=0; pos<buffSize; pos++)
if (ip1[pos]!=ip2[pos])
break;
return pos;
}
static void crash(int errorCode){
/* abort if AFL/libfuzzer, exit otherwise */
#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION /* could also use __AFL_COMPILER */
abort();
#else
exit(errorCode);
#endif
}
static void roundTripCheck(const void* srcBuff, size_t srcBuffSize)
{
size_t const cBuffSize = ZSTD_compressBound(srcBuffSize);
void* cBuff = malloc(cBuffSize);
void* rBuff = malloc(cBuffSize);
if (!cBuff || !rBuff) {
fprintf(stderr, "not enough memory ! \n");
exit (1);
}
{ size_t const result = roundTripTest(rBuff, cBuffSize, cBuff, cBuffSize, srcBuff, srcBuffSize);
if (ZSTD_isError(result)) {
fprintf(stderr, "roundTripTest error : %s \n", ZSTD_getErrorName(result));
crash(1);
}
if (result != srcBuffSize) {
fprintf(stderr, "Incorrect regenerated size : %u != %u\n", (unsigned)result, (unsigned)srcBuffSize);
crash(1);
}
if (checkBuffers(srcBuff, rBuff, srcBuffSize) != srcBuffSize) {
fprintf(stderr, "Silent decoding corruption !!!");
crash(1);
}
}
free(cBuff);
free(rBuff);
}
static size_t getFileSize(const char* infilename)
{
int r;
#if defined(_MSC_VER)
struct _stat64 statbuf;
r = _stat64(infilename, &statbuf);
if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
#else
struct stat statbuf;
r = stat(infilename, &statbuf);
if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
#endif
return (size_t)statbuf.st_size;
}
static int isDirectory(const char* infilename)
{
int r;
#if defined(_MSC_VER)
struct _stat64 statbuf;
r = _stat64(infilename, &statbuf);
if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
#else
struct stat statbuf;
r = stat(infilename, &statbuf);
if (!r && S_ISDIR(statbuf.st_mode)) return 1;
#endif
return 0;
}
/** loadFile() :
* requirement : `buffer` size >= `fileSize` */
static void loadFile(void* buffer, const char* fileName, size_t fileSize)
{
FILE* const f = fopen(fileName, "rb");
if (isDirectory(fileName)) {
fprintf(stderr, "Ignoring %s directory \n", fileName);
exit(2);
}
if (f==NULL) {
fprintf(stderr, "Impossible to open %s \n", fileName);
exit(3);
}
{ size_t const readSize = fread(buffer, 1, fileSize, f);
if (readSize != fileSize) {
fprintf(stderr, "Error reading %s \n", fileName);
exit(5);
} }
fclose(f);
}
static void fileCheck(const char* fileName)
{
size_t const fileSize = getFileSize(fileName);
void* buffer = malloc(fileSize);
if (!buffer) {
fprintf(stderr, "not enough memory \n");
exit(4);
}
loadFile(buffer, fileName, fileSize);
roundTripCheck(buffer, fileSize);
free (buffer);
}
int main(int argCount, const char** argv) {
if (argCount < 2) {
fprintf(stderr, "Error : no argument : need input file \n");
exit(9);
}
fileCheck(argv[1]);
fprintf(stderr, "no pb detected\n");
return 0;
}

309
tests/zstreamtest.c
View File

@ -1494,6 +1494,309 @@ _output_error:
goto _cleanup; goto _cleanup;
} }
static int fuzzerTests_newAPI_opaque(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests)
{
U32 const maxSrcLog = bigTests ? 24 : 22;
static const U32 maxSampleLog = 19;
size_t const srcBufferSize = (size_t)1<<maxSrcLog;
BYTE* cNoiseBuffer[5];
size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog);
BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize);
size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
BYTE* const cBuffer = (BYTE*)malloc (cBufferSize);
size_t const dstBufferSize = srcBufferSize;
BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize);
U32 result = 0;
U32 testNb = 0;
U32 coreSeed = seed;
ZSTD_CCtx* zc = ZSTD_createCCtx(); /* will be reset sometimes */
ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */
ZSTD_DStream* const zd_noise = ZSTD_createDStream();
clock_t const startClock = clock();
const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */
size_t dictSize = 0;
U32 oldTestLog = 0;
U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests;
U32 const nbThreadsMax = bigTests ? 5 : 1;
ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
/* allocations */
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
!copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise ,
"Not enough memory, fuzzer tests cancelled");
/* Create initial samples */
RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) ); /* ensure at least one init */
/* catch up testNb */
for (testNb=1; testNb < startTest; testNb++)
FUZ_rand(&coreSeed);
/* test loop */
for ( ; (testNb <= nbTests) || (FUZ_GetClockSpan(startClock) < g_clockTime) ; testNb++ ) {
U32 lseed;
const BYTE* srcBuffer;
size_t totalTestSize, totalGenSize, cSize;
XXH64_state_t xxhState;
U64 crcOrig;
U32 resetAllowed = 1;
size_t maxTestSize;
/* init */
if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
FUZ_rand(&coreSeed);
lseed = coreSeed ^ prime32;
/* states full reset (deliberately not synchronized) */
/* some issues can only happen when reusing states */
if ((FUZ_rand(&lseed) & 0xFF) == 131) {
DISPLAYLEVEL(5, "Creating new context \n");
ZSTD_freeCCtx(zc);
zc = ZSTD_createCCtx();
CHECK(zc==NULL, "ZSTD_createCCtx allocation error");
resetAllowed=0;
}
if ((FUZ_rand(&lseed) & 0xFF) == 132) {
ZSTD_freeDStream(zd);
zd = ZSTD_createDStream();
CHECK(zd==NULL, "ZSTD_createDStream allocation error");
ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */
}
/* srcBuffer selection [0-4] */
{ U32 buffNb = FUZ_rand(&lseed) & 0x7F;
if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
else {
buffNb >>= 3;
if (buffNb & 7) {
const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
buffNb = tnb[buffNb >> 3];
} else {
const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
buffNb = tnb[buffNb >> 3];
} }
srcBuffer = cNoiseBuffer[buffNb];
}
/* compression init */
CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) ); /* cancel previous dict /*/
if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
&& oldTestLog /* at least one test happened */ && resetAllowed) {
maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1;
{ int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1;
CHECK_Z (ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_compressionLevel, compressionLevel));
}
} else {
U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const cLevelCandidate = (FUZ_rand(&lseed) %
(ZSTD_maxCLevel() -
(MAX(testLog, dictLog) / 3))) +
1;
U32 const cLevel = MIN(cLevelCandidate, cLevelMax);
maxTestSize = FUZ_rLogLength(&lseed, testLog);
oldTestLog = testLog;
/* random dictionary selection */
dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
dict = srcBuffer + dictStart;
if (!dictSize) dict=NULL;
}
{ U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize);
/* mess with compression parameters */
cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1;
cParams.searchLength += (FUZ_rand(&lseed) & 3) - 1;
cParams.targetLength = (U32)(cParams.targetLength * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128)));
cParams = ZSTD_adjustCParams(cParams, 0, 0);
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_windowLog, cParams.windowLog) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_hashLog, cParams.hashLog) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_chainLog, cParams.chainLog) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_searchLog, cParams.searchLog) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_minMatch, cParams.searchLength) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_targetLength, cParams.targetLength) );
/* unconditionally set, to be sync with decoder */
/* mess with frame parameters */
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_checksumFlag, FUZ_rand(&lseed) & 1) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_contentSizeFlag, FUZ_rand(&lseed) & 1) );
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
DISPLAYLEVEL(5, "pledgedSrcSize : %u \n", (U32)pledgedSrcSize);
if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_refDictContent, FUZ_rand(&lseed) & 1) );
/* multi-threading parameters */
{ U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
U32 const nbThreads = MIN(nbThreadsCandidate, nbThreadsMax);
CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_nbThreads, nbThreads) );
if (nbThreads > 1) {
U32 const jobLog = FUZ_rand(&lseed) % (testLog+1);
CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_overlapSizeLog, FUZ_rand(&lseed) % 10) );
CHECK_Z( ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog)) );
}
}
if (FUZ_rand(&lseed) & 1) CHECK_Z (ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_forceMaxWindow, FUZ_rand(&lseed) & 1) );
if (FUZ_rand(&lseed) & 1) CHECK_Z (ZSTD_CCtxParam_setParameter(cctxParams, ZSTD_p_test, FUZ_rand(&lseed) & 1) );
/* Apply parameters */
CHECK_Z (ZSTD_CCtx_applyCCtxParams(zc, cctxParams) );
if (FUZ_rand(&lseed) & 1) {
CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
if (dict && dictSize) {
/* test that compression parameters are rejected (correctly) after loading a non-NULL dictionary */
size_t const setError = ZSTD_CCtx_applyCCtxParams(zc, cctxParams);
CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_applyCCtxParams should have failed");
} } else {
CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) );
}
} }
/* multi-segments compression test */
XXH64_reset(&xxhState, 0);
{ ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
for (cSize=0, totalTestSize=0 ; (totalTestSize < maxTestSize) ; ) {
/* compress random chunks into randomly sized dst buffers */
size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
ZSTD_EndDirective const flush = (FUZ_rand(&lseed) & 15) ? ZSTD_e_continue : ZSTD_e_flush;
ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
outBuff.size = outBuff.pos + dstBuffSize;
CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, flush) );
DISPLAYLEVEL(5, "compress consumed %u bytes (total : %u) \n",
(U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos));
XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
totalTestSize += inBuff.pos;
}
/* final frame epilogue */
{ size_t remainingToFlush = (size_t)(-1);
while (remainingToFlush) {
ZSTD_inBuffer inBuff = { NULL, 0, 0 };
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
outBuff.size = outBuff.pos + adjustedDstSize;
DISPLAYLEVEL(5, "End-flush into dst buffer of size %u \n", (U32)adjustedDstSize);
remainingToFlush = ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end);
CHECK(ZSTD_isError(remainingToFlush),
"ZSTD_compress_generic w/ ZSTD_e_end error : %s",
ZSTD_getErrorName(remainingToFlush) );
} }
crcOrig = XXH64_digest(&xxhState);
cSize = outBuff.pos;
DISPLAYLEVEL(5, "Frame completed : %u bytes \n", (U32)cSize);
}
/* multi - fragments decompression test */
if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
DISPLAYLEVEL(5, "resetting DCtx (dict:%08X) \n", (U32)(size_t)dict);
CHECK_Z( ZSTD_resetDStream(zd) );
} else {
DISPLAYLEVEL(5, "using dict of size %u \n", (U32)dictSize);
CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
}
{ size_t decompressionResult = 1;
ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
for (totalGenSize = 0 ; decompressionResult ; ) {
size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
inBuff.size = inBuff.pos + readCSrcSize;
outBuff.size = inBuff.pos + dstBuffSize;
DISPLAYLEVEL(5, "ZSTD_decompressStream input %u bytes (pos:%u/%u)\n",
(U32)readCSrcSize, (U32)inBuff.pos, (U32)cSize);
decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
DISPLAYLEVEL(5, "inBuff.pos = %u \n", (U32)readCSrcSize);
}
CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (U32)outBuff.pos, (U32)totalTestSize);
CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (U32)inBuff.pos, (U32)cSize);
{ U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
CHECK (crcDest!=crcOrig, "decompressed data corrupted");
} }
/*===== noisy/erroneous src decompression test =====*/
/* add some noise */
{ U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
} }
/* try decompression on noisy data */
CHECK_Z( ZSTD_initDStream(zd_noise) ); /* note : no dictionary */
{ ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
while (outBuff.pos < dstBufferSize) {
size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize);
outBuff.size = outBuff.pos + adjustedDstSize;
inBuff.size = inBuff.pos + adjustedCSrcSize;
{ size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
if (ZSTD_isError(decompressError)) break; /* error correctly detected */
/* Good so far, but no more progress possible */
if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
} } } }
DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
_cleanup:
ZSTD_freeCCtx(zc);
ZSTD_freeDStream(zd);
ZSTD_freeDStream(zd_noise);
ZSTD_freeCCtxParams(cctxParams);
free(cNoiseBuffer[0]);
free(cNoiseBuffer[1]);
free(cNoiseBuffer[2]);
free(cNoiseBuffer[3]);
free(cNoiseBuffer[4]);
free(copyBuffer);
free(cBuffer);
free(dstBuffer);
return result;
_output_error:
result = 1;
goto _cleanup;
}
/*-******************************************************* /*-*******************************************************
* Command line * Command line
@ -1514,7 +1817,7 @@ int FUZ_usage(const char* programName)
return 0; return 0;
} }
typedef enum { simple_api, mt_api, advanced_api } e_api; typedef enum { simple_api, mt_api, advanced_api, advanced_api_opaque } e_api;
int main(int argc, const char** argv) int main(int argc, const char** argv)
{ {
@ -1541,6 +1844,7 @@ int main(int argc, const char** argv)
if (!strcmp(argument, "--mt")) { selected_api=mt_api; continue; } if (!strcmp(argument, "--mt")) { selected_api=mt_api; continue; }
if (!strcmp(argument, "--newapi")) { selected_api=advanced_api; continue; } if (!strcmp(argument, "--newapi")) { selected_api=advanced_api; continue; }
if (!strcmp(argument, "--opaqueapi")) { selected_api=advanced_api_opaque; continue; }
if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; } if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
argument++; argument++;
@ -1656,6 +1960,9 @@ int main(int argc, const char** argv)
case advanced_api : case advanced_api :
result = fuzzerTests_newAPI(seed, nbTests, testNb, ((double)proba) / 100, bigTests); result = fuzzerTests_newAPI(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
break; break;
case advanced_api_opaque :
result = fuzzerTests_newAPI_opaque(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
break;
default : default :
assert(0); /* impossible */ assert(0); /* impossible */
} }