IO and bench : ZSTD_NEWAPI is the only remaining code path
removed the other 2 code paths (single thread, and ZSTDMT ones) keeping only the new advanced API, for easier code coverage. It shall also fix identified issue with Visual Studio which doesn't have ZSTD_NEWAPI defined.
This commit is contained in:
parent
9d377c6e88
commit
d3b9547aa4
@ -37,8 +37,7 @@ endif
|
||||
|
||||
CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
|
||||
-I$(ZSTDDIR)/dictBuilder \
|
||||
-DXXH_NAMESPACE=ZSTD_ \
|
||||
-DZSTD_NEWAPI
|
||||
-DXXH_NAMESPACE=ZSTD_
|
||||
CFLAGS ?= -O3
|
||||
DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
||||
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
|
||||
|
@ -287,7 +287,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
U64 const clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
|
||||
U32 nbLoops = 0;
|
||||
ZSTD_CDict* cdict = NULL;
|
||||
#ifdef ZSTD_NEWAPI
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbThreads, g_nbThreads);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag);
|
||||
@ -306,25 +305,10 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength);
|
||||
ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy);
|
||||
ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize);
|
||||
#else
|
||||
size_t const avgSize = MIN(blockSize, (srcSize / nbFiles));
|
||||
ZSTD_parameters zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
|
||||
ZSTD_customMem const cmem = { NULL, NULL, NULL };
|
||||
if (comprParams->windowLog) zparams.cParams.windowLog = comprParams->windowLog;
|
||||
if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog;
|
||||
if (comprParams->hashLog) zparams.cParams.hashLog = comprParams->hashLog;
|
||||
if (comprParams->searchLog) zparams.cParams.searchLog = comprParams->searchLog;
|
||||
if (comprParams->searchLength) zparams.cParams.searchLength = comprParams->searchLength;
|
||||
if (comprParams->targetLength) zparams.cParams.targetLength = comprParams->targetLength;
|
||||
if (comprParams->strategy) zparams.cParams.strategy = comprParams->strategy;
|
||||
cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dm_auto, zparams.cParams, cmem);
|
||||
if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
|
||||
#endif
|
||||
do {
|
||||
U32 blockNb;
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
|
||||
size_t rSize;
|
||||
#ifdef ZSTD_NEWAPI
|
||||
ZSTD_outBuffer out = { blockTable[blockNb].cPtr, blockTable[blockNb].cRoom, 0 };
|
||||
ZSTD_inBuffer in = { blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize, 0 };
|
||||
size_t cError = 1;
|
||||
@ -336,29 +320,6 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_getErrorName(cError));
|
||||
}
|
||||
rSize = out.pos;
|
||||
#else /* ! ZSTD_NEWAPI */
|
||||
if (dictBufferSize) {
|
||||
rSize = ZSTD_compress_usingCDict(ctx,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
|
||||
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
|
||||
cdict);
|
||||
} else {
|
||||
# ifdef ZSTD_MULTITHREAD /* note : limitation : MT single-pass does not support compression with dictionary */
|
||||
rSize = ZSTDMT_compressCCtx(mtctx,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
|
||||
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
|
||||
cLevel);
|
||||
# else
|
||||
rSize = ZSTD_compress_advanced (ctx,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
|
||||
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
|
||||
NULL, 0, zparams);
|
||||
# endif
|
||||
}
|
||||
if (ZSTD_isError(rSize))
|
||||
EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s",
|
||||
ZSTD_getErrorName(rSize));
|
||||
#endif /* ZSTD_NEWAPI */
|
||||
blockTable[blockNb].cSize = rSize;
|
||||
}
|
||||
nbLoops++;
|
||||
|
@ -393,11 +393,7 @@ typedef struct {
|
||||
size_t srcBufferSize;
|
||||
void* dstBuffer;
|
||||
size_t dstBufferSize;
|
||||
#if !defined(ZSTD_NEWAPI) && defined(ZSTD_MULTITHREAD)
|
||||
ZSTDMT_CCtx* cctx;
|
||||
#else
|
||||
ZSTD_CStream* cctx;
|
||||
#endif
|
||||
} cRess_t;
|
||||
|
||||
static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
|
||||
@ -406,24 +402,9 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
|
||||
cRess_t ress;
|
||||
memset(&ress, 0, sizeof(ress));
|
||||
|
||||
#ifdef ZSTD_NEWAPI
|
||||
ress.cctx = ZSTD_createCCtx();
|
||||
if (ress.cctx == NULL)
|
||||
EXM_THROW(30, "allocation error : can't create ZSTD_CCtx");
|
||||
#elif defined(ZSTD_MULTITHREAD)
|
||||
ress.cctx = ZSTDMT_createCCtx(g_nbThreads);
|
||||
if (ress.cctx == NULL)
|
||||
EXM_THROW(30, "allocation error : can't create ZSTDMT_CCtx");
|
||||
if ((cLevel==ZSTD_maxCLevel()) && (g_overlapLog==FIO_OVERLAP_LOG_NOTSET))
|
||||
/* use complete window for overlap */
|
||||
ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, 9);
|
||||
if (g_overlapLog != FIO_OVERLAP_LOG_NOTSET)
|
||||
ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_overlapSectionLog, g_overlapLog);
|
||||
#else
|
||||
ress.cctx = ZSTD_createCStream();
|
||||
if (ress.cctx == NULL)
|
||||
EXM_THROW(30, "allocation error : can't create ZSTD_CStream");
|
||||
#endif
|
||||
ress.srcBufferSize = ZSTD_CStreamInSize();
|
||||
ress.srcBuffer = malloc(ress.srcBufferSize);
|
||||
ress.dstBufferSize = ZSTD_CStreamOutSize();
|
||||
@ -437,68 +418,38 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
|
||||
if (dictFileName && (dictBuffer==NULL))
|
||||
EXM_THROW(32, "allocation error : can't create dictBuffer");
|
||||
|
||||
#ifdef ZSTD_NEWAPI
|
||||
{ /* frame parameters */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
|
||||
/* compression level */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, cLevel) );
|
||||
/* long distance matching */
|
||||
CHECK( ZSTD_CCtx_setParameter(
|
||||
ress.cctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashLog, g_ldmHashLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch) );
|
||||
if (g_ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) {
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmBucketSizeLog, g_ldmBucketSizeLog) );
|
||||
}
|
||||
if (g_ldmHashEveryLog != FIO_LDM_PARAM_NOTSET) {
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog) );
|
||||
}
|
||||
/* compression parameters */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_windowLog, comprParams->windowLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams->chainLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams->hashLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams->searchLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams->searchLength) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams->targetLength) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams->strategy) );
|
||||
/* multi-threading */
|
||||
DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads);
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
|
||||
/* dictionary */
|
||||
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just to load dictionary with good compression parameters */
|
||||
CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
|
||||
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_contentSizeFlag, 1) ); /* always enable content size, when available (note: supposed to be default anyway) */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_dictIDFlag, g_dictIDFlag) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_checksumFlag, g_checksumFlag) );
|
||||
/* compression level */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionLevel, cLevel) );
|
||||
/* long distance matching */
|
||||
CHECK( ZSTD_CCtx_setParameter(
|
||||
ress.cctx, ZSTD_p_enableLongDistanceMatching, g_ldmFlag) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashLog, g_ldmHashLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmMinMatch, g_ldmMinMatch) );
|
||||
if (g_ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) {
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmBucketSizeLog, g_ldmBucketSizeLog) );
|
||||
}
|
||||
#elif defined(ZSTD_MULTITHREAD)
|
||||
{ ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
|
||||
params.fParams.checksumFlag = g_checksumFlag;
|
||||
params.fParams.noDictIDFlag = !g_dictIDFlag;
|
||||
if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog;
|
||||
if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog;
|
||||
if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog;
|
||||
if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog;
|
||||
if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
|
||||
if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
|
||||
if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy;
|
||||
CHECK( ZSTDMT_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) );
|
||||
ZSTDMT_setMTCtxParameter(ress.cctx, ZSTDMT_p_sectionSize, g_blockSize);
|
||||
if (g_ldmHashEveryLog != FIO_LDM_PARAM_NOTSET) {
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_ldmHashEveryLog, g_ldmHashEveryLog) );
|
||||
}
|
||||
#else
|
||||
{ ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize);
|
||||
params.fParams.checksumFlag = g_checksumFlag;
|
||||
params.fParams.noDictIDFlag = !g_dictIDFlag;
|
||||
if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog;
|
||||
if (comprParams->chainLog) params.cParams.chainLog = comprParams->chainLog;
|
||||
if (comprParams->hashLog) params.cParams.hashLog = comprParams->hashLog;
|
||||
if (comprParams->searchLog) params.cParams.searchLog = comprParams->searchLog;
|
||||
if (comprParams->searchLength) params.cParams.searchLength = comprParams->searchLength;
|
||||
if (comprParams->targetLength) params.cParams.targetLength = comprParams->targetLength;
|
||||
if (comprParams->strategy) params.cParams.strategy = (ZSTD_strategy) comprParams->strategy;
|
||||
CHECK( ZSTD_initCStream_advanced(ress.cctx, dictBuffer, dictBuffSize, params, srcSize) );
|
||||
}
|
||||
#endif
|
||||
/* compression parameters */
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_windowLog, comprParams->windowLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_chainLog, comprParams->chainLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_hashLog, comprParams->hashLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_searchLog, comprParams->searchLog) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_minMatch, comprParams->searchLength) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_targetLength, comprParams->targetLength) );
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_compressionStrategy, (U32)comprParams->strategy) );
|
||||
/* multi-threading */
|
||||
DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads);
|
||||
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
|
||||
/* dictionary */
|
||||
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* just to load dictionary with good compression parameters */
|
||||
CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
|
||||
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reset */
|
||||
|
||||
free(dictBuffer);
|
||||
}
|
||||
|
||||
@ -509,11 +460,7 @@ static void FIO_freeCResources(cRess_t ress)
|
||||
{
|
||||
free(ress.srcBuffer);
|
||||
free(ress.dstBuffer);
|
||||
#if !defined(ZSTD_NEWAPI) && defined(ZSTD_MULTITHREAD)
|
||||
ZSTDMT_freeCCtx(ress.cctx);
|
||||
#else
|
||||
ZSTD_freeCStream(ress.cctx); /* never fails */
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@ -814,14 +761,8 @@ static int FIO_compressFilename_internal(cRess_t ress,
|
||||
}
|
||||
|
||||
/* init */
|
||||
#ifdef ZSTD_NEWAPI
|
||||
if (fileSize!=UTIL_FILESIZE_UNKNOWN) /* when src is stdin, fileSize==0, but is effectively unknown */
|
||||
if (fileSize != UTIL_FILESIZE_UNKNOWN)
|
||||
ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize);
|
||||
#elif defined(ZSTD_MULTITHREAD)
|
||||
CHECK( ZSTDMT_resetCStream(ress.cctx, (fileSize==UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize) );
|
||||
#else
|
||||
CHECK( ZSTD_resetCStream(ress.cctx, (fileSize==UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize) );
|
||||
#endif
|
||||
|
||||
/* Main compression loop */
|
||||
while (1) {
|
||||
@ -833,14 +774,8 @@ static int FIO_compressFilename_internal(cRess_t ress,
|
||||
|
||||
while (inBuff.pos != inBuff.size) {
|
||||
ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
|
||||
#ifdef ZSTD_NEWAPI
|
||||
CHECK( ZSTD_compress_generic(ress.cctx,
|
||||
&outBuff, &inBuff, ZSTD_e_continue) );
|
||||
#elif defined(ZSTD_MULTITHREAD)
|
||||
CHECK( ZSTDMT_compressStream(ress.cctx, &outBuff, &inBuff) );
|
||||
#else
|
||||
CHECK( ZSTD_compressStream(ress.cctx, &outBuff, &inBuff) );
|
||||
#endif
|
||||
|
||||
/* Write compressed stream */
|
||||
if (outBuff.pos) {
|
||||
@ -871,21 +806,15 @@ static int FIO_compressFilename_internal(cRess_t ress,
|
||||
{ size_t result = 1;
|
||||
while (result != 0) {
|
||||
ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
|
||||
#ifdef ZSTD_NEWAPI
|
||||
ZSTD_inBuffer inBuff = { NULL, 0, 0 };
|
||||
result = ZSTD_compress_generic(ress.cctx,
|
||||
&outBuff, &inBuff, ZSTD_e_end);
|
||||
#elif defined(ZSTD_MULTITHREAD)
|
||||
result = ZSTDMT_endStream(ress.cctx, &outBuff);
|
||||
#else
|
||||
result = ZSTD_endStream(ress.cctx, &outBuff);
|
||||
#endif
|
||||
if (ZSTD_isError(result)) {
|
||||
EXM_THROW(26, "Compression error during frame end : %s",
|
||||
ZSTD_getErrorName(result));
|
||||
}
|
||||
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
|
||||
if (sizeCheck!=outBuff.pos)
|
||||
if (sizeCheck != outBuff.pos)
|
||||
EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName);
|
||||
}
|
||||
compressedfilesize += outBuff.pos;
|
||||
|
Loading…
Reference in New Issue
Block a user