AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Requested Changes

Browse Source 
Add Comment
Simplify Interface (Remove resultSet)
Reorder Arguments
Remove customBench displayLevel
Reorder bench.h
Change benchFiles return type to match advanced
Rename stuff
This commit is contained in:
George Lu 2018-06-15 16:21:08 -04:00
parent 8522346322
commit 0d1ee22990
4 changed files with 188 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

231
programs/bench.c
View File

@ -110,13 +110,25 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
return r; \ return r; \
} }
/* error without displaying */
#define EXM_THROW_ND(errorNum, retType, ...) { \
retType r; \
memset(&r, 0, sizeof(retType)); \
DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
DEBUGOUTPUT("Error %i : ", errorNum); \
DEBUGOUTPUT(__VA_ARGS__); \
DEBUGOUTPUT(" \n"); \
r.error = errorNum; \
return r; \
}
/* ************************************* /* *************************************
* Benchmark Parameters * Benchmark Parameters
***************************************/ ***************************************/
#define BMK_LDM_PARAM_NOTSET 9999 #define BMK_LDM_PARAM_NOTSET 9999
BMK_advancedParams_t BMK_defaultAdvancedParams(void) { BMK_advancedParams_t BMK_initAdvancedParams(void) {
BMK_advancedParams_t res = { BMK_advancedParams_t res = {
BMK_both, /* mode */ BMK_both, /* mode */
BMK_timeMode, /* loopMode */ BMK_timeMode, /* loopMode */
@ -124,7 +136,6 @@ BMK_advancedParams_t BMK_defaultAdvancedParams(void) {
0, /* blockSize */ 0, /* blockSize */
0, /* nbWorkers */ 0, /* nbWorkers */
0, /* realTime */ 0, /* realTime */
1, /* separateFiles */
0, /* additionalParam */ 0, /* additionalParam */
0, /* ldmFlag */ 0, /* ldmFlag */
0, /* ldmMinMatch */ 0, /* ldmMinMatch */
@ -239,7 +250,7 @@ static size_t local_defaultCompress(
return out.pos; return out.pos;
} }
/* addiional argument is just the context */ /* additional argument is just the context */
static size_t local_defaultDecompress( static size_t local_defaultDecompress(
const void* srcBuffer, size_t srcSize, const void* srcBuffer, size_t srcSize,
void* dstBuffer, size_t dstSize, void* dstBuffer, size_t dstSize,
@ -269,14 +280,13 @@ static size_t local_defaultDecompress(
/* initFn will be measured once, bench fn will be measured x times */ /* initFn will be measured once, bench fn will be measured x times */
/* benchFn should return error value or out Size */ /* benchFn should return error value or out Size */
/* takes # of blocks and list of size & stuff for each. */ /* takes # of blocks and list of size & stuff for each. */
BMK_customReturn_t BMK_benchCustom( BMK_customReturn_t BMK_benchFunction(
const char* functionName, size_t blockCount, size_t blockCount,
const void* const * const srcBuffers, const size_t* srcSizes, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes,
void* const * const dstBuffers, const size_t* dstSizes, void* const * const dstBlockBuffers, const size_t* dstBlockCapacities,
size_t (*initFn)(void*), size_t (*benchFn)(const void*, size_t, void*, size_t, void*), size_t (*initFn)(void*), void* initPayload,
void* initPayload, void* benchPayload, size_t (*benchFn)(const void*, size_t, void*, size_t, void*), void* benchPayload,
unsigned mode, unsigned iter, unsigned mode, unsigned iter) {
int displayLevel) {
size_t srcSize = 0, dstSize = 0, ind = 0; size_t srcSize = 0, dstSize = 0, ind = 0;
unsigned toAdd = 1; unsigned toAdd = 1;
@ -287,24 +297,22 @@ BMK_customReturn_t BMK_benchCustom(
{ {
unsigned i; unsigned i;
for(i = 0; i < blockCount; i++) { for(i = 0; i < blockCount; i++) {
memset(dstBuffers[i], 0xE5, dstSizes[i]); /* warm up and erase result buffer */ memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */
} }
UTIL_sleepMilli(5); /* give processor time to other processes */ UTIL_sleepMilli(5); /* give processor time to other processes */
UTIL_waitForNextTick(); UTIL_waitForNextTick();
} }
/* display last 17 char's of functionName*/
if (strlen(functionName)>17) functionName += strlen(functionName)-17;
if(!iter) { if(!iter) {
if(mode == BMK_iterMode) { if(mode == BMK_iterMode) {
EXM_THROW(1, BMK_customReturn_t, "nbLoops must be nonzero \n"); EXM_THROW_ND(1, BMK_customReturn_t, "nbLoops must be nonzero \n");
} }
} }
for(ind = 0; ind < blockCount; ind++) { for(ind = 0; ind < blockCount; ind++) {
srcSize += srcSizes[ind]; srcSize += srcBlockSizes[ind];
} }
switch(mode) { switch(mode) {
@ -318,13 +326,13 @@ BMK_customReturn_t BMK_benchCustom(
unsigned i, j; unsigned i, j;
/* Overheat protection */ /* Overheat protection */
if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) { if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) {
DISPLAYLEVEL(2, "\rcooling down ... \r"); DEBUGOUTPUT("\rcooling down ... \r");
UTIL_sleep(COOLPERIOD_SEC); UTIL_sleep(COOLPERIOD_SEC);
coolTime = UTIL_getTime(); coolTime = UTIL_getTime();
} }
for(i = 0; i < blockCount; i++) { for(i = 0; i < blockCount; i++) {
memset(dstBuffers[i], 0xD6, dstSizes[i]); /* warm up and erase result buffer */ memset(dstBlockBuffers[i], 0xD6, dstBlockCapacities[i]); /* warm up and erase result buffer */
} }
clockStart = UTIL_getTime(); clockStart = UTIL_getTime();
@ -332,10 +340,10 @@ BMK_customReturn_t BMK_benchCustom(
for(i = 0; i < nbLoops; i++) { for(i = 0; i < nbLoops; i++) {
for(j = 0; j < blockCount; j++) { for(j = 0; j < blockCount; j++) {
size_t res = (*benchFn)(srcBuffers[j], srcSizes[j], dstBuffers[j], dstSizes[j], benchPayload); size_t res = (*benchFn)(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacities[j], benchPayload);
if(ZSTD_isError(res)) { if(ZSTD_isError(res)) {
EXM_THROW(2, BMK_customReturn_t, "%s() failed on block %u of size %u : %s \n", EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n",
functionName, j, (U32)dstSizes[j], ZSTD_getErrorName(res)); j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res));
} else if (toAdd) { } else if (toAdd) {
dstSize += res; dstSize += res;
} }
@ -362,10 +370,10 @@ BMK_customReturn_t BMK_benchCustom(
clockStart = UTIL_getTime(); clockStart = UTIL_getTime();
for(i = 0; i < iter; i++) { for(i = 0; i < iter; i++) {
for(j = 0; j < blockCount; j++) { for(j = 0; j < blockCount; j++) {
size_t res = (*benchFn)(srcBuffers[j], srcSizes[j], dstBuffers[j], dstSizes[j], benchPayload); size_t res = (*benchFn)(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacities[j], benchPayload);
if(ZSTD_isError(res)) { if(ZSTD_isError(res)) {
EXM_THROW(2, BMK_customReturn_t, "%s() failed on block %u of size %u : %s \n", EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n",
functionName, j, (U32)dstSizes[j], ZSTD_getErrorName(res)); j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res));
} else if(toAdd) { } else if(toAdd) {
dstSize += res; dstSize += res;
} }
@ -374,18 +382,18 @@ BMK_customReturn_t BMK_benchCustom(
} }
totalTime = UTIL_clockSpanNano(clockStart); totalTime = UTIL_clockSpanNano(clockStart);
if(!totalTime) { if(!totalTime) {
EXM_THROW(3, BMK_customReturn_t, "Cycle count (%u) too short to measure \n", iter); EXM_THROW_ND(3, BMK_customReturn_t, "Cycle count (%u) too short to measure \n", iter);
} else { } else {
fastest = totalTime / iter; fastest = totalTime / iter;
} }
break; break;
} }
default: default:
EXM_THROW(4, BMK_customReturn_t, "Unknown Mode \n"); EXM_THROW_ND(4, BMK_customReturn_t, "Unknown Mode \n");
} }
retval.error = 0; retval.error = 0;
retval.result.time = fastest; retval.result.nanoSecPerRun = fastest;
retval.result.size = dstSize; retval.result.sumOfReturn = dstSize;
return retval; return retval;
} }
@ -503,23 +511,23 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
cctxprep.adv = adv; cctxprep.adv = adv;
/* Compression */ /* Compression */
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize); DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
compressionResults = BMK_benchCustom("ZSTD_compress_generic", nbBlocks, compressionResults = BMK_benchFunction(nbBlocks,
srcPtrs, srcSizes, cPtrs, cSizes, srcPtrs, srcSizes, cPtrs, cSizes,
&local_initCCtx, &local_defaultCompress, &local_initCCtx, (void*)&cctxprep,
(void*)&cctxprep, (void*)(ctx), &local_defaultCompress, (void*)(ctx),
adv->loopMode, adv->nbSeconds, displayLevel); adv->loopMode, adv->nbSeconds);
if(compressionResults.error) { if(compressionResults.error) {
results.error = compressionResults.error; results.error = compressionResults.error;
return results; return results;
} }
results.result.cSize = compressionResults.result.size; results.result.cSize = compressionResults.result.sumOfReturn;
ratio = (double)srcSize / (double)results.result.cSize; ratio = (double)srcSize / (double)results.result.cSize;
markNb = (markNb+1) % NB_MARKS; markNb = (markNb+1) % NB_MARKS;
{ {
int const ratioAccuracy = (ratio < 10.) ? 3 : 2; int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
double const compressionSpeed = ((double)srcSize / compressionResults.result.time) * 1000; double const compressionSpeed = ((double)srcSize / compressionResults.result.nanoSecPerRun) * 1000;
int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
results.result.cSpeed = compressionSpeed * 1000000; results.result.cSpeed = compressionSpeed * 1000000;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r",
@ -536,11 +544,11 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
dctxprep.dctx = dctx; dctxprep.dctx = dctx;
dctxprep.dictBuffer = dictBuffer; dctxprep.dictBuffer = dictBuffer;
dctxprep.dictBufferSize = dictBufferSize; dctxprep.dictBufferSize = dictBufferSize;
decompressionResults = BMK_benchCustom("ZSTD_decompress_generic", nbBlocks, decompressionResults = BMK_benchFunction(nbBlocks,
(const void * const *)cPtrs, cSizes, resPtrs, resSizes, (const void * const *)cPtrs, cSizes, resPtrs, resSizes,
&local_initDCtx, &local_defaultDecompress, &local_initDCtx, (void*)&dctxprep,
(void*)&dctxprep, (void*)(dctx), &local_defaultDecompress, (void*)(dctx),
adv->loopMode, adv->nbSeconds, displayLevel); adv->loopMode, adv->nbSeconds);
if(decompressionResults.error) { if(decompressionResults.error) {
results.error = decompressionResults.error; results.error = decompressionResults.error;
@ -551,7 +559,7 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
{ int const ratioAccuracy = (ratio < 10.) ? 3 : 2; { int const ratioAccuracy = (ratio < 10.) ? 3 : 2;
double const compressionSpeed = results.result.cSpeed / 1000000; double const compressionSpeed = results.result.cSpeed / 1000000;
int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1;
double const decompressionSpeed = ((double)srcSize / decompressionResults.result.time) * 1000; double const decompressionSpeed = ((double)srcSize / decompressionResults.result.nanoSecPerRun) * 1000;
results.result.dSpeed = decompressionSpeed * 1000000; results.result.dSpeed = decompressionSpeed * 1000000;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r",
marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize,
@ -634,7 +642,7 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
int displayLevel, const char* displayName) { int displayLevel, const char* displayName) {
const BMK_advancedParams_t adv = BMK_defaultAdvancedParams(); const BMK_advancedParams_t adv = BMK_initAdvancedParams();
return BMK_benchMemAdvanced(srcBuffer, srcSize, return BMK_benchMemAdvanced(srcBuffer, srcSize,
fileSizes, nbFiles, fileSizes, nbFiles,
cLevel, comprParams, cLevel, comprParams,
@ -685,31 +693,20 @@ static size_t BMK_findMaxMem(U64 requiredMem)
return (size_t)(requiredMem); return (size_t)(requiredMem);
} }
ERROR_STRUCT(BMK_result_t*, BMK_returnPtr_t); static BMK_return_t BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
/* returns average stats over all range [cLevel, cLevelLast] */
static BMK_returnPtr_t BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
const size_t* fileSizes, unsigned nbFiles, const size_t* fileSizes, unsigned nbFiles,
const int cLevel, const int cLevelLast, const ZSTD_compressionParameters* comprParams, const int cLevel, const ZSTD_compressionParameters* comprParams,
const void* dictBuffer, size_t dictBufferSize, const void* dictBuffer, size_t dictBufferSize,
int displayLevel, const char* displayName, int displayLevel, const char* displayName,
BMK_advancedParams_t const * const adv) BMK_advancedParams_t const * const adv)
{ {
int l; BMK_return_t res;
BMK_result_t* res = (BMK_result_t*)malloc(sizeof(BMK_result_t) * (cLevelLast - cLevel + 1));
BMK_returnPtr_t ret;
const char* pch = strrchr(displayName, '\\'); /* Windows */ const char* pch = strrchr(displayName, '\\'); /* Windows */
ret.error = 0;
ret.result = res;
if (!pch) pch = strrchr(displayName, '/'); /* Linux */ if (!pch) pch = strrchr(displayName, '/'); /* Linux */
if (pch) displayName = pch+1; if (pch) displayName = pch+1;
if(res == NULL) {
EXM_THROW(12, BMK_returnPtr_t, "not enough memory\n");
}
if (adv->realTime) { if (adv->realTime) {
DISPLAYLEVEL(2, "Note : switching to real-time priority \n"); DISPLAYLEVEL(2, "Note : switching to real-time priority \n");
SET_REALTIME_PRIORITY; SET_REALTIME_PRIORITY;
@ -718,23 +715,14 @@ static BMK_returnPtr_t BMK_benchCLevel(const void* srcBuffer, size_t benchedSize
if (displayLevel == 1 && !adv->additionalParam) if (displayLevel == 1 && !adv->additionalParam)
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, adv->nbSeconds, (U32)(adv->blockSize>>10)); DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, adv->nbSeconds, (U32)(adv->blockSize>>10));
for (l=cLevel; l <= cLevelLast; l++) { res = BMK_benchMemCtxless(srcBuffer, benchedSize,
BMK_return_t rettmp; fileSizes, nbFiles,
if (l==0) continue; /* skip level 0 */ cLevel, comprParams,
rettmp = BMK_benchMemCtxless(srcBuffer, benchedSize, dictBuffer, dictBufferSize,
fileSizes, nbFiles, displayLevel, displayName,
l, comprParams, adv);
dictBuffer, dictBufferSize,
displayLevel, displayName,
adv);
if(rettmp.error) {
ret.error = rettmp.error;
return ret;
}
res[l-cLevel] = rettmp.result;
}
return ret; return res;
} }
@ -776,8 +764,8 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize,
return 0; return 0;
} }
static BMK_returnSet_t BMK_benchFileTable(const char* const * const fileNamesTable, unsigned const nbFiles, static BMK_return_t BMK_benchFileTable(const char* const * const fileNamesTable, unsigned const nbFiles,
const char* const dictFileName, int const cLevel, int const cLevelLast, const char* const dictFileName, int const cLevel,
const ZSTD_compressionParameters* const compressionParams, int displayLevel, const ZSTD_compressionParameters* const compressionParams, int displayLevel,
const BMK_advancedParams_t * const adv) const BMK_advancedParams_t * const adv)
{ {
@ -786,23 +774,20 @@ static BMK_returnSet_t BMK_benchFileTable(const char* const * const fileNamesTab
void* dictBuffer = NULL; void* dictBuffer = NULL;
size_t dictBufferSize = 0; size_t dictBufferSize = 0;
size_t* const fileSizes = (size_t*)calloc(nbFiles, sizeof(size_t)); size_t* const fileSizes = (size_t*)calloc(nbFiles, sizeof(size_t));
BMK_returnSet_t res; BMK_return_t res;
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
res.result.cLevel = cLevel; if (!fileSizes) EXM_THROW(12, BMK_return_t, "not enough memory for fileSizes");
res.result.cLevelLast = cLevelLast;
res.result.results = NULL;
if (!fileSizes) EXM_THROW(12, BMK_returnSet_t, "not enough memory for fileSizes");
/* Load dictionary */ /* Load dictionary */
if (dictFileName != NULL) { if (dictFileName != NULL) {
U64 const dictFileSize = UTIL_getFileSize(dictFileName); U64 const dictFileSize = UTIL_getFileSize(dictFileName);
if (dictFileSize > 64 MB) if (dictFileSize > 64 MB)
EXM_THROW(10, BMK_returnSet_t, "dictionary file %s too large", dictFileName); EXM_THROW(10, BMK_return_t, "dictionary file %s too large", dictFileName);
dictBufferSize = (size_t)dictFileSize; dictBufferSize = (size_t)dictFileSize;
dictBuffer = malloc(dictBufferSize); dictBuffer = malloc(dictBufferSize);
if (dictBuffer==NULL) if (dictBuffer==NULL)
EXM_THROW(11, BMK_returnSet_t, "not enough memory for dictionary (%u bytes)", EXM_THROW(11, BMK_return_t, "not enough memory for dictionary (%u bytes)",
(U32)dictBufferSize); (U32)dictBufferSize);
{ {
int errorCode = BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1, displayLevel); int errorCode = BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1, displayLevel);
@ -819,7 +804,7 @@ static BMK_returnSet_t BMK_benchFileTable(const char* const * const fileNamesTab
if (benchedSize < totalSizeToLoad) if (benchedSize < totalSizeToLoad)
DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20)); DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
srcBuffer = malloc(benchedSize); srcBuffer = malloc(benchedSize);
if (!srcBuffer) EXM_THROW(12, BMK_returnSet_t, "not enough memory"); if (!srcBuffer) EXM_THROW(12, BMK_return_t, "not enough memory");
/* Load input buffer */ /* Load input buffer */
{ {
@ -830,17 +815,18 @@ static BMK_returnSet_t BMK_benchFileTable(const char* const * const fileNamesTab
} }
} }
/* Bench */ /* Bench */
/*
if (adv->separateFiles) { if (adv->separateFiles) {
const BYTE* srcPtr = (const BYTE*)srcBuffer; const BYTE* srcPtr = (const BYTE*)srcBuffer;
U32 fileNb; U32 fileNb;
res.result.results = (BMK_result_t**)malloc(sizeof(BMK_result_t*) * nbFiles); res.result.results = (BMK_result_t**)malloc(sizeof(BMK_result_t*) * nbFiles);
res.result.nbFiles = nbFiles; res.result.nbFiles = nbFiles;
if(res.result.results == NULL) EXM_THROW(12, BMK_returnSet_t, "not enough memory"); if(res.result.results == NULL) EXM_THROW(12, BMK_return_t, "not enough memory");
for (fileNb=0; fileNb<nbFiles; fileNb++) { for (fileNb=0; fileNb<nbFiles; fileNb++) {
size_t const fileSize = fileSizes[fileNb]; size_t const fileSize = fileSizes[fileNb];
BMK_returnPtr_t errorOrPtr = BMK_benchCLevel(srcPtr, fileSize, BMK_return_t errorOrPtr = BMK_benchCLevel(srcPtr, fileSize,
fileSizes+fileNb, 1, fileSizes+fileNb, 1,
cLevel, cLevelLast, compressionParams, cLevel, compressionParams,
dictBuffer, dictBufferSize, dictBuffer, dictBufferSize,
displayLevel, fileNamesTable[fileNb], displayLevel, fileNamesTable[fileNb],
adv); adv);
@ -852,119 +838,82 @@ static BMK_returnSet_t BMK_benchFileTable(const char* const * const fileNamesTab
srcPtr += fileSize; srcPtr += fileSize;
} }
} else { } else */
{
char mfName[20] = {0}; char mfName[20] = {0};
res.result.nbFiles = 1;
snprintf (mfName, sizeof(mfName), " %u files", nbFiles); snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
{ {
BMK_returnPtr_t errorOrPtr;
const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0]; const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
res.result.results = (BMK_result_t**)malloc(sizeof(BMK_result_t*)); res = BMK_benchCLevel(srcBuffer, benchedSize,
errorOrPtr = BMK_benchCLevel(srcBuffer, benchedSize,
fileSizes, nbFiles, fileSizes, nbFiles,
cLevel, cLevelLast, compressionParams, cLevel, compressionParams,
dictBuffer, dictBufferSize, dictBuffer, dictBufferSize,
displayLevel, displayName, displayLevel, displayName,
adv); adv);
if(res.result.results == NULL) EXM_THROW(12, BMK_returnSet_t, "not enough memory");
if(errorOrPtr.error) {
res.error = errorOrPtr.error;
return res;
}
res.result.results[0] = errorOrPtr.result;
} } } }
/* clean up */ /* clean up */
free(srcBuffer); free(srcBuffer);
free(dictBuffer); free(dictBuffer);
free(fileSizes); free(fileSizes);
res.error = 0;
return res; return res;
} }
static BMK_returnSet_t BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, static BMK_return_t BMK_syntheticTest(int cLevel, double compressibility,
const ZSTD_compressionParameters* compressionParams, const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t * const adv) int displayLevel, const BMK_advancedParams_t * const adv)
{ {
char name[20] = {0}; char name[20] = {0};
size_t benchedSize = 10000000; size_t benchedSize = 10000000;
void* const srcBuffer = malloc(benchedSize); void* const srcBuffer = malloc(benchedSize);
BMK_returnSet_t res; BMK_return_t res;
BMK_returnPtr_t errPtr;
res.result.results = (BMK_result_t**)calloc(1,sizeof(BMK_result_t*));
res.result.nbFiles = 1;
res.result.cLevel = cLevel;
res.result.cLevelLast = cLevelLast;
/* Memory allocation */ /* Memory allocation */
if (!srcBuffer || !res.result.results) EXM_THROW(21, BMK_returnSet_t, "not enough memory"); if (!srcBuffer) EXM_THROW(21, BMK_return_t, "not enough memory");
/* Fill input buffer */ /* Fill input buffer */
RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0); RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
/* Bench */ /* Bench */
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
errPtr = BMK_benchCLevel(srcBuffer, benchedSize, res = BMK_benchCLevel(srcBuffer, benchedSize,
&benchedSize, 1, &benchedSize, 1,
cLevel, cLevelLast, compressionParams, cLevel, compressionParams,
NULL, 0, NULL, 0,
displayLevel, name, adv); displayLevel, name, adv);
if(errPtr.error) {
res.error = errPtr.error;
return res;
}
res.result.results[0] = errPtr.result;
/* clean up */ /* clean up */
free((void*)srcBuffer); free((void*)srcBuffer);
res.error = 0;
return res; return res;
} }
BMK_returnSet_t BMK_benchFilesAdvanced(const char** fileNamesTable, unsigned nbFiles, BMK_return_t BMK_benchFilesAdvanced(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName, const char* dictFileName,
int cLevel, int cLevelLast, int cLevel, const ZSTD_compressionParameters* compressionParams,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t * const adv) int displayLevel, const BMK_advancedParams_t * const adv)
{ {
double const compressibility = (double)g_compressibilityDefault / 100; double const compressibility = (double)g_compressibilityDefault / 100;
if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel(); /* if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
if (cLevelLast < cLevel) cLevelLast = cLevel; if (cLevelLast < cLevel) cLevelLast = cLevel;
if (cLevelLast > cLevel) if (cLevelLast > cLevel)
DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); */
if (nbFiles == 0) { if (nbFiles == 0) {
return BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams, displayLevel, adv); return BMK_syntheticTest(cLevel, compressibility, compressionParams, displayLevel, adv);
} }
else { else {
return BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, displayLevel, adv); return BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, compressionParams, displayLevel, adv);
} }
} }
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, BMK_return_t BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName, const char* dictFileName,
int cLevel, int cLevelLast, int cLevel, const ZSTD_compressionParameters* compressionParams,
const ZSTD_compressionParameters* compressionParams,
int displayLevel) { int displayLevel) {
const BMK_advancedParams_t adv = BMK_defaultAdvancedParams(); const BMK_advancedParams_t adv = BMK_initAdvancedParams();
return BMK_benchFilesAdvanced(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, displayLevel, &adv).error; return BMK_benchFilesAdvanced(fileNamesTable, nbFiles, dictFileName, cLevel, compressionParams, displayLevel, &adv);
}
/* errorable or just return? */
BMK_result_t BMK_getResult(BMK_resultSet_t resultSet, unsigned fileIdx, int cLevel) {
assert(resultSet.nbFiles > fileIdx);
assert(resultSet.cLevel <= cLevel && cLevel <= resultSet.cLevelLast);
return resultSet.results[fileIdx][cLevel - resultSet.cLevel];
}
void BMK_freeResultSet(BMK_resultSet_t src) {
unsigned i;
if(src.results == NULL) { return; }
for(i = 0; i < src.nbFiles; i++) {
free(src.results[i]);
}
free(src.results);
} }

150
programs/bench.h
View File

@ -19,17 +19,12 @@ extern "C" {
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
#include "zstd.h" /* ZSTD_compressionParameters */ #include "zstd.h" /* ZSTD_compressionParameters */
typedef enum { /* Creates a struct of type typeName with an int type .error field
BMK_timeMode = 0, * and a .result field of some baseType. Functions with return
BMK_iterMode = 1 * typeName pass a successful result with .error = 0 and .result
} BMK_loopMode_t; * with the intended result, while returning an error will result
* in .error != 0.
typedef enum { */
BMK_both = 0,
BMK_decodeOnly = 1,
BMK_compressOnly = 2
} BMK_mode_t;
#define ERROR_STRUCT(baseType, typeName) typedef struct { \ #define ERROR_STRUCT(baseType, typeName) typedef struct { \
int error; \ int error; \
baseType result; \ baseType result; \
@ -42,49 +37,50 @@ typedef struct {
} BMK_result_t; } BMK_result_t;
typedef struct { typedef struct {
int cLevel; size_t sumOfReturn; /* sum of return values */
int cLevelLast; U64 nanoSecPerRun; /* time per iteration */
unsigned nbFiles;
BMK_result_t** results;
} BMK_resultSet_t;
typedef struct {
size_t size;
U64 time;
} BMK_customResult_t; } BMK_customResult_t;
ERROR_STRUCT(BMK_result_t, BMK_return_t); ERROR_STRUCT(BMK_result_t, BMK_return_t);
ERROR_STRUCT(BMK_resultSet_t, BMK_returnSet_t);
ERROR_STRUCT(BMK_customResult_t, BMK_customReturn_t); ERROR_STRUCT(BMK_customResult_t, BMK_customReturn_t);
/* want all 0 to be default, but wb ldmBucketSizeLog/ldmHashEveryLog */ typedef enum {
BMK_timeMode = 0,
BMK_iterMode = 1
} BMK_loopMode_t;
typedef enum {
BMK_both = 0,
BMK_decodeOnly = 1,
BMK_compressOnly = 2
} BMK_mode_t;
typedef struct { typedef struct {
BMK_mode_t mode; /* 0: all, 1: compress only 2: decode only */ BMK_mode_t mode; /* 0: all, 1: compress only 2: decode only */
BMK_loopMode_t loopMode; /* if loopmode, then nbSeconds = nbLoops */ BMK_loopMode_t loopMode; /* if loopmode, then nbSeconds = nbLoops */
unsigned nbSeconds; /* default timing is in nbSeconds. If nbCycles != 0 then use that */ unsigned nbSeconds; /* default timing is in nbSeconds */
size_t blockSize; /* Maximum allowable size of a block*/ size_t blockSize; /* Maximum allowable size of a block*/
unsigned nbWorkers; /* multithreading */ unsigned nbWorkers; /* multithreading */
unsigned realTime; unsigned realTime; /* real time priority */
unsigned separateFiles; int additionalParam; /* used by python speed benchmark */
int additionalParam; unsigned ldmFlag; /* enables long distance matching */
unsigned ldmFlag; unsigned ldmMinMatch; /* below: parameters for long distance matching, see zstd.1.md for meaning */
unsigned ldmMinMatch; unsigned ldmHashLog;
unsigned ldmHashLog;
unsigned ldmBucketSizeLog; unsigned ldmBucketSizeLog;
unsigned ldmHashEveryLog; unsigned ldmHashEveryLog;
} BMK_advancedParams_t; } BMK_advancedParams_t;
/* returns default parameters used by nonAdvanced functions */ /* returns default parameters used by nonAdvanced functions */
BMK_advancedParams_t BMK_defaultAdvancedParams(void); BMK_advancedParams_t BMK_initAdvancedParams(void);
/* functionName - name of function /* called in cli */
* blockCount - number of blocks (size of srcBuffers, srcSizes, dstBuffers, dstSizes) /* fileNamesTable - name of files to benchmark
* initFn - (*initFn)(initPayload) is run once per benchmark * nbFiles - number of files (size of fileNamesTable)
* benchFn - (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstSizes[i], benchPayload) * dictFileName - name of dictionary file to load
* is run a variable number of times, specified by mode and iter args * cLevel - lowest compression level to benchmark
* mode - if 0, iter will be interpreted as the minimum number of seconds to run * cLevellast - highest compression level to benchmark (everything in the range [cLevel, cLevellast]) will be benchmarked
* iter - see mode * compressionParams - basic compression Parameters
* displayLevel - what gets printed * displayLevel - what gets printed
* 0 : no display; * 0 : no display;
* 1 : errors; * 1 : errors;
@ -92,16 +88,20 @@ BMK_advancedParams_t BMK_defaultAdvancedParams(void);
* 3 : + progression; * 3 : + progression;
* 4 : + information * 4 : + information
* return * return
* .error will give a nonzero value if any error has occured * .error will give a nonzero error value if an error has occured
* .result will contain the speed (B/s) and time per loop (ns) * .result - if .error = 0, .result will return the time taken to compression speed
* (.cSpeed), decompression speed (.dSpeed), and copmressed size (.cSize) of the original
* file
*/ */
BMK_customReturn_t BMK_benchCustom(const char* functionName, size_t blockCount, BMK_return_t BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
const void* const * const srcBuffers, const size_t* srcSizes, int cLevel, const ZSTD_compressionParameters* compressionParams,
void* const * const dstBuffers, const size_t* dstSizes, int displayLevel);
size_t (*initFn)(void*), size_t (*benchFn)(const void*, size_t, void*, size_t, void*),
void* initPayload, void* benchPayload, /* See benchFiles for normal parameter uses and return, see advancedParams_t for adv */
unsigned mode, unsigned iter, BMK_return_t BMK_benchFilesAdvanced(const char** fileNamesTable, unsigned nbFiles,
int displayLevel); const char* dictFileName,
int cLevel, const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* const adv);
/* basic benchmarking function, called in paramgrill ctx, dctx must be provided */ /* basic benchmarking function, called in paramgrill ctx, dctx must be provided */
/* srcBuffer - data source, expected to be valid compressed data if in Decode Only Mode /* srcBuffer - data source, expected to be valid compressed data if in Decode Only Mode
@ -112,8 +112,12 @@ BMK_customReturn_t BMK_benchCustom(const char* functionName, size_t blockCount,
* dictBufferSize - size of dictBuffer, 0 otherwise * dictBufferSize - size of dictBuffer, 0 otherwise
* ctx - Compression Context * ctx - Compression Context
* dctx - Decompression Context * dctx - Decompression Context
* diplayLevel - see BMK_benchCustom * diplayLevel - see BMK_benchFiles
* displayName - name used in display * displayName - name used by display
* return
* .error will give a nonzero value if an error has occured
* .result - if .error = 0, will give the same results as benchFiles
* but for the data stored in srcBuffer
*/ */
BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, unsigned nbFiles, const size_t* fileSizes, unsigned nbFiles,
@ -122,6 +126,7 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize,
ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx,
int displayLevel, const char* displayName); int displayLevel, const char* displayName);
/* See benchMem for normal parameter uses and return, see advancedParams_t for adv */
BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
const size_t* fileSizes, unsigned nbFiles, const size_t* fileSizes, unsigned nbFiles,
const int cLevel, const ZSTD_compressionParameters* comprParams, const int cLevel, const ZSTD_compressionParameters* comprParams,
@ -130,29 +135,34 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize,
int displayLevel, const char* displayName, int displayLevel, const char* displayName,
const BMK_advancedParams_t* adv); const BMK_advancedParams_t* adv);
/* called in cli */ /* This function benchmarks the running time two functions (function specifics described */
/* fileNamesTable - name of files to benchmark
* nbFiles - number of files (size of fileNamesTable) /* blockCount - number of blocks (size of srcBuffers, srcSizes, dstBuffers, dstCapacities)
* dictFileName - name of dictionary file to load * srcBuffers - an array of buffers to be operated on by benchFn
* cLevel - lowest compression level to benchmark * srcSizes - an array of the sizes of above buffers
* cLevellast - highest compression level to benchmark (everything in the range [cLevel, cLevellast]) will be benchmarked * dstBuffers - an array of buffers to be written into by benchFn
* compressionParams - basic compression Parameters * dstCapacities - an array of the capacities of above buffers.
* displayLevel - see BMK_benchCustom * initFn - (*initFn)(initPayload) is run once per benchmark
* benchFn - (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload)
* is run a variable number of times, specified by mode and iter args
* mode - if 0, iter will be interpreted as the minimum number of seconds to run
* iter - see mode
* return
* .error will give a nonzero value if ZSTD_isError() is nonzero for any of the return
* of the calls to initFn and benchFn, or if benchFunction errors internally
* .result - if .error = 0, then .result will contain the sum of all return values of
* benchFn on the first iteration through all of the blocks (.sumOfReturn) and also
* the time per run of benchFn (.nanoSecPerRun). For the former, this
* is generally intended to be used on functions which return the # of bytes written
* into dstBuffer, hence this value will be the total amount of bytes written to
* dstBuffer.
*/ */
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, BMK_customReturn_t BMK_benchFunction(size_t blockCount,
int cLevel, int cLevelLast, const ZSTD_compressionParameters* compressionParams, const void* const * const srcBuffers, const size_t* srcSizes,
int displayLevel); void* const * const dstBuffers, const size_t* dstCapacities,
size_t (*initFn)(void*), void* initPayload,
BMK_returnSet_t BMK_benchFilesAdvanced(const char** fileNamesTable, unsigned nbFiles, size_t (*benchFn)(const void*, size_t, void*, size_t, void*), void* benchPayload,
const char* dictFileName, unsigned mode, unsigned iter);
int cLevel, int cLevelLast,
const ZSTD_compressionParameters* compressionParams,
int displayLevel, const BMK_advancedParams_t* adv);
/* get data from resultSet */
/* when aggregated (separateFiles = 0), just be getResult(r,0,cl) */
BMK_result_t BMK_getResult(BMK_resultSet_t results, unsigned fileIdx, int cLevel);
void BMK_freeResultSet(BMK_resultSet_t src);
#endif /* BENCH_H_121279284357 */ #endif /* BENCH_H_121279284357 */

16
programs/zstdcli.c
View File

@ -398,7 +398,7 @@ int main(int argCount, const char* argv[])
setRealTimePrio = 0, setRealTimePrio = 0,
singleThread = 0, singleThread = 0,
ultra=0; ultra=0;
BMK_advancedParams_t adv = BMK_defaultAdvancedParams(); BMK_advancedParams_t adv = BMK_initAdvancedParams();
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
size_t blockSize = 0; size_t blockSize = 0;
zstd_operation_mode operation = zom_compress; zstd_operation_mode operation = zom_compress;
@ -802,7 +802,6 @@ int main(int argCount, const char* argv[])
/* Check if benchmark is selected */ /* Check if benchmark is selected */
if (operation==zom_bench) { if (operation==zom_bench) {
#ifndef ZSTD_NOBENCH #ifndef ZSTD_NOBENCH
adv.separateFiles = separateFiles;
adv.blockSize = blockSize; adv.blockSize = blockSize;
adv.nbWorkers = nbWorkers; adv.nbWorkers = nbWorkers;
adv.realTime = setRealTimePrio; adv.realTime = setRealTimePrio;
@ -816,7 +815,18 @@ int main(int argCount, const char* argv[])
if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) { if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) {
adv.ldmHashEveryLog = g_ldmHashEveryLog; adv.ldmHashEveryLog = g_ldmHashEveryLog;
} }
BMK_freeResultSet(BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, g_displayLevel, &adv).result);
for(; cLevel <= cLevelLast; cLevel++) {
if(separateFiles) {
unsigned i;
for(i = 0; i < filenameIdx; i++) {
BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, cLevel, &compressionParams, g_displayLevel, &adv);
}
} else {
BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &adv);
}
}
#else #else
(void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles;
#endif #endif

8
tests/fullbench.c
View File

@ -431,14 +431,16 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
/* benchmark loop */ /* benchmark loop */
{ {
BMK_customReturn_t r = BMK_benchCustom(benchName, 1, &src, &srcSize, (void * const * const)&dstBuff, &dstBuffSize, &local_nothing, benchFunction, BMK_customReturn_t r = BMK_benchFunction(1, &src, &srcSize,
NULL, buff2, BMK_timeMode, 1, 2); (void * const * const)&dstBuff, &dstBuffSize,
&local_nothing, NULL,
benchFunction, buff2, BMK_timeMode, 1);
if(r.error) { if(r.error) {
DISPLAY("ERROR %d ! ! \n", r.error); DISPLAY("ERROR %d ! ! \n", r.error);
exit(1); exit(1);
} }
DISPLAY("%2u#Speed: %f MB/s - Size: %f MB\n", benchNb, (double)srcSize / r.result.time * 1000, (double)r.result.size / 1000000); DISPLAY("%2u#Speed: %f MB/s - Size: %f MB - %s\n", benchNb, (double)srcSize / r.result.nanoSecPerRun * 1000, (double)r.result.sumOfReturn / 1000000, benchName);
} }
_cleanOut: _cleanOut: