AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #1206 from GeorgeLu97/FullBenchParam

Browse Source 
Fullbench Param
This commit is contained in:
Yann Collet 2018-07-02 13:33:53 -07:00 committed by GitHub
commit 1bf0784975
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 163 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

217
tests/fullbench.c
View File

@ -46,6 +46,8 @@
#define KNUTH 2654435761U
#define MAX_MEM (1984 MB)
#define DEFAULT_CLEVEL 1
#define COMPRESSIBILITY_DEFAULT 0.50
static const size_t g_sampleSize = 10000000;
@ -90,15 +92,59 @@ static size_t BMK_findMaxMem(U64 requiredMem)
return (size_t) requiredMem;
}
/*_*******************************************************
* Argument Parsing
*********************************************************/
#define ERROR_OUT(msg) { DISPLAY("%s \n", msg); exit(1); }
static unsigned readU32FromChar(const char** stringPtr)
{
const char errorMsg[] = "error: numeric value too large";
unsigned result = 0;
while ((**stringPtr >='0') && (**stringPtr <='9')) {
unsigned const max = (((unsigned)(-1)) / 10) - 1;
if (result > max) ERROR_OUT(errorMsg);
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
}
if ((**stringPtr=='K') || (**stringPtr=='M')) {
unsigned const maxK = ((unsigned)(-1)) >> 10;
if (result > maxK) ERROR_OUT(errorMsg);
result <<= 10;
if (**stringPtr=='M') {
if (result > maxK) ERROR_OUT(errorMsg);
result <<= 10;
}
(*stringPtr)++; /* skip `K` or `M` */
if (**stringPtr=='i') (*stringPtr)++;
if (**stringPtr=='B') (*stringPtr)++;
}
return result;
}
static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
{
size_t const comSize = strlen(longCommand);
int const result = !strncmp(*stringPtr, longCommand, comSize);
if (result) *stringPtr += comSize;
return result;
}
/*_*******************************************************
* Benchmark wrappers
*********************************************************/
static ZSTD_CCtx* g_zcc = NULL;
size_t local_ZSTD_compress(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
{
(void)buff2;
return ZSTD_compress(dst, dstSize, src, srcSize, 1);
ZSTD_parameters p;
ZSTD_frameParameters f = {1 /* contentSizeHeader*/, 0, 0};
p.fParams = f;
p.cParams = *(ZSTD_compressionParameters*)buff2;
return ZSTD_compress_advanced (g_zcc,dst, dstSize, src, srcSize, NULL ,0, p);
//return ZSTD_compress(dst, dstSize, src, srcSize, cLevel);
}
static size_t g_cSize = 0;
@ -132,8 +178,11 @@ size_t local_ZSTD_compressStream(const void* src, size_t srcSize, void* dst, siz
{
ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn;
(void)buff2;
ZSTD_initCStream(g_cstream, 1);
ZSTD_parameters p;
ZSTD_frameParameters f = {1 /* contentSizeHeader*/, 0, 0};
p.fParams = f;
p.cParams = *(ZSTD_compressionParameters*)buff2;
ZSTD_initCStream_advanced(g_cstream, NULL, 0, p, ZSTD_CONTENTSIZE_UNKNOWN);
buffOut.dst = dst;
buffOut.size = dstCapacity;
buffOut.pos = 0;
@ -150,7 +199,6 @@ static size_t local_ZSTD_compress_generic_end(const void* src, size_t srcSize, v
ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn;
(void)buff2;
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1);
buffOut.dst = dst;
buffOut.size = dstCapacity;
buffOut.pos = 0;
@ -166,7 +214,6 @@ static size_t local_ZSTD_compress_generic_continue(const void* src, size_t srcSi
ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn;
(void)buff2;
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1);
buffOut.dst = dst;
buffOut.size = dstCapacity;
buffOut.pos = 0;
@ -183,7 +230,6 @@ static size_t local_ZSTD_compress_generic_T2_end(const void* src, size_t srcSize
ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn;
(void)buff2;
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_nbWorkers, 2);
buffOut.dst = dst;
buffOut.size = dstCapacity;
@ -200,7 +246,6 @@ static size_t local_ZSTD_compress_generic_T2_continue(const void* src, size_t sr
ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn;
(void)buff2;
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, 1);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_nbWorkers, 2);
buffOut.dst = dst;
buffOut.size = dstCapacity;
@ -230,13 +275,14 @@ static size_t local_ZSTD_decompressStream(const void* src, size_t srcSize, void*
return buffOut.pos;
}
static ZSTD_CCtx* g_zcc = NULL;
#ifndef ZSTD_DLL_IMPORT
size_t local_ZSTD_compressContinue(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
{
(void)buff2;
ZSTD_compressBegin(g_zcc, 1 /* compressionLevel */);
ZSTD_parameters p;
ZSTD_frameParameters f = {1 /* contentSizeHeader*/, 0, 0};
p.fParams = f;
p.cParams = *(ZSTD_compressionParameters*)buff2;
ZSTD_compressBegin_advanced(g_zcc, NULL, 0, p, srcSize);
return ZSTD_compressEnd(g_zcc, dst, dstCapacity, src, srcSize);
}
@ -244,10 +290,13 @@ size_t local_ZSTD_compressContinue(const void* src, size_t srcSize, void* dst, s
size_t local_ZSTD_compressContinue_extDict(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* buff2)
{
BYTE firstBlockBuf[FIRST_BLOCK_SIZE];
(void)buff2;
ZSTD_parameters p;
ZSTD_frameParameters f = {1 , 0, 0};
p.fParams = f;
p.cParams = *(ZSTD_compressionParameters*)buff2;
ZSTD_compressBegin_advanced(g_zcc, NULL, 0, p, srcSize);
memcpy(firstBlockBuf, src, FIRST_BLOCK_SIZE);
ZSTD_compressBegin(g_zcc, 1);
{ size_t const compressResult = ZSTD_compressContinue(g_zcc, dst, dstCapacity, firstBlockBuf, FIRST_BLOCK_SIZE);
if (ZSTD_isError(compressResult)) { DISPLAY("local_ZSTD_compressContinue_extDict error : %s\n", ZSTD_getErrorName(compressResult)); return compressResult; }
@ -284,11 +333,11 @@ size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize, void* dst,
/*_*******************************************************
* Bench functions
*********************************************************/
static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams)
{
BYTE* dstBuff;
size_t const dstBuffSize = ZSTD_compressBound(srcSize);
void* buff2;
void* buff2, *buff1;
const char* benchName;
BMK_benchFn_t benchFunction;
BMK_customReturn_t r;
@ -298,14 +347,14 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
switch(benchNb)
{
case 1:
benchFunction = local_ZSTD_compress; benchName = "compress(1)";
benchFunction = local_ZSTD_compress; benchName = "compress";
break;
case 2:
benchFunction = local_ZSTD_decompress; benchName = "decompress";
break;
#ifndef ZSTD_DLL_IMPORT
case 11:
benchFunction = local_ZSTD_compressContinue; benchName = "compressContinue(1)";
benchFunction = local_ZSTD_compressContinue; benchName = "compressContinue";
break;
case 12:
benchFunction = local_ZSTD_compressContinue_extDict; benchName = "compressContinue_extDict";
@ -321,7 +370,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
break;
#endif
case 41:
benchFunction = local_ZSTD_compressStream; benchName = "compressStream(1)";
benchFunction = local_ZSTD_compressStream; benchName = "compressStream";
break;
case 42:
benchFunction = local_ZSTD_decompressStream; benchName = "decompressStream";
@ -350,26 +399,59 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
free(dstBuff); free(buff2);
return 12;
}
buff1 = buff2;
if (g_zcc==NULL) g_zcc = ZSTD_createCCtx();
if (g_zdc==NULL) g_zdc = ZSTD_createDCtx();
if (g_cstream==NULL) g_cstream = ZSTD_createCStream();
if (g_dstream==NULL) g_dstream = ZSTD_createDStream();
/* DISPLAY("params: cLevel %d, wlog %d hlog %d clog %d slog %d slen %d tlen %d strat %d \n"
, cLevel, cparams->windowLog, cparams->hashLog, cparams->chainLog, cparams->searchLog,
cparams->searchLength, cparams->targetLength, cparams->strategy);*/
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_windowLog, cparams->windowLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_hashLog, cparams->hashLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_chainLog, cparams->chainLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_searchLog, cparams->searchLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_minMatch, cparams->searchLength);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_targetLength, cparams->targetLength);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_p_compressionStrategy, cparams->strategy);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_windowLog, cparams->windowLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_hashLog, cparams->hashLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_chainLog, cparams->chainLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_searchLog, cparams->searchLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_minMatch, cparams->searchLength);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_targetLength, cparams->targetLength);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_p_compressionStrategy, cparams->strategy);
/* Preparation */
switch(benchNb)
{
case 1:
buff2 = (void*)cparams;
break;
case 2:
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel);
break;
#ifndef ZSTD_DLL_IMPORT
case 11:
buff2 = (void*)cparams;
break;
case 12:
buff2 = (void*)cparams;
break;
case 13 :
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel);
break;
case 31: /* ZSTD_decodeLiteralsBlock */
{ blockProperties_t bp;
ZSTD_frameHeader zfp;
size_t frameHeaderSize, skippedSize;
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
ZSTD_getcBlockSize(dstBuff+frameHeaderSize, dstBuffSize, &bp); /* Get 1st block type */
@ -389,8 +471,8 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
const BYTE* ip = dstBuff;
const BYTE* iend;
size_t frameHeaderSize, cBlockSize;
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); /* it would be better to use direct block compression here */
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel); /* it would be better to use direct block compression here */
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_frameHeaderSize_min);
if (frameHeaderSize==0) frameHeaderSize = ZSTD_frameHeaderSize_min;
ip += frameHeaderSize; /* Skip frame Header */
@ -412,8 +494,11 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
case 31:
goto _cleanOut;
#endif
case 41 :
buff2 = (void*)cparams;
break;
case 42 :
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel);
break;
/* test functions */
@ -442,8 +527,8 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
}
_cleanOut:
free(buff1);
free(dstBuff);
free(buff2);
ZSTD_freeCCtx(g_zcc); g_zcc=NULL;
ZSTD_freeDCtx(g_zdc); g_zdc=NULL;
ZSTD_freeCStream(g_cstream); g_cstream=NULL;
@ -452,7 +537,7 @@ _cleanOut:
}
static int benchSample(U32 benchNb)
static int benchSample(U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams)
{
size_t const benchedSize = g_sampleSize;
const char* name = "Sample 10MiB";
@ -468,16 +553,16 @@ static int benchSample(U32 benchNb)
DISPLAY("\r%79s\r", "");
DISPLAY(" %s : \n", name);
if (benchNb)
benchMem(origBuff, benchedSize, benchNb);
benchMem(origBuff, benchedSize, benchNb, cLevel, cparams);
else
for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb);
for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb, cLevel, cparams);
free(origBuff);
return 0;
}
static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchNb)
static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams)
{
/* Loop for each file */
int fileIdx;
@ -522,9 +607,9 @@ static int benchFiles(const char** fileNamesTable, const int nbFiles, U32 benchN
DISPLAY("\r%79s\r", "");
DISPLAY(" %s : \n", inFileName);
if (benchNb)
benchMem(origBuff, benchedSize, benchNb);
benchMem(origBuff, benchedSize, benchNb, cLevel, cparams);
else
for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb);
for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb, cLevel, cparams);
free(origBuff);
}
@ -549,6 +634,8 @@ static int usage_advanced(const char* exename)
DISPLAY( " -b# : test only function # \n");
DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS);
DISPLAY( " -P# : sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100);
DISPLAY( " -l# : benchmark functions at that compression level (default : %i)\n", DEFAULT_CLEVEL);
DISPLAY( " --zstd : custom parameter selection. Format same as zstdcli \n");
return 0;
}
@ -565,6 +652,8 @@ int main(int argc, const char** argv)
const char* exename = argv[0];
const char* input_filename = NULL;
U32 benchNb = 0, main_pause = 0;
int cLevel = DEFAULT_CLEVEL;
ZSTD_compressionParameters cparams = ZSTD_getCParams(cLevel, 0, 0);
DISPLAY(WELCOME_MESSAGE);
if (argc<1) return badusage(exename);
@ -573,11 +662,29 @@ int main(int argc, const char** argv)
const char* argument = argv[i];
assert(argument != NULL);
/* Commands (note : aggregated commands are allowed) */
if (argument[0]=='-') {
if (longCommandWArg(&argument, "--zstd=")) {
for ( ; ;) {
if (longCommandWArg(&argument, "windowLog=") || longCommandWArg(&argument, "wlog=")) { cparams.windowLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { cparams.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { cparams.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { cparams.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { cparams.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { cparams.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { cparams.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = (int)readU32FromChar(&argument); cparams = ZSTD_getCParams(cLevel, 0, 0); if (argument[0]==',') { argument++; continue; } else break; }
DISPLAY("invalid compression parameter \n");
return 1;
}
while (argument[1]!=0) {
argument++;
if (argument[0] != 0) {
DISPLAY("invalid --zstd= format\n");
return 1; // check the end of string
} else {
continue;
}
} else if (argument[0]=='-') { /* Commands (note : aggregated commands are allowed) */
argument++;
while (argument[0]!=0) {
switch(argument[0])
{
@ -590,34 +697,34 @@ int main(int argc, const char** argv)
/* Select specific algorithm to bench */
case 'b':
benchNb = 0;
while ((argument[1]>= '0') && (argument[1]<= '9')) {
benchNb *= 10;
benchNb += argument[1] - '0';
{
argument++;
benchNb = readU32FromChar(&argument);
break;
}
break;
/* Modify Nb Iterations */
case 'i':
if ((argument[1] >='0') && (argument[1] <='9')) {
int iters = argument[1] - '0';
BMK_SetNbIterations(iters);
{
argument++;
BMK_SetNbIterations((int)readU32FromChar(&argument));
}
break;
/* Select compressibility of synthetic sample */
case 'P':
{ U32 proba32 = 0;
while ((argument[1]>= '0') && (argument[1]<= '9')) {
proba32 *= 10;
proba32 += argument[1] - '0';
argument++;
}
g_compressibility = (double)proba32 / 100.;
{ argument++;
g_compressibility = (double)readU32FromChar(&argument) / 100.;
}
break;
case 'l':
{ argument++;
cLevel = readU32FromChar(&argument);
cparams = ZSTD_getCParams(cLevel, 0, 0);
}
break;
/* Unknown command */
default : return badusage(exename);
@ -630,10 +737,12 @@ int main(int argc, const char** argv)
if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
}
if (filenamesStart==0) /* no input file */
result = benchSample(benchNb);
result = benchSample(benchNb, cLevel, &cparams);
else
result = benchFiles(argv+filenamesStart, argc-filenamesStart, benchNb);
result = benchFiles(argv+filenamesStart, argc-filenamesStart, benchNb, cLevel, &cparams);
if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; }