AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #1623 from facebook/fullbench

Browse Source 
fullbench minor improvements
This commit is contained in:
Yann Collet 2019-05-31 14:40:19 -07:00 committed by GitHub
commit b13a9207f9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 126 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
lib/compress/zstd_compress.c
View File

@ -930,12 +930,12 @@ size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset)
@return : 0, or an error code if one value is beyond authorized range */
size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
{
BOUNDCHECK(ZSTD_c_windowLog, cParams.windowLog);
BOUNDCHECK(ZSTD_c_chainLog, cParams.chainLog);
BOUNDCHECK(ZSTD_c_hashLog, cParams.hashLog);
BOUNDCHECK(ZSTD_c_searchLog, cParams.searchLog);
BOUNDCHECK(ZSTD_c_minMatch, cParams.minMatch);
BOUNDCHECK(ZSTD_c_targetLength,cParams.targetLength);
BOUNDCHECK(ZSTD_c_windowLog, (int)cParams.windowLog);
BOUNDCHECK(ZSTD_c_chainLog, (int)cParams.chainLog);
BOUNDCHECK(ZSTD_c_hashLog, (int)cParams.hashLog);
BOUNDCHECK(ZSTD_c_searchLog, (int)cParams.searchLog);
BOUNDCHECK(ZSTD_c_minMatch, (int)cParams.minMatch);
BOUNDCHECK(ZSTD_c_targetLength,(int)cParams.targetLength);
BOUNDCHECK(ZSTD_c_strategy, cParams.strategy);
return 0;
}
@ -951,7 +951,7 @@ ZSTD_clampCParams(ZSTD_compressionParameters cParams)
if ((int)val<bounds.lowerBound) val=(type)bounds.lowerBound; \
else if ((int)val>bounds.upperBound) val=(type)bounds.upperBound; \
}
# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, int)
# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, unsigned)
CLAMP(ZSTD_c_windowLog, cParams.windowLog);
CLAMP(ZSTD_c_chainLog, cParams.chainLog);
CLAMP(ZSTD_c_hashLog, cParams.hashLog);

194
tests/fullbench.c
View File

@ -15,7 +15,7 @@
#include "util.h" /* Compiler options, UTIL_GetFileSize */
#include <stdlib.h> /* malloc */
#include <stdio.h> /* fprintf, fopen, ftello64 */
#include <assert.h> /* assert */
#include <assert.h>
#include "timefn.h" /* UTIL_clockSpanNano, UTIL_getTime */
#include "mem.h" /* U32 */
@ -31,8 +31,8 @@
#include "zstd.h" /* ZSTD_versionString */
#include "util.h" /* time functions */
#include "datagen.h"
#include "benchfn.h" /* CustomBench*/
#include "benchzstd.h" /* MB_UNIT */
#include "benchfn.h" /* CustomBench */
#include "benchzstd.h" /* MB_UNIT */
/*_************************************
@ -51,7 +51,7 @@
#define DEFAULT_CLEVEL 1
#define COMPRESSIBILITY_DEFAULT 0.50
static const size_t g_sampleSize = 10000000;
static const size_t kSampleSizeDefault = 10000000;
#define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */
@ -61,12 +61,12 @@ static const size_t g_sampleSize = 10000000;
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define CONTROL(c) { if (!(c)) { abort(); } } /* like assert(), but cannot be disabled */
/*_************************************
* Benchmark Parameters
**************************************/
static unsigned g_nbIterations = NBLOOPS;
static double g_compressibility = COMPRESSIBILITY_DEFAULT;
/*_*******************************************************
@ -100,12 +100,12 @@ static ZSTD_CCtx* g_zcc = NULL;
static size_t
local_ZSTD_compress(const void* src, size_t srcSize,
void* dst, size_t dstSize,
void* buff2)
void* payload)
{
ZSTD_parameters p;
ZSTD_frameParameters f = { 1 /* contentSizeHeader*/, 0, 0 };
p.fParams = f;
p.cParams = *(ZSTD_compressionParameters*)buff2;
p.cParams = *(ZSTD_compressionParameters*)payload;
return ZSTD_compress_advanced (g_zcc, dst, dstSize, src, srcSize, NULL ,0, p);
//return ZSTD_compress(dst, dstSize, src, srcSize, cLevel);
}
@ -126,7 +126,7 @@ extern size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* ctx, const void* src, size_t s
static size_t local_ZSTD_decodeLiteralsBlock(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
{
(void)src; (void)srcSize; (void)dst; (void)dstSize;
return ZSTD_decodeLiteralsBlock((ZSTD_DCtx*)g_zdc, buff2, g_cSize);
return ZSTD_decodeLiteralsBlock(g_zdc, buff2, g_cSize);
}
static size_t local_ZSTD_decodeSeqHeaders(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
@ -141,14 +141,14 @@ static ZSTD_CStream* g_cstream= NULL;
static size_t
local_ZSTD_compressStream(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* buff2)
void* payload)
{
ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn;
ZSTD_parameters p;
ZSTD_frameParameters f = {1 /* contentSizeHeader*/, 0, 0};
p.fParams = f;
p.cParams = *(ZSTD_compressionParameters*)buff2;
p.cParams = *(ZSTD_compressionParameters*)payload;
ZSTD_initCStream_advanced(g_cstream, NULL, 0, p, ZSTD_CONTENTSIZE_UNKNOWN);
buffOut.dst = dst;
buffOut.size = dstCapacity;
@ -161,23 +161,39 @@ local_ZSTD_compressStream(const void* src, size_t srcSize,
return buffOut.pos;
}
static size_t
local_ZSTD_compressStream_freshCCtx(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* payload)
{
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
size_t r;
assert(cctx != NULL);
r = local_ZSTD_compressStream(src, srcSize, dst, dstCapacity, payload);
ZSTD_freeCCtx(cctx);
return r;
}
static size_t
local_ZSTD_compress_generic_end(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* buff2)
void* payload)
{
(void)buff2;
(void)payload;
return ZSTD_compress2(g_cstream, dst, dstCapacity, src, srcSize);
}
static size_t
local_ZSTD_compress_generic_continue(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* buff2)
void* payload)
{
ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn;
(void)buff2;
(void)payload;
buffOut.dst = dst;
buffOut.size = dstCapacity;
buffOut.pos = 0;
@ -192,9 +208,9 @@ local_ZSTD_compress_generic_continue(const void* src, size_t srcSize,
static size_t
local_ZSTD_compress_generic_T2_end(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* buff2)
void* payload)
{
(void)buff2;
(void)payload;
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_nbWorkers, 2);
return ZSTD_compress2(g_cstream, dst, dstCapacity, src, srcSize);
}
@ -202,11 +218,11 @@ local_ZSTD_compress_generic_T2_end(const void* src, size_t srcSize,
static size_t
local_ZSTD_compress_generic_T2_continue(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* buff2)
void* payload)
{
ZSTD_outBuffer buffOut;
ZSTD_inBuffer buffIn;
(void)buff2;
(void)payload;
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_nbWorkers, 2);
buffOut.dst = dst;
buffOut.size = dstCapacity;
@ -242,27 +258,28 @@ local_ZSTD_decompressStream(const void* src, size_t srcSize,
#ifndef ZSTD_DLL_IMPORT
static size_t local_ZSTD_compressContinue(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* buff2)
void* payload)
{
ZSTD_parameters p;
ZSTD_frameParameters f = { 1 /* contentSizeHeader*/, 0, 0 };
p.fParams = f;
p.cParams = *(ZSTD_compressionParameters*)buff2;
p.cParams = *(ZSTD_compressionParameters*)payload;
ZSTD_compressBegin_advanced(g_zcc, NULL, 0, p, srcSize);
return ZSTD_compressEnd(g_zcc, dst, dstCapacity, src, srcSize);
}
#define FIRST_BLOCK_SIZE 8
static size_t local_ZSTD_compressContinue_extDict(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* buff2)
static size_t
local_ZSTD_compressContinue_extDict(const void* src, size_t srcSize,
void* dst, size_t dstCapacity,
void* payload)
{
BYTE firstBlockBuf[FIRST_BLOCK_SIZE];
ZSTD_parameters p;
ZSTD_frameParameters f = { 1, 0, 0 };
ZSTD_frameParameters const f = { 1, 0, 0 };
p.fParams = f;
p.cParams = *(ZSTD_compressionParameters*)buff2;
p.cParams = *(ZSTD_compressionParameters*)payload;
ZSTD_compressBegin_advanced(g_zcc, NULL, 0, p, srcSize);
memcpy(firstBlockBuf, src, FIRST_BLOCK_SIZE);
@ -318,7 +335,7 @@ static int benchMem(unsigned benchNb,
size_t dstBuffSize = ZSTD_compressBound(srcSize);
BYTE* dstBuff;
void* dstBuff2;
void* buff2;
void* payload;
const char* benchName;
BMK_benchFn_t benchFunction;
int errorcode = 0;
@ -355,6 +372,9 @@ static int benchMem(unsigned benchNb,
case 42:
benchFunction = local_ZSTD_decompressStream; benchName = "decompressStream";
break;
case 43:
benchFunction = local_ZSTD_compressStream_freshCCtx; benchName = "compressStream_freshCCtx";
break;
case 51:
benchFunction = local_ZSTD_compress_generic_continue; benchName = "compress_generic, continue";
break;
@ -379,7 +399,7 @@ static int benchMem(unsigned benchNb,
free(dstBuff); free(dstBuff2);
return 12;
}
buff2 = dstBuff2;
payload = dstBuff2;
if (g_zcc==NULL) g_zcc = ZSTD_createCCtx();
if (g_zdc==NULL) g_zdc = ZSTD_createDCtx();
if (g_cstream==NULL) g_cstream = ZSTD_createCStream();
@ -412,62 +432,66 @@ static int benchMem(unsigned benchNb,
switch(benchNb)
{
case 1:
buff2 = &cparams;
payload = &cparams;
break;
case 2:
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel);
g_cSize = ZSTD_compress(dstBuff2, dstBuffSize, src, srcSize, cLevel);
break;
#ifndef ZSTD_DLL_IMPORT
case 11:
buff2 = &cparams;
payload = &cparams;
break;
case 12:
buff2 = &cparams;
payload = &cparams;
break;
case 13 :
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel);
g_cSize = ZSTD_compress(dstBuff2, dstBuffSize, src, srcSize, cLevel);
break;
case 31: /* ZSTD_decodeLiteralsBlock */
{ blockProperties_t bp;
ZSTD_frameHeader zfp;
size_t frameHeaderSize, skippedSize;
case 31: /* ZSTD_decodeLiteralsBlock : starts literals block in dstBuff2 */
{ size_t frameHeaderSize;
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_FRAMEHEADERSIZE_MIN);
if (frameHeaderSize==0) frameHeaderSize = ZSTD_FRAMEHEADERSIZE_MIN;
ZSTD_getcBlockSize(dstBuff+frameHeaderSize, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed) {
DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n");
goto _cleanOut;
frameHeaderSize = ZSTD_frameHeaderSize(dstBuff, ZSTD_FRAMEHEADERSIZE_PREFIX);
CONTROL(!ZSTD_isError(frameHeaderSize));
/* check block is compressible, hence contains a literals section */
{ blockProperties_t bp;
ZSTD_getcBlockSize(dstBuff+frameHeaderSize, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed) {
DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n");
goto _cleanOut;
} }
{ size_t const skippedSize = frameHeaderSize + ZSTD_blockHeaderSize;
memcpy(dstBuff2, dstBuff+skippedSize, g_cSize-skippedSize);
}
skippedSize = frameHeaderSize + ZSTD_blockHeaderSize;
memcpy(buff2, dstBuff+skippedSize, g_cSize-skippedSize);
srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */
ZSTD_decompressBegin(g_zdc);
break;
}
case 32: /* ZSTD_decodeSeqHeaders */
{ blockProperties_t bp;
ZSTD_frameHeader zfp;
const BYTE* ip = dstBuff;
const BYTE* iend;
size_t frameHeaderSize, cBlockSize;
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel); /* it would be better to use direct block compression here */
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
frameHeaderSize = ZSTD_getFrameHeader(&zfp, dstBuff, ZSTD_FRAMEHEADERSIZE_MIN);
if (frameHeaderSize==0) frameHeaderSize = ZSTD_FRAMEHEADERSIZE_MIN;
ip += frameHeaderSize; /* Skip frame Header */
cBlockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed) {
DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n");
goto _cleanOut;
{ size_t const cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel);
CONTROL(cSize > ZSTD_FRAMEHEADERSIZE_PREFIX);
}
iend = ip + ZSTD_blockHeaderSize + cBlockSize; /* End of first block */
ip += ZSTD_blockHeaderSize; /* skip block header */
/* Skip frame Header */
{ size_t const frameHeaderSize = ZSTD_frameHeaderSize(dstBuff, ZSTD_FRAMEHEADERSIZE_PREFIX);
CONTROL(!ZSTD_isError(frameHeaderSize));
ip += frameHeaderSize;
}
/* Find end of block */
{ size_t const cBlockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed) {
DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n");
goto _cleanOut;
}
iend = ip + ZSTD_blockHeaderSize + cBlockSize; /* End of first block */
}
ip += ZSTD_blockHeaderSize; /* skip block header */
ZSTD_decompressBegin(g_zdc);
assert(iend > ip);
CONTROL(iend > ip);
ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, (size_t)(iend-ip)); /* skip literal segment */
g_cSize = (size_t)(iend-ip);
memcpy(buff2, ip, g_cSize); /* copy rest of block (it starts by SeqHeader) */
memcpy(dstBuff2, ip, g_cSize); /* copy rest of block (it starts by SeqHeader) */
srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */
break;
}
@ -476,10 +500,13 @@ static int benchMem(unsigned benchNb,
goto _cleanOut;
#endif
case 41 :
buff2 = &cparams;
payload = &cparams;
break;
case 42 :
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, cLevel);
g_cSize = ZSTD_compress(payload, dstBuffSize, src, srcSize, cLevel);
break;
case 43 :
payload = &cparams;
break;
/* test functions */
@ -498,10 +525,10 @@ static int benchMem(unsigned benchNb,
BMK_runTime_t bestResult;
bestResult.sumOfReturn = 0;
bestResult.nanoSecPerRun = (double)TIMELOOP_NANOSEC * 2000000000; /* hopefully large enough : must be larger than any potential measurement */
assert(tfs != NULL);
CONTROL(tfs != NULL);
bp.benchFn = benchFunction;
bp.benchPayload = buff2;
bp.benchPayload = payload;
bp.initFn = NULL;
bp.initPayload = NULL;
bp.errorFn = ZSTD_isError;
@ -549,21 +576,19 @@ _cleanOut:
static int benchSample(U32 benchNb,
size_t benchedSize, double compressibility,
int cLevel, ZSTD_compressionParameters cparams)
{
size_t const benchedSize = g_sampleSize;
const char* const name = "Sample 10MiB";
/* Allocation */
void* const origBuff = malloc(benchedSize);
if (!origBuff) { DISPLAY("\nError: not enough memory!\n"); return 12; }
/* Fill buffer */
RDG_genBuffer(origBuff, benchedSize, g_compressibility, 0.0, 0);
RDG_genBuffer(origBuff, benchedSize, compressibility, 0.0, 0);
/* bench */
DISPLAY("\r%70s\r", "");
DISPLAY(" %s : \n", name);
DISPLAY(" Sample %u bytes : \n", (unsigned)benchedSize);
if (benchNb) {
benchMem(benchNb, origBuff, benchedSize, cLevel, cparams);
} else { /* 0 == run all tests */
@ -696,10 +721,11 @@ static int usage_advanced(const char* exename)
usage(exename);
DISPLAY( "\nAdvanced options :\n");
DISPLAY( " -b# : test only function # \n");
DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS);
DISPLAY( " -P# : sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100);
DISPLAY( " -l# : benchmark functions at that compression level (default : %i)\n", DEFAULT_CLEVEL);
DISPLAY( " --zstd : custom parameter selection. Format same as zstdcli \n");
DISPLAY( " -P# : sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100);
DISPLAY( " -B# : sample size (default : %u)\n", (unsigned)kSampleSizeDefault);
DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS);
return 0;
}
@ -718,13 +744,15 @@ int main(int argc, const char** argv)
U32 benchNb = 0, main_pause = 0;
int cLevel = DEFAULT_CLEVEL;
ZSTD_compressionParameters cparams = ZSTD_getCParams(cLevel, 0, 0);
size_t sampleSize = kSampleSizeDefault;
double compressibility = COMPRESSIBILITY_DEFAULT;
DISPLAY(WELCOME_MESSAGE);
if (argc<1) return badusage(exename);
for (argNb=1; argNb<argc; argNb++) {
const char* argument = argv[argNb];
assert(argument != NULL);
CONTROL(argument != NULL);
if (longCommandWArg(&argument, "--zstd=")) {
for ( ; ;) {
@ -767,21 +795,29 @@ int main(int argc, const char** argv)
benchNb = readU32FromChar(&argument);
break;
/* Modify Nb Iterations */
case 'i':
/* Select compression level to use */
case 'l':
argument++;
g_nbIterations = readU32FromChar(&argument);
cLevel = (int)readU32FromChar(&argument);
cparams = ZSTD_getCParams(cLevel, 0, 0);
break;
/* Select compressibility of synthetic sample */
case 'P':
argument++;
g_compressibility = (double)readU32FromChar(&argument) / 100.;
compressibility = (double)readU32FromChar(&argument) / 100.;
break;
case 'l':
/* Select size of synthetic sample */
case 'B':
argument++;
cLevel = (int)readU32FromChar(&argument);
cparams = ZSTD_getCParams(cLevel, 0, 0);
sampleSize = (size_t)readU32FromChar(&argument);
break;
/* Modify Nb Iterations */
case 'i':
argument++;
g_nbIterations = readU32FromChar(&argument);
break;
/* Unknown command */
@ -798,7 +834,7 @@ int main(int argc, const char** argv)
if (filenamesStart==0) /* no input file */
result = benchSample(benchNb, cLevel, cparams);
result = benchSample(benchNb, sampleSize, compressibility, cLevel, cparams);
else
result = benchFiles(benchNb, argv+filenamesStart, argc-filenamesStart, cLevel, cparams);

8
tests/fuzzer.c
View File

@ -2464,7 +2464,7 @@ static unsigned readU32FromChar(const char** stringPtr)
* If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
* @return 0 and doesn't modify *stringPtr otherwise.
*/
static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
static int longCommandWArg(const char** stringPtr, const char* longCommand)
{
size_t const comSize = strlen(longCommand);
int const result = !strncmp(*stringPtr, longCommand, comSize);
@ -2524,7 +2524,7 @@ int main(int argc, const char** argv)
case 'i':
argument++; maxDuration = 0;
nbTests = readU32FromChar(&argument);
nbTests = (int)readU32FromChar(&argument);
break;
case 'T':
@ -2544,12 +2544,12 @@ int main(int argc, const char** argv)
case 't':
argument++;
testNb = readU32FromChar(&argument);
testNb = (int)readU32FromChar(&argument);
break;
case 'P': /* compressibility % */
argument++;
proba = readU32FromChar(&argument);
proba = (int)readU32FromChar(&argument);
if (proba>100) proba = 100;
break;