AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

updating cold benchmark

Browse Source
This commit is contained in:
Bimba Shrestha 2020-06-12 08:53:58 -07:00 committed by W. Felix Handte
parent 9c628238d3
commit 80053bdae3
1 changed files with 25 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
contrib/largeNbDicts/largeNbDicts.c
View File

@ -156,6 +156,18 @@ createDictionaryBuffer(const char* dictionaryName,
} }
} }
static ZSTD_CDict* createCDictForDedicatedDictSearch(const void* dict, size_t dictSize, int compressionLevel)
{
ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
ZSTD_CCtxParams_init(params, compressionLevel);
ZSTD_CCtxParams_setParameter(params, ZSTD_c_enableDedicatedDictSearch, 1);
ZSTD_CCtxParams_setParameter(params, ZSTD_c_compressionLevel, compressionLevel);
ZSTD_CDict* cdict = ZSTD_createCDict_advanced2(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, params, ZSTD_defaultCMem);
ZSTD_freeCCtxParams(params);
return cdict;
}
/*! BMK_loadFiles() : /*! BMK_loadFiles() :
* Loads `buffer`, with content from files listed within `fileNamesTable`. * Loads `buffer`, with content from files listed within `fileNamesTable`.
@ -449,12 +461,14 @@ static void freeCDictCollection(cdict_collection_t cdictc)
} }
/* returns .buffers=NULL if operation fails */ /* returns .buffers=NULL if operation fails */
static cdict_collection_t createCDictCollection(const void* dictBuffer, size_t dictSize, size_t nbCDict, int cLevel) static cdict_collection_t createCDictCollection(const void* dictBuffer, size_t dictSize, size_t nbCDict, int cLevel, int dedicatedDictSearch)
{ {
ZSTD_CDict** const cdicts = malloc(nbCDict * sizeof(ZSTD_CDict*)); ZSTD_CDict** const cdicts = malloc(nbCDict * sizeof(ZSTD_CDict*));
if (cdicts==NULL) return kNullCDictCollection; if (cdicts==NULL) return kNullCDictCollection;
for (size_t dictNb=0; dictNb < nbCDict; dictNb++) { for (size_t dictNb=0; dictNb < nbCDict; dictNb++) {
cdicts[dictNb] = ZSTD_createCDict(dictBuffer, dictSize, cLevel); cdicts[dictNb] = dedicatedDictSearch ?
createCDictForDedicatedDictSearch(dictBuffer, dictSize, cLevel) :
ZSTD_createCDict(dictBuffer, dictSize, cLevel);
CONTROL(cdicts[dictNb] != NULL); CONTROL(cdicts[dictNb] != NULL);
} }
cdict_collection_t cdictc; cdict_collection_t cdictc;
@ -720,7 +734,8 @@ int bench(const char** fileNameTable, unsigned nbFiles,
const char* dictionary, const char* dictionary,
size_t blockSize, int clevel, size_t blockSize, int clevel,
unsigned nbDictMax, unsigned nbBlocks, unsigned nbDictMax, unsigned nbBlocks,
unsigned nbRounds, int benchCompression) unsigned nbRounds, int benchCompression,
int dedicatedDictSearch)
{ {
int result = 0; int result = 0;
@ -775,7 +790,9 @@ int bench(const char** fileNameTable, unsigned nbFiles,
DICTSIZE); DICTSIZE);
CONTROL(dictBuffer.ptr != NULL); CONTROL(dictBuffer.ptr != NULL);
ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer.ptr, dictBuffer.size, clevel); ZSTD_CDict* const cdict = dedicatedDictSearch ?
createCDictForDedicatedDictSearch(dictBuffer.ptr, dictBuffer.size, clevel) :
ZSTD_createCDict(dictBuffer.ptr, dictBuffer.size, clevel);
CONTROL(cdict != NULL); CONTROL(cdict != NULL);
size_t const cTotalSizeNoDict = compressBlocks(NULL, dstSlices, srcSlices, NULL, clevel); size_t const cTotalSizeNoDict = compressBlocks(NULL, dstSlices, srcSlices, NULL, clevel);
@ -798,7 +815,7 @@ int bench(const char** fileNameTable, unsigned nbFiles,
unsigned const nbDicts = nbDictMax ? nbDictMax : nbBlocks; unsigned const nbDicts = nbDictMax ? nbDictMax : nbBlocks;
cdict_collection_t const cdictionaries = createCDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts, clevel); cdict_collection_t const cdictionaries = createCDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts, clevel, dedicatedDictSearch);
CONTROL(cdictionaries.cdicts != NULL); CONTROL(cdictionaries.cdicts != NULL);
ddict_collection_t const ddictionaries = createDDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts); ddict_collection_t const ddictionaries = createDDictCollection(dictBuffer.ptr, dictBuffer.size, nbDicts);
@ -924,6 +941,7 @@ int main (int argc, const char** argv)
{ {
int recursiveMode = 0; int recursiveMode = 0;
int benchCompression = 1; int benchCompression = 1;
int dedicatedDictSearch = 0;
unsigned nbRounds = BENCH_TIME_DEFAULT_S; unsigned nbRounds = BENCH_TIME_DEFAULT_S;
const char* const exeName = argv[0]; const char* const exeName = argv[0];
@ -953,6 +971,7 @@ int main (int argc, const char** argv)
if (longCommandWArg(&argument, "--nbDicts=")) { nbDicts = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--nbDicts=")) { nbDicts = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--nbBlocks=")) { nbBlocks = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--nbBlocks=")) { nbBlocks = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--clevel=")) { cLevel = (int)readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--clevel=")) { cLevel = (int)readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--dedicated-dict-search")) { dedicatedDictSearch = 1; continue; }
if (longCommandWArg(&argument, "-")) { cLevel = (int)readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "-")) { cLevel = (int)readU32FromChar(&argument); continue; }
/* anything that's not a command is a filename */ /* anything that's not a command is a filename */
nameTable[nameIdx++] = argument; nameTable[nameIdx++] = argument;
@ -970,7 +989,7 @@ int main (int argc, const char** argv)
nameTable = NULL; /* UTIL_createFileNamesTable() takes ownership of nameTable */ nameTable = NULL; /* UTIL_createFileNamesTable() takes ownership of nameTable */
} }
int result = bench(filenameTable->fileNames, (unsigned)filenameTable->tableSize, dictionary, blockSize, cLevel, nbDicts, nbBlocks, nbRounds, benchCompression); int result = bench(filenameTable->fileNames, (unsigned)filenameTable->tableSize, dictionary, blockSize, cLevel, nbDicts, nbBlocks, nbRounds, benchCompression, dedicatedDictSearch);
UTIL_freeFileNamesTable(filenameTable); UTIL_freeFileNamesTable(filenameTable);
free(nameTable); free(nameTable);