AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fullbench can measure speed of zbuff

Browse Source
This commit is contained in:
Yann Collet 2016-03-15 20:47:23 +01:00
parent 6cb41356e8
commit 34b20ec2c6
3 changed files with 74 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
NEWS
View File

@ -1,6 +1,7 @@
v0.6.0 v0.6.0
Stronger high compression modes, thanks to Przemyslaw Skibinski Stronger high compression modes, thanks to Przemyslaw Skibinski
API : ZSTD_getFrameParams() provides size of decompressed content API : ZSTD_getFrameParams() provides size of decompressed content
New : highest compression modes require `--ultra` command to fully unleash their capacity
v0.5.1 v0.5.1
New : Optimal parsing => Very high compression modes, thanks to Przemyslaw Skibinski New : Optimal parsing => Very high compression modes, thanks to Przemyslaw Skibinski

4
programs/Makefile
View File

@ -112,11 +112,11 @@ zstd-frugal: $(ZSTD_FILES) $(ZSTDDIR)/zbuff.c zstdcli.c fileio.c
zstd-small: clean zstd-small: clean
CFLAGS="-Os -s" $(MAKE) zstd-frugal CFLAGS="-Os -s" $(MAKE) zstd-frugal
fullbench : $(ZSTD_FILES) \ fullbench : $(ZSTD_FILES) $(ZSTDDIR)/zbuff.c \
datagen.c fullbench.c datagen.c fullbench.c
$(CC) $(FLAGS) $^ -o $@$(EXT) $(CC) $(FLAGS) $^ -o $@$(EXT)
fullbench32: $(ZSTD_FILES) \ fullbench32: $(ZSTD_FILES) $(ZSTDDIR)/zbuff.c \
datagen.c fullbench.c datagen.c fullbench.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT) $(CC) -m32 $(FLAGS) $^ -o $@$(EXT)

117
programs/fullbench.c
View File

@ -1,6 +1,6 @@
/* /*
fullbench.c - Detailed bench program for zstd fullbench.c - Detailed bench program for zstd
Copyright (C) Yann Collet 2014-2015 Copyright (C) Yann Collet 2014-2016
GPL v2 License GPL v2 License
@ -19,11 +19,10 @@
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at : You can contact the author at :
- zstd source repository : https://github.com/Cyan4973/zstd - zstd homepage : http://www.zstd.net
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
*/ */
/************************************** /*_************************************
* Compiler Options * Compiler Options
**************************************/ **************************************/
/* Disable some Visual warning messages */ /* Disable some Visual warning messages */
@ -44,7 +43,7 @@
#endif #endif
/************************************** /*_************************************
* Includes * Includes
**************************************/ **************************************/
#include <stdlib.h> /* malloc */ #include <stdlib.h> /* malloc */
@ -63,10 +62,11 @@
#include "mem.h" #include "mem.h"
#include "zstd_static.h" #include "zstd_static.h"
#include "fse_static.h" #include "fse_static.h"
#include "zbuff.h"
#include "datagen.h" #include "datagen.h"
/************************************** /*_************************************
* Compiler Options * Compiler Options
**************************************/ **************************************/
/* S_ISREG & gettimeofday() are not supported by MSVC */ /* S_ISREG & gettimeofday() are not supported by MSVC */
@ -75,7 +75,7 @@
#endif #endif
/************************************** /*_************************************
* Constants * Constants
**************************************/ **************************************/
#define PROGRAM_DESCRIPTION "Zstandard speed analyzer" #define PROGRAM_DESCRIPTION "Zstandard speed analyzer"
@ -100,13 +100,13 @@
static const size_t sampleSize = 10000000; static const size_t sampleSize = 10000000;
/************************************** /*_************************************
* Macros * Macros
**************************************/ **************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
/************************************** /*_************************************
* Benchmark Parameters * Benchmark Parameters
**************************************/ **************************************/
static int nbIterations = NBLOOPS; static int nbIterations = NBLOOPS;
@ -119,7 +119,7 @@ void BMK_SetNbIterations(int nbLoops)
} }
/********************************************************* /*_*******************************************************
* Private functions * Private functions
*********************************************************/ *********************************************************/
@ -156,8 +156,7 @@ static int BMK_GetMilliStart(void)
static int BMK_GetMilliSpan( int nTimeStart ) static int BMK_GetMilliSpan( int nTimeStart )
{ {
int nSpan = BMK_GetMilliStart() - nTimeStart; int nSpan = BMK_GetMilliStart() - nTimeStart;
if ( nSpan < 0 ) if ( nSpan < 0 ) nSpan += 0x100000 * 1000;
nSpan += 0x100000 * 1000;
return nSpan; return nSpan;
} }
@ -165,16 +164,15 @@ static int BMK_GetMilliSpan( int nTimeStart )
static size_t BMK_findMaxMem(U64 requiredMem) static size_t BMK_findMaxMem(U64 requiredMem)
{ {
size_t step = 64 MB; size_t step = 64 MB;
BYTE* testmem=NULL; void* testmem = NULL;
requiredMem = (((requiredMem >> 26) + 1) << 26); requiredMem = (((requiredMem >> 26) + 1) << 26);
if (requiredMem > MAX_MEM) requiredMem = MAX_MEM; if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
requiredMem += 2*step; requiredMem += 2*step;
while (!testmem) while (!testmem) {
{
requiredMem -= step; requiredMem -= step;
testmem = (BYTE*) malloc ((size_t)requiredMem); testmem = malloc ((size_t)requiredMem);
} }
free (testmem); free (testmem);
@ -197,12 +195,11 @@ static U64 BMK_GetFileSize(char* infilename)
} }
/********************************************************* /*_*******************************************************
* Benchmark wrappers * Benchmark wrappers
*********************************************************/ *********************************************************/
typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t; typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
typedef struct typedef struct {
{
blockType_t blockType; blockType_t blockType;
U32 unusedBits; U32 unusedBits;
U32 origSize; U32 origSize;
@ -211,9 +208,6 @@ typedef struct
static size_t g_cSize = 0; static size_t g_cSize = 0;
static ZSTD_DCtx* g_dctxPtr = NULL; static ZSTD_DCtx* g_dctxPtr = NULL;
extern size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr);
extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, const void* src, size_t srcSize);
size_t local_ZSTD_compress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) size_t local_ZSTD_compress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
{ {
(void)buff2; (void)buff2;
@ -233,6 +227,8 @@ size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, co
return ZSTD_decodeLiteralsBlock((ZSTD_DCtx*)g_dctxPtr, buff2, g_cSize); return ZSTD_decodeLiteralsBlock((ZSTD_DCtx*)g_dctxPtr, buff2, g_cSize);
} }
extern size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr);
extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr, FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, const void* src, size_t srcSize);
size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize) size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
{ {
U32 DTableML[FSE_DTABLE_SIZE_U32(10)], DTableLL[FSE_DTABLE_SIZE_U32(10)], DTableOffb[FSE_DTABLE_SIZE_U32(9)]; /* MLFSELog, LLFSELog and OffFSELog are not public values */ U32 DTableML[FSE_DTABLE_SIZE_U32(10)], DTableLL[FSE_DTABLE_SIZE_U32(10)], DTableOffb[FSE_DTABLE_SIZE_U32(9)]; /* MLFSELog, LLFSELog and OffFSELog are not public values */
@ -244,8 +240,34 @@ size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const
} }
static ZBUFF_CCtx* g_zbcc = NULL;
size_t local_ZBUFF_compress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
{
size_t compressedSize;
size_t srcRead = srcSize, dstWritten = dstSize;
(void)buff2;
ZBUFF_compressInit(g_zbcc, 1);
ZBUFF_compressContinue(g_zbcc, dst, &dstWritten, src, &srcRead);
compressedSize = dstWritten;
dstWritten = dstSize-compressedSize;
ZBUFF_compressEnd(g_zbcc, ((char*)dst)+compressedSize, &dstWritten);
compressedSize += dstWritten;
return compressedSize;
}
/********************************************************* static ZBUFF_DCtx* g_zbdc = NULL;
size_t local_ZBUFF_decompress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
{
size_t srcRead = g_cSize, dstWritten = dstSize;
(void)src; (void)srcSize;
ZBUFF_decompressInit(g_zbdc);
ZBUFF_decompressContinue(g_zbdc, dst, &dstWritten, buff2, &srcRead);
return dstWritten;
}
/*_*******************************************************
* Bench functions * Bench functions
*********************************************************/ *********************************************************/
size_t benchMem(void* src, size_t srcSize, U32 benchNb) size_t benchMem(void* src, size_t srcSize, U32 benchNb)
@ -274,6 +296,12 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
case 32: case 32:
benchFunction = local_ZSTD_decodeSeqHeaders; benchName = "ZSTD_decodeSeqHeaders"; benchFunction = local_ZSTD_decodeSeqHeaders; benchName = "ZSTD_decodeSeqHeaders";
break; break;
case 41:
benchFunction = local_ZBUFF_compress; benchName = "ZBUFF_compressContinue";
break;
case 42:
benchFunction = local_ZBUFF_decompress; benchName = "ZBUFF_decompressContinue";
break;
default : default :
return 0; return 0;
} }
@ -283,8 +311,7 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
dstBuff = (BYTE*)malloc(dstBuffSize); dstBuff = (BYTE*)malloc(dstBuffSize);
buff2 = (BYTE*)malloc(dstBuffSize); buff2 = (BYTE*)malloc(dstBuffSize);
g_dctxPtr = ZSTD_createDCtx(); g_dctxPtr = ZSTD_createDCtx();
if ((!dstBuff) || (!buff2)) if ((!dstBuff) || (!buff2)) {
{
DISPLAY("\nError: not enough memory!\n"); DISPLAY("\nError: not enough memory!\n");
free(dstBuff); free(buff2); free(dstBuff); free(buff2);
return 12; return 12;
@ -301,8 +328,7 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
blockProperties_t bp; blockProperties_t bp;
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1); g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp); /* Get 1st block type */ ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp); /* Get 1st block type */
if (bp.blockType != bt_compressed) if (bp.blockType != bt_compressed) {
{
DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n"); DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n");
goto _cleanOut; goto _cleanOut;
} }
@ -332,6 +358,13 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */ srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */
break; break;
} }
case 41 :
if (g_zbcc==NULL) g_zbcc=ZBUFF_createCCtx();
break;
case 42 :
if (g_zbdc==NULL) g_zbdc=ZBUFF_createDCtx();
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
break;
/* test functions */ /* test functions */
/* by convention, test functions can be added > 100 */ /* by convention, test functions can be added > 100 */
@ -352,8 +385,7 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
milliTime = BMK_GetMilliStart(); milliTime = BMK_GetMilliStart();
while(BMK_GetMilliStart() == milliTime); while(BMK_GetMilliStart() == milliTime);
milliTime = BMK_GetMilliStart(); milliTime = BMK_GetMilliStart();
while(BMK_GetMilliSpan(milliTime) < TIMELOOP) while(BMK_GetMilliSpan(milliTime) < TIMELOOP) {
{
errorCode = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize); errorCode = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize);
if (ZSTD_isError(errorCode)) { DISPLAY("ERROR ! %s() => %s !! \n", benchName, ZSTD_getErrorName(errorCode)); exit(1); } if (ZSTD_isError(errorCode)) { DISPLAY("ERROR ! %s() => %s !! \n", benchName, ZSTD_getErrorName(errorCode)); exit(1); }
nbRounds++; nbRounds++;
@ -383,8 +415,7 @@ int benchSample(U32 benchNb)
/* Allocation */ /* Allocation */
origBuff = (char*) malloc((size_t)benchedSize); origBuff = (char*) malloc((size_t)benchedSize);
if(!origBuff) if(!origBuff) {
{
DISPLAY("\nError: not enough memory!\n"); DISPLAY("\nError: not enough memory!\n");
return 12; return 12;
} }
@ -407,11 +438,9 @@ int benchSample(U32 benchNb)
int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb) int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
{ {
int fileIdx=0;
/* Loop for each file */ /* Loop for each file */
while (fileIdx<nbFiles) int fileIdx=0;
{ while (fileIdx<nbFiles) {
FILE* inFile; FILE* inFile;
char* inFileName; char* inFileName;
U64 inFileSize; U64 inFileSize;
@ -422,8 +451,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
/* Check file existence */ /* Check file existence */
inFileName = fileNamesTable[fileIdx++]; inFileName = fileNamesTable[fileIdx++];
inFile = fopen( inFileName, "rb" ); inFile = fopen( inFileName, "rb" );
if (inFile==NULL) if (inFile==NULL) {
{
DISPLAY( "Pb opening %s\n", inFileName); DISPLAY( "Pb opening %s\n", inFileName);
return 11; return 11;
} }
@ -432,15 +460,13 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
inFileSize = BMK_GetFileSize(inFileName); inFileSize = BMK_GetFileSize(inFileName);
benchedSize = (size_t) BMK_findMaxMem(inFileSize*3) / 3; benchedSize = (size_t) BMK_findMaxMem(inFileSize*3) / 3;
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
if (benchedSize < inFileSize) if (benchedSize < inFileSize) {
{
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
} }
/* Alloc */ /* Alloc */
origBuff = (char*) malloc((size_t)benchedSize); origBuff = (char*) malloc((size_t)benchedSize);
if(!origBuff) if(!origBuff) {
{
DISPLAY("\nError: not enough memory!\n"); DISPLAY("\nError: not enough memory!\n");
fclose(inFile); fclose(inFile);
return 12; return 12;
@ -451,8 +477,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
readSize = fread(origBuff, 1, benchedSize, inFile); readSize = fread(origBuff, 1, benchedSize, inFile);
fclose(inFile); fclose(inFile);
if(readSize != benchedSize) if(readSize != benchedSize) {
{
DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
free(origBuff); free(origBuff);
return 13; return 13;
@ -471,7 +496,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
} }
int usage(char* exename) static int usage(char* exename)
{ {
DISPLAY( "Usage :\n"); DISPLAY( "Usage :\n");
DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename); DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename);
@ -480,7 +505,7 @@ int usage(char* exename)
return 0; return 0;
} }
int usage_advanced(void) static int usage_advanced(void)
{ {
DISPLAY( "\nAdvanced options :\n"); DISPLAY( "\nAdvanced options :\n");
DISPLAY( " -b# : test only function # \n"); DISPLAY( " -b# : test only function # \n");
@ -489,7 +514,7 @@ int usage_advanced(void)
return 0; return 0;
} }
int badusage(char* exename) static int badusage(char* exename)
{ {
DISPLAY("Wrong parameters\n"); DISPLAY("Wrong parameters\n");
usage(exename); usage(exename);