AuroraMiddleware/lz4
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

bench.c based on zstd

Browse Source
This commit is contained in:
Przemyslaw Skibinski 2016-11-04 14:26:12 +01:00
parent e06fcd9039
commit 8b8c726a5c
3 changed files with 449 additions and 345 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

746
programs/bench.c
View File

@ -23,36 +23,18 @@
- LZ4 source repository : https://github.com/lz4/lz4
*/
/*-************************************
* Compiler Options
***************************************/
#if defined(_MSC_VER) || defined(_WIN32)
# define _CRT_SECURE_NO_WARNINGS
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */
#endif
/* Unix Large Files support (>4GB) */
#define _FILE_OFFSET_BITS 64
#if (defined(__sun__) && (!defined(__LP64__))) /* Sun Solaris 32-bits requires specific definitions */
# define _LARGEFILE_SOURCE
#elif ! defined(__LP64__) /* No point defining Large file for 64 bit */
# define _LARGEFILE64_SOURCE
#endif
#if defined(__MINGW32__) && !defined(_POSIX_SOURCE)
# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */
#endif
/*-************************************
/* *************************************
* Includes
***************************************/
#include <stdlib.h> /* malloc */
#include <stdio.h> /* fprintf, fopen */
#include <sys/types.h> /* stat64 */
#include <sys/stat.h> /* stat64 */
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
#include <string.h> /* strlen */
#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_sleep */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* memset */
#include <stdio.h> /* fprintf, fopen, ftello64 */
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
#include "datagen.h" /* RDG_genBuffer */
#include "xxhash.h"
#include "lz4.h"
@ -61,348 +43,454 @@ static int LZ4_compress_local(const char* src, char* dst, int srcSize, int dstSi
#include "lz4hc.h"
#define COMPRESSOR1 LZ4_compress_HC
#define DEFAULTCOMPRESSOR COMPRESSOR0
#include "xxhash.h"
#define LZ4_isError(errcode) (errcode==0)
/*-************************************
* Compiler specifics
***************************************/
#if !defined(S_ISREG)
# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
#endif
/*-************************************
* Basic Types
***************************************/
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
#endif
/*-************************************
/* *************************************
* Constants
***************************************/
#define NBLOOPS 3
#define TIMELOOP_S 1
#define TIMELOOP_CLOCK (TIMELOOP_S * CLOCKS_PER_SEC)
#ifndef LZ4_GIT_COMMIT_STRING
# define LZ4_GIT_COMMIT_STRING ""
#else
# define LZ4_GIT_COMMIT_STRING LZ4_EXPAND_AND_QUOTE(LZ4_GIT_COMMIT)
#endif
#define NBSECONDS 3
#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */
#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */
#define COOLPERIOD_SEC 10
#define KB *(1 <<10)
#define MB *(1 <<20)
#define GB *(1U<<30)
#define MAX_MEM (2 GB - 64 MB)
#define DEFAULT_CHUNKSIZE (4 MB)
static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
static U32 g_compressibilityDefault = 50;
/*-************************************
* Local structures
/* *************************************
* console display
***************************************/
struct chunkParameters
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \
{ g_time = clock(); DISPLAY(__VA_ARGS__); \
if (g_displayLevel>=4) fflush(stdout); } }
static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
static clock_t g_time = 0;
/* *************************************
* Exceptions
***************************************/
#ifndef DEBUG
# define DEBUG 0
#endif
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
#define EXM_THROW(error, ...) \
{ \
DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
DISPLAYLEVEL(1, "Error %i : ", error); \
DISPLAYLEVEL(1, __VA_ARGS__); \
DISPLAYLEVEL(1, "\n"); \
exit(error); \
}
/* *************************************
* Benchmark Parameters
***************************************/
static U32 g_nbSeconds = NBSECONDS;
static size_t g_blockSize = 0;
int g_additionalParam = 0;
void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
void BMK_SetNbSeconds(unsigned nbSeconds)
{
U32 id;
char* origBuffer;
char* compressedBuffer;
int origSize;
int compressedSize;
};
g_nbSeconds = nbSeconds;
DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbSeconds);
}
void BMK_SetBlockSize(size_t blockSize)
{
g_blockSize = blockSize;
DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10));
}
/* ********************************************************
* Bench functions
**********************************************************/
typedef struct
{
const char* srcPtr;
size_t srcSize;
char* cPtr;
size_t cRoom;
size_t cSize;
char* resPtr;
size_t resSize;
} blockParam_t;
struct compressionParameters
{
int (*compressionFunction)(const char* src, char* dst, int srcSize, int dstSize, int cLevel);
int (*decompressionFunction)(const char* src, char* dst, int dstSize);
};
#define MIN(a,b) ((a)<(b) ? (a) : (b))
#define MAX(a,b) ((a)>(b) ? (a) : (b))
/*-************************************
* Macro
***************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
/*-************************************
* Benchmark Parameters
***************************************/
static int g_chunkSize = DEFAULT_CHUNKSIZE;
static int g_nbIterations = NBLOOPS;
static int BMK_pause = 0;
void BMK_setBlocksize(int bsize) { g_chunkSize = bsize; }
void BMK_setNbIterations(int nbLoops)
static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
const char* displayName, int cLevel,
const size_t* fileSizes, U32 nbFiles)
{
g_nbIterations = nbLoops;
DISPLAY("- %i iterations -\n", g_nbIterations);
size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
size_t const maxCompressedSize = LZ4_compressBound((int)srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
void* const compressedBuffer = malloc(maxCompressedSize);
void* const resultBuffer = malloc(srcSize);
U32 nbBlocks;
UTIL_time_t ticksPerSecond;
struct compressionParameters compP;
int cfunctionId;
/* checks */
if (!compressedBuffer || !resultBuffer || !blockTable)
EXM_THROW(31, "allocation error : not enough memory");
/* init */
if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */
UTIL_initTimer(&ticksPerSecond);
/* Init */
if (cLevel < LZ4HC_MIN_CLEVEL) cfunctionId = 0; else cfunctionId = 1;
switch (cfunctionId)
{
#ifdef COMPRESSOR0
case 0 : compP.compressionFunction = COMPRESSOR0; break;
#endif
#ifdef COMPRESSOR1
case 1 : compP.compressionFunction = COMPRESSOR1; break;
#endif
default : compP.compressionFunction = DEFAULTCOMPRESSOR;
}
/* Init blockTable data */
{ const char* srcPtr = (const char*)srcBuffer;
char* cPtr = (char*)compressedBuffer;
char* resPtr = (char*)resultBuffer;
U32 fileNb;
for (nbBlocks=0, fileNb=0; fileNb<nbFiles; fileNb++) {
size_t remaining = fileSizes[fileNb];
U32 const nbBlocksforThisFile = (U32)((remaining + (blockSize-1)) / blockSize);
U32 const blockEnd = nbBlocks + nbBlocksforThisFile;
for ( ; nbBlocks<blockEnd; nbBlocks++) {
size_t const thisBlockSize = MIN(remaining, blockSize);
blockTable[nbBlocks].srcPtr = srcPtr;
blockTable[nbBlocks].cPtr = cPtr;
blockTable[nbBlocks].resPtr = resPtr;
blockTable[nbBlocks].srcSize = thisBlockSize;
blockTable[nbBlocks].cRoom = LZ4_compressBound((int)thisBlockSize);
srcPtr += thisBlockSize;
cPtr += blockTable[nbBlocks].cRoom;
resPtr += thisBlockSize;
remaining -= thisBlockSize;
} } }
/* warmimg up memory */
RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
/* Bench */
{ U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL);
U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
UTIL_time_t coolTime;
U64 const maxTime = (g_nbSeconds * TIMELOOP_MICROSEC) + 100;
U64 totalCTime=0, totalDTime=0;
U32 cCompleted=0, dCompleted=0;
# define NB_MARKS 4
const char* const marks[NB_MARKS] = { " |", " /", " =", "\\" };
U32 markNb = 0;
size_t cSize = 0;
double ratio = 0.;
UTIL_getTime(&coolTime);
DISPLAYLEVEL(2, "\r%79s\r", "");
while (!cCompleted | !dCompleted) {
UTIL_time_t clockStart;
U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
/* overheat protection */
if (UTIL_clockSpanMicro(coolTime, ticksPerSecond) > ACTIVEPERIOD_MICROSEC) {
DISPLAYLEVEL(2, "\rcooling down ... \r");
UTIL_sleep(COOLPERIOD_SEC);
UTIL_getTime(&coolTime);
}
/* Compression */
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */
UTIL_sleepMilli(1); /* give processor time to other processes */
UTIL_waitForNextTick(ticksPerSecond);
UTIL_getTime(&clockStart);
if (!cCompleted) { /* still some time to do compression tests */
U32 nbLoops = 0;
do {
U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
size_t const rSize = compP.compressionFunction(blockTable[blockNb].srcPtr, blockTable[blockNb].cPtr, (int)blockTable[blockNb].srcSize, (int)blockTable[blockNb].cRoom, cLevel);
if (LZ4_isError(rSize)) EXM_THROW(1, "LZ4_compress() failed");
blockTable[blockNb].cSize = rSize;
}
nbLoops++;
} while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
{ U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
if (clockSpan < fastestC*nbLoops) fastestC = clockSpan / nbLoops;
totalCTime += clockSpan;
cCompleted = totalCTime>maxTime;
} }
cSize = 0;
{ U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
ratio = (double)srcSize / (double)cSize;
markNb = (markNb+1) % NB_MARKS;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
(double)srcSize / fastestC );
(void)fastestD; (void)crcOrig; /* unused when decompression disabled */
#if 1
/* Decompression */
if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */
UTIL_sleepMilli(1); /* give processor time to other processes */
UTIL_waitForNextTick(ticksPerSecond);
UTIL_getTime(&clockStart);
if (!dCompleted) {
U32 nbLoops = 0;
do {
U32 blockNb;
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
size_t const regenSize = LZ4_decompress_safe(blockTable[blockNb].cPtr, blockTable[blockNb].resPtr, (int)blockTable[blockNb].cSize, (int)blockTable[blockNb].srcSize);
if (LZ4_isError(regenSize)) {
DISPLAY("LZ4_decompress_safe() failed on block %u \n", blockNb);
clockLoop = 0; /* force immediate test end */
break;
}
blockTable[blockNb].resSize = regenSize;
}
nbLoops++;
} while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
{ U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
if (clockSpan < fastestD*nbLoops) fastestD = clockSpan / nbLoops;
totalDTime += clockSpan;
dCompleted = totalDTime>maxTime;
} }
markNb = (markNb+1) % NB_MARKS;
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
(double)srcSize / fastestC,
(double)srcSize / fastestD );
/* CRC Checking */
{ U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
if (crcOrig!=crcCheck) {
size_t u;
DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
for (u=0; u<srcSize; u++) {
if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u]) {
U32 segNb, bNb, pos;
size_t bacc = 0;
DISPLAY("Decoding error at pos %u ", (U32)u);
for (segNb = 0; segNb < nbBlocks; segNb++) {
if (bacc + blockTable[segNb].srcSize > u) break;
bacc += blockTable[segNb].srcSize;
}
pos = (U32)(u - bacc);
bNb = pos / (128 KB);
DISPLAY("(block %u, sub %u, pos %u) \n", segNb, bNb, pos);
break;
}
if (u==srcSize-1) { /* should never happen */
DISPLAY("no difference detected\n");
} }
break;
} } /* CRC Checking */
#endif
} /* for (testNb = 1; testNb <= (g_nbSeconds + !g_nbSeconds); testNb++) */
if (g_displayLevel == 1) {
double cSpeed = (double)srcSize / fastestC;
double dSpeed = (double)srcSize / fastestD;
if (g_additionalParam)
DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam);
else
DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName);
}
DISPLAYLEVEL(2, "%2i#\n", cLevel);
} /* Bench */
/* clean up */
free(blockTable);
free(compressedBuffer);
free(resultBuffer);
return 0;
}
void BMK_setPause(void) { BMK_pause = 1; }
/*-*******************************************************
* Private functions
**********************************************************/
/** BMK_getClockSpan() :
works even if overflow; Typical max span ~ 30 mn */
static clock_t BMK_getClockSpan (clock_t clockStart)
{
return clock() - clockStart;
}
static size_t BMK_findMaxMem(U64 requiredMem)
{
size_t const step = 64 MB;
void* testmem = NULL;
BYTE* testmem = NULL;
requiredMem = (((requiredMem >> 26) + 1) << 26);
requiredMem += 2*step;
if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
requiredMem += step;
if (requiredMem > maxMemory) requiredMem = maxMemory;
while (!testmem) {
if (requiredMem > step) requiredMem -= step;
else requiredMem >>= 1;
testmem = malloc ((size_t)requiredMem);
}
free (testmem);
do {
testmem = (BYTE*)malloc((size_t)requiredMem);
requiredMem -= step;
} while (!testmem);
/* keep some space available */
if (requiredMem > step) requiredMem -= step;
else requiredMem >>= 1;
return (size_t)requiredMem;
free(testmem);
return (size_t)(requiredMem);
}
static U64 BMK_GetFileSize(const char* infilename)
static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
const char* displayName, int cLevel, int cLevelLast,
const size_t* fileSizes, unsigned nbFiles)
{
int r;
#if defined(_MSC_VER)
struct _stat64 statbuf;
r = _stat64(infilename, &statbuf);
#else
struct stat statbuf;
r = stat(infilename, &statbuf);
#endif
if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
return (U64)statbuf.st_size;
}
int l;
const char* pch = strrchr(displayName, '\\'); /* Windows */
if (!pch) pch = strrchr(displayName, '/'); /* Linux */
if (pch) displayName = pch+1;
/*-*******************************************************
* Public function
**********************************************************/
SET_HIGH_PRIORITY;
int BMK_benchLevel(const char** fileNamesTable, int nbFiles, int cLevel)
{
int fileIdx=0;
char* orig_buff;
struct compressionParameters compP;
int cfunctionId;
if (g_displayLevel == 1 && !g_additionalParam)
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", LZ4_VERSION_STRING, LZ4_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10));
U64 totals = 0;
U64 totalz = 0;
double totalc = 0.;
double totald = 0.;
/* Init */
if (cLevel < LZ4HC_MIN_CLEVEL) cfunctionId = 0; else cfunctionId = 1;
switch (cfunctionId)
{
#ifdef COMPRESSOR0
case 0 : compP.compressionFunction = COMPRESSOR0; break;
#endif
#ifdef COMPRESSOR1
case 1 : compP.compressionFunction = COMPRESSOR1; break;
#endif
default : compP.compressionFunction = DEFAULTCOMPRESSOR;
}
/* Loop for each file */
while (fileIdx<nbFiles) {
const char* inFileName = fileNamesTable[fileIdx++];
FILE* const inFile = fopen( inFileName, "rb" );
U64 const inFileSize = BMK_GetFileSize(inFileName);
size_t benchedSize = BMK_findMaxMem(inFileSize * 2) / 2;
unsigned nbChunks;
int maxCompressedChunkSize;
size_t readSize;
char* compressedBuffer;
struct chunkParameters* chunkP;
U32 crcOrig;
/* Check file and memory conditions */
if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; }
if (inFileSize==0) { DISPLAY( "file is empty\n"); fclose(inFile); return 11; }
if (benchedSize==0) { DISPLAY( "not enough memory\n"); fclose(inFile); return 11; }
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
if (benchedSize < inFileSize) {
DISPLAY("Not enough memory for '%s' full size; testing %u MB only...\n", inFileName, (unsigned)(benchedSize>>20));
}
/* Allocation */
nbChunks = (unsigned)(benchedSize / g_chunkSize) + 1;
chunkP = (struct chunkParameters*) malloc(nbChunks * sizeof(struct chunkParameters));
orig_buff = (char*)malloc(benchedSize);
maxCompressedChunkSize = LZ4_compressBound(g_chunkSize);
{ size_t const compressedBuffSize = (size_t)(nbChunks * maxCompressedChunkSize);
compressedBuffer = (char*)malloc(compressedBuffSize); }
if (!orig_buff || !compressedBuffer){
DISPLAY("\nError: not enough memory!\n");
free(orig_buff);
free(compressedBuffer);
free(chunkP);
fclose(inFile);
return 12;
}
/* Init chunks data */
{ unsigned i;
size_t remaining = benchedSize;
char* in = orig_buff;
char* out = compressedBuffer;
for (i=0; i<nbChunks; i++) {
chunkP[i].id = i;
chunkP[i].origBuffer = in; in += g_chunkSize;
if (remaining > (size_t)g_chunkSize) { chunkP[i].origSize = g_chunkSize; remaining -= g_chunkSize; } else { chunkP[i].origSize = (int)remaining; remaining = 0; }
chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
chunkP[i].compressedSize = 0;
} }
/* Fill input buffer */
DISPLAY("Loading %s... \r", inFileName);
if (strlen(inFileName)>16) inFileName += strlen(inFileName)-16; /* can only display 16 characters */
readSize = fread(orig_buff, 1, benchedSize, inFile);
fclose(inFile);
if (readSize != benchedSize) {
DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
free(orig_buff);
free(compressedBuffer);
free(chunkP);
return 13;
}
/* Calculating input Checksum */
crcOrig = XXH32(orig_buff, benchedSize,0);
/* Bench */
{ int loopNb;
size_t cSize = 0;
double fastestC = 100000000., fastestD = 100000000.;
double ratio = 0.;
U32 crcCheck = 0;
DISPLAY("\r%79s\r", "");
for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) {
int nbLoops = 0;
clock_t clockStart, clockEnd;
unsigned chunkNb;
/* Compression */
DISPLAY("%2i#%1i-%-14.14s : %9i ->\r", cLevel, loopNb, inFileName, (int)benchedSize);
{ size_t i; for (i=0; i<benchedSize; i++) compressedBuffer[i]=(char)i; } /* warmimg up memory */
clockStart = clock();
while (clock() == clockStart);
clockStart = clock();
while (BMK_getClockSpan(clockStart) < TIMELOOP_CLOCK) {
for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
chunkP[chunkNb].compressedSize = compP.compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, maxCompressedChunkSize, cLevel);
nbLoops++;
}
clockEnd = BMK_getClockSpan(clockStart);
nbLoops += !nbLoops; /* avoid division by zero */
if ((double)clockEnd < fastestC*nbLoops) fastestC = (double)clockEnd/nbLoops;
cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += chunkP[chunkNb].compressedSize;
ratio = (double)cSize/(double)benchedSize*100.;
DISPLAY("%2i#%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r",
cLevel, loopNb, inFileName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / (fastestC / CLOCKS_PER_SEC) / 1000000);
/* Decompression */
{ size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } /* zeroing area, for CRC checking */
nbLoops = 0;
clockStart = clock();
while (clock() == clockStart);
clockStart = clock();
while (BMK_getClockSpan(clockStart) < TIMELOOP_CLOCK) {
for (chunkNb=0; chunkNb<nbChunks; chunkNb++)
//chunkP[chunkNb].compressedSize = LZ4_decompress_fast(chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origBuffer, chunkP[chunkNb].origSize);
LZ4_decompress_safe(chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedSize, chunkP[chunkNb].origSize);
nbLoops++;
}
clockEnd = BMK_getClockSpan(clockStart);
nbLoops += !nbLoops; /* avoid division by zero */
if ((double)clockEnd < fastestD*nbLoops) fastestD = (double)clockEnd/nbLoops;
DISPLAY("%2i#%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s \r",
cLevel, loopNb, inFileName, (int)benchedSize, (int)cSize, ratio,
(double)benchedSize / (fastestC / CLOCKS_PER_SEC) / 1000000, (double)benchedSize / (fastestD / CLOCKS_PER_SEC) / 1000000 );
/* CRC Checking */
crcCheck = XXH32(orig_buff, benchedSize,0);
if (crcOrig!=crcCheck) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", inFileName, (unsigned)crcOrig, (unsigned)crcCheck); break; }
}
if (crcOrig==crcCheck) {
if (ratio < 100.)
DISPLAY("%2i#%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s \n",
cLevel, inFileName, (int)benchedSize, (int)cSize, ratio,
(double)benchedSize / (fastestC / CLOCKS_PER_SEC) / 1000000, (double)benchedSize / (fastestD / CLOCKS_PER_SEC) / 1000000 );
else
DISPLAY("%2i#%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n",
cLevel, inFileName, (int)benchedSize, (int)cSize, ratio,
(double)benchedSize / (fastestC / CLOCKS_PER_SEC) / 1000000, (double)benchedSize / (fastestD / CLOCKS_PER_SEC) / 1000000 );
}
totals += benchedSize;
totalz += cSize;
totalc += fastestC;
totald += fastestD;
}
free(orig_buff);
free(compressedBuffer);
free(chunkP);
}
if (nbFiles > 1)
DISPLAY("%2i#%-16.16s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s , %6.1f MB/s\n", cLevel, " TOTAL",
(long long unsigned)totals, (long long unsigned int)totalz, (double)totalz/(double)totals*100.,
(double)totals/(totalc/CLOCKS_PER_SEC)/1000000, (double)totals/(totald/CLOCKS_PER_SEC)/1000000);
if (BMK_pause) { DISPLAY("\npress enter...\n"); (void)getchar(); }
return 0;
}
int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel, int cLevelLast)
{
int i, res = 0;
if (cLevel > LZ4HC_MAX_CLEVEL) cLevel = LZ4HC_MAX_CLEVEL;
if (cLevelLast > LZ4HC_MAX_CLEVEL) cLevelLast = LZ4HC_MAX_CLEVEL;
if (cLevelLast < cLevel) cLevelLast = cLevel;
if (cLevelLast > cLevel) DISPLAY("Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
for (i=cLevel; i<=cLevelLast; i++) {
res = BMK_benchLevel(fileNamesTable, nbFiles, i);
if (res != 0) break;
for (l=cLevel; l <= cLevelLast; l++) {
BMK_benchMem(srcBuffer, benchedSize,
displayName, l,
fileSizes, nbFiles);
}
}
/*! BMK_loadFiles() :
Loads `buffer` with content of files listed within `fileNamesTable`.
At most, fills `buffer` entirely */
static void BMK_loadFiles(void* buffer, size_t bufferSize,
size_t* fileSizes,
const char** fileNamesTable, unsigned nbFiles)
{
size_t pos = 0, totalSize = 0;
unsigned n;
for (n=0; n<nbFiles; n++) {
FILE* f;
U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
if (UTIL_isDirectory(fileNamesTable[n])) {
DISPLAYLEVEL(2, "Ignoring %s directory... \n", fileNamesTable[n]);
fileSizes[n] = 0;
continue;
}
f = fopen(fileNamesTable[n], "rb");
if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
pos += readSize; }
fileSizes[n] = (size_t)fileSize;
totalSize += (size_t)fileSize;
fclose(f);
}
return res;
if (totalSize == 0) EXM_THROW(12, "no data to bench");
}
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
int cLevel, int cLevelLast)
{
void* srcBuffer;
size_t benchedSize;
size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
char mfName[20] = {0};
if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");
/* Memory allocation & restrictions */
benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
if (benchedSize==0) EXM_THROW(12, "not enough memory");
if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
if (benchedSize < totalSizeToLoad)
DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
srcBuffer = malloc(benchedSize);
if (!srcBuffer) EXM_THROW(12, "not enough memory");
/* Load input buffer */
BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
/* Bench */
snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
{ const char* displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
BMK_benchCLevel(srcBuffer, benchedSize,
displayName, cLevel, cLevelLast,
fileSizes, nbFiles);
}
/* clean up */
free(srcBuffer);
free(fileSizes);
}
static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility)
{
char name[20] = {0};
size_t benchedSize = 10000000;
void* const srcBuffer = malloc(benchedSize);
/* Memory allocation */
if (!srcBuffer) EXM_THROW(21, "not enough memory");
/* Fill input buffer */
RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
/* Bench */
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1);
/* clean up */
free(srcBuffer);
}
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
int cLevel, int cLevelLast)
{
double const compressibility = (double)g_compressibilityDefault / 100;
if (nbFiles == 0)
BMK_syntheticTest(cLevel, cLevelLast, compressibility);
else
BMK_benchFileTable(fileNamesTable, nbFiles, cLevel, cLevelLast);
return 0;
}

19
programs/bench.h
View File

@ -1,6 +1,6 @@
/*
bench.h - Demo program to benchmark open-source compression algorithm
Copyright (C) Yann Collet 2012-2015
Copyright (C) Yann Collet 2012-2016
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -20,15 +20,18 @@
- LZ4 source repository : https://github.com/lz4/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
#pragma once
#ifndef BENCH_H_125623623633
#define BENCH_H_125623623633
#include <stddef.h>
/* Main function */
int BMK_benchFiles(const char** fileNamesTable, int nbFiles, int cLevel, int cLevelLast);
int BMK_benchLevel(const char** fileNamesTable, int nbFiles, int cLevel);
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
int cLevel, int cLevelLast);
/* Set Parameters */
void BMK_setBlocksize(int bsize);
void BMK_setNbIterations(int nbLoops);
void BMK_setPause(void);
void BMK_SetNbSeconds(unsigned nbLoops);
void BMK_SetBlockSize(size_t blockSize);
void BMK_setAdditionalParam(int additionalParam);
void BMK_setNotificationLevel(unsigned level);
#endif /* BENCH_H_125623623633 */

29
programs/util.h
View File

@ -1,12 +1,25 @@
/**
* Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
/*
util.h - utility functions
Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
You can contact the author at :
- LZ4 source repository : https://github.com/lz4/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
#ifndef UTIL_H_MODULE
#define UTIL_H_MODULE