AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/origin/playTests' into dev

Browse Source
This commit is contained in:
inikep 2016-05-24 15:45:29 +02:00
commit c8f5509b32
5 changed files with 42 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
lib/common/zstd_internal.h
View File

@ -201,6 +201,16 @@ typedef struct {
U32 rep[ZSTD_REP_INIT];
} ZSTD_optimal_t;
#if ZSTD_OPT_DEBUG == 3
#include ".debug/zstd_stats.h"
#else
struct ZSTD_stats_s { U32 unused; };
MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; }
MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; }
MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; }
MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }
#endif // #if ZSTD_OPT_DEBUG == 3
typedef struct {
void* buffer;
U32* offsetStart;
@ -237,7 +247,7 @@ typedef struct {
U32 cachedPrice;
U32 cachedLitLength;
const BYTE* cachedLiterals;
ZSTD_stats_t* stats;
ZSTD_stats_t stats;
} seqStore_t;
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);

12
lib/compress/.debug/zstd_stats.h
View File

@ -38,8 +38,6 @@ extern "C" {
#endif
#if ZSTD_OPT_DEBUG == 3
/*-*************************************
* Types
***************************************/
@ -156,16 +154,6 @@ MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, co
stats->totalLitSum += litLength;
}
#else
struct ZSTD_stats_s { U32 unused; };
MEM_STATIC ZSTD_stats_t* ZSTD_statsAlloc(void) { return NULL; }
MEM_STATIC void ZSTD_statsFree(struct ZSTD_stats_s* stats) { (void)stats; }
MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; }
MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; }
MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; }
MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }
#endif // #if ZSTD_OPT_DEBUG == 3
#if defined (__cplusplus)
}

15
lib/compress/zstd_compress.c
View File

@ -57,7 +57,6 @@
#include "fse_static.h"
#include "huf_static.h"
#include "zstd_internal.h"
#include ".debug/zstd_stats.h"
/*-*************************************
@ -888,7 +887,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B
printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n",
pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
#endif
ZSTD_statsUpdatePrices(seqStorePtr->stats, litLength, literals, offsetCode, matchCode);
ZSTD_statsUpdatePrices(&seqStorePtr->stats, litLength, literals, offsetCode, matchCode);
/* copy Literals */
ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
@ -1780,7 +1779,7 @@ _storeSequence:
{ size_t const lastLLSize = iend - anchor;
memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize;
ZSTD_statsUpdatePrices(seqStorePtr->stats, lastLLSize, anchor, 0, 0);
ZSTD_statsUpdatePrices(&seqStorePtr->stats, lastLLSize, anchor, 0, 0);
}
}
@ -2052,15 +2051,14 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart;
const U32 maxDist = 1 << zc->params.cParams.windowLog;
ZSTD_stats_t* stats = ZSTD_statsAlloc();
zc->seqStore.stats = stats;
ZSTD_stats_t* stats = &zc->seqStore.stats;
ZSTD_statsInit(stats);
while (remaining) {
size_t cSize;
ZSTD_statsResetFreqs(stats);
if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) { ZSTD_statsFree(stats); return ERROR(dstSize_tooSmall); } /* not enough space to store compressed block */
if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
if (remaining < blockSize) blockSize = remaining;
if ((U32)(ip+blockSize - zc->base) > zc->loadedDictEnd + maxDist) {
@ -2071,11 +2069,11 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
}
cSize = ZSTD_compressBlock_internal(zc, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize);
if (ZSTD_isError(cSize)) { ZSTD_statsFree(stats); return cSize; }
if (ZSTD_isError(cSize)) return cSize;
if (cSize == 0) { /* block is not compressible */
cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize);
if (ZSTD_isError(cSize)) { ZSTD_statsFree(stats); return cSize; }
if (ZSTD_isError(cSize)) return cSize;
} else {
op[0] = (BYTE)(cSize>>16);
op[1] = (BYTE)(cSize>>8);
@ -2091,7 +2089,6 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* zc,
}
ZSTD_statsPrint(stats, zc->params.cParams.searchLength);
ZSTD_statsFree(stats);
return op-ostart;
}

51
programs/fullbench.c
View File

@ -213,19 +213,6 @@ size_t local_ZSTD_decompressContinue(void* dst, size_t dstCapacity, void* buff2,
/*_*******************************************************
* Bench functions
*********************************************************/
void* BMK_allocFunction(size_t size)
{
void* address = malloc(size);
/* printf("alloc %p, %d \n", address, (int)size); */
return address;
}
void BMK_freeFunction(void* address)
{
/* printf("free %p \n", address); */
free(address);
}
static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
{
BYTE* dstBuff;
@ -233,7 +220,6 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
BYTE* buff2;
const char* benchName;
size_t (*benchFunction)(void* dst, size_t dstSize, void* verifBuff, const void* src, size_t srcSize);
ZSTD_customMem customMem = { BMK_allocFunction, BMK_freeFunction };
double bestTime = 100000000.;
/* Selection */
@ -261,14 +247,8 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
benchFunction = local_ZBUFF_compress; benchName = "ZBUFF_compressContinue";
break;
case 42:
benchFunction = local_ZBUFF_compress; benchName = "ZBUFF_createCCtx_advanced/ZBUFF_compressContinue";
break;
case 43:
benchFunction = local_ZBUFF_decompress; benchName = "ZBUFF_decompressContinue";
break;
case 44:
benchFunction = local_ZBUFF_decompress; benchName = "ZBUFF_createDCtx_advanced/ZBUFF_decompressContinue";
break;
default :
return 0;
}
@ -343,16 +323,9 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
if (g_zbcc==NULL) g_zbcc = ZBUFF_createCCtx();
break;
case 42 :
if (g_zbcc==NULL) g_zbcc = ZBUFF_createCCtx_advanced(customMem);
break;
case 43 :
if (g_zbdc==NULL) g_zbdc = ZBUFF_createDCtx();
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
break;
case 44 :
if (g_zbdc==NULL) g_zbdc = ZBUFF_createDCtx_advanced(customMem);
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
break;
/* test functions */
/* by convention, test functions can be added > 100 */
@ -382,29 +355,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
averageTime = (((double)BMK_clockSpan(clockStart)) / CLOCKS_PER_SEC) / nbRounds;
if (averageTime < bestTime) bestTime = averageTime;
DISPLAY("%2i- %-30.30s : %7.1f MB/s (%9u)\r", loopNb, benchName, (double)srcSize / (1 MB) / bestTime, (U32)benchResult);
}
/* free allocated structures */
switch(benchNb)
{
case 11 :
if (g_zcc) { ZSTD_freeCCtx(g_zcc); g_zcc=NULL; }
break;
case 12 :
case 31:
case 32:
if (g_zdc) { ZSTD_freeDCtx(g_zdc); g_zdc=NULL; }
break;
case 41 :
case 42 :
if (g_zbcc) { ZBUFF_freeCCtx(g_zbcc); g_zbcc=NULL; }
break;
case 43 :
case 44 :
if (g_zbdc) { ZBUFF_freeDCtx(g_zbdc); g_zbdc=NULL; }
break;
default : ;
} }
}}
DISPLAY("%2u\n", benchNb);
_cleanOut:

28
programs/zbufftest.c
View File

@ -131,7 +131,20 @@ static unsigned FUZ_highbit32(U32 v32)
}
*/
static int basicUnitTests(U32 seed, double compressibility)
void* ZBUFF_allocFunction(size_t size)
{
void* address = malloc(size);
/* DISPLAYLEVEL(4, "alloc %p, %d \n", address, (int)size); */
return address;
}
void ZBUFF_freeFunction(void* address)
{
/* if (address) DISPLAYLEVEL(4, "free %p \n", address); */
free(address);
}
static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem)
{
int testResult = 0;
size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
@ -143,8 +156,8 @@ static int basicUnitTests(U32 seed, double compressibility)
U32 randState = seed;
size_t result, cSize, readSize, genSize;
U32 testNb=0;
ZBUFF_CCtx* zc = ZBUFF_createCCtx();
ZBUFF_DCtx* zd = ZBUFF_createDCtx();
ZBUFF_CCtx* zc = ZBUFF_createCCtx_advanced(customMem);
ZBUFF_DCtx* zd = ZBUFF_createDCtx_advanced(customMem);
/* Create compressible test buffer */
if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) {
@ -489,6 +502,8 @@ int main(int argc, const char** argv)
int result=0;
U32 mainPause = 0;
const char* programName = argv[0];
ZSTD_customMem customMem = { ZBUFF_allocFunction, ZBUFF_freeFunction };
ZSTD_customMem customNULL = { NULL, NULL };
/* Check command line */
for(argNb=1; argNb<argc; argNb++) {
@ -587,7 +602,12 @@ int main(int argc, const char** argv)
if (nbTests<=0) nbTests=1;
if (testNb==0) result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */
if (testNb==0) {
result = basicUnitTests(0, ((double)proba) / 100, customNULL); /* constant seed for predictability */
if (!result)
result = basicUnitTests(0, ((double)proba) / 100, customMem); /* use custom memory allocation functions */
}
if (!result)
result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);