Changed Library source tree
- no more zstdhc ; zstd.h is enough - ZSTD_compress() now needs a compressionLevel - separated zstd_compress.c and zstd_decompress.c - updated zstdcli, fullbench, fuzzer with new API
This commit is contained in:
Yann Collet
parent
530918b409
commit
5be2dd25f2
@ -57,7 +57,8 @@ unsigned ZSTD_versionNumber (void);
|
||||
* Simple functions
|
||||
***************************************/
|
||||
size_t ZSTD_compress( void* dst, size_t maxDstSize,
|
||||
const void* src, size_t srcSize);
|
||||
const void* src, size_t srcSize,
|
||||
int compressionLevel);
|
||||
|
||||
size_t ZSTD_decompress( void* dst, size_t maxOriginalSize,
|
||||
const void* src, size_t compressedSize);
|
||||
@ -100,7 +101,7 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
|
||||
ZSTD_compressCCtx() :
|
||||
Same as ZSTD_compress(), but requires a ZSTD_CCtx working space already allocated
|
||||
*/
|
||||
size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
|
||||
size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
|
||||
@ -55,7 +55,6 @@
|
||||
***************************************/
|
||||
#include <stdlib.h> /* malloc */
|
||||
#include <string.h> /* memset */
|
||||
#include "zstdhc_static.h"
|
||||
#include "zstd_static.h"
|
||||
#include "zstd_internal.h"
|
||||
#include "mem.h"
|
||||
@ -77,7 +76,7 @@
|
||||
#define BLOCKSIZE (128 KB) /* define, for static allocation */
|
||||
#define WORKPLACESIZE (BLOCKSIZE*3)
|
||||
|
||||
struct ZSTD_HC_CCtx_s
|
||||
struct ZSTD_CCtx_s
|
||||
{
|
||||
const BYTE* end; /* next block here to continue on current prefix */
|
||||
const BYTE* base; /* All regular indexes relative to this position */
|
||||
@ -85,7 +84,7 @@ struct ZSTD_HC_CCtx_s
|
||||
U32 dictLimit; /* below that point, need extDict */
|
||||
U32 lowLimit; /* below that point, no more data */
|
||||
U32 nextToUpdate; /* index from which to continue dictionary update */
|
||||
ZSTD_HC_parameters params;
|
||||
ZSTD_parameters params;
|
||||
void* workSpace;
|
||||
size_t workSpaceSize;
|
||||
|
||||
@ -95,12 +94,12 @@ struct ZSTD_HC_CCtx_s
|
||||
};
|
||||
|
||||
|
||||
ZSTD_HC_CCtx* ZSTD_HC_createCCtx(void)
|
||||
ZSTD_CCtx* ZSTD_createCCtx(void)
|
||||
{
|
||||
return (ZSTD_HC_CCtx*) calloc(1, sizeof(ZSTD_HC_CCtx));
|
||||
return (ZSTD_CCtx*) calloc(1, sizeof(ZSTD_CCtx));
|
||||
}
|
||||
|
||||
size_t ZSTD_HC_freeCCtx(ZSTD_HC_CCtx* cctx)
|
||||
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
|
||||
{
|
||||
free(cctx->workSpace);
|
||||
free(cctx);
|
||||
@ -108,45 +107,45 @@ size_t ZSTD_HC_freeCCtx(ZSTD_HC_CCtx* cctx)
|
||||
}
|
||||
|
||||
|
||||
/** ZSTD_HC_validateParams
|
||||
/** ZSTD_validateParams
|
||||
correct params value to remain within authorized range
|
||||
optimize for srcSize if srcSize > 0 */
|
||||
void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, U64 srcSizeHint)
|
||||
void ZSTD_validateParams(ZSTD_parameters* params, U64 srcSizeHint)
|
||||
{
|
||||
const U32 btPlus = (params->strategy == ZSTD_HC_btlazy2);
|
||||
const U32 btPlus = (params->strategy == ZSTD_btlazy2);
|
||||
|
||||
/* validate params */
|
||||
if (params->windowLog > ZSTD_HC_WINDOWLOG_MAX) params->windowLog = ZSTD_HC_WINDOWLOG_MAX;
|
||||
if (params->windowLog < ZSTD_HC_WINDOWLOG_MIN) params->windowLog = ZSTD_HC_WINDOWLOG_MIN;
|
||||
if (params->windowLog > ZSTD_WINDOWLOG_MAX) params->windowLog = ZSTD_WINDOWLOG_MAX;
|
||||
if (params->windowLog < ZSTD_WINDOWLOG_MIN) params->windowLog = ZSTD_WINDOWLOG_MIN;
|
||||
|
||||
/* correct params, to use less memory */
|
||||
if ((srcSizeHint > 0) && (srcSizeHint < (1<<ZSTD_HC_WINDOWLOG_MAX)))
|
||||
if ((srcSizeHint > 0) && (srcSizeHint < (1<<ZSTD_WINDOWLOG_MAX)))
|
||||
{
|
||||
U32 srcLog = ZSTD_highbit((U32)srcSizeHint-1) + 1;
|
||||
if (params->windowLog > srcLog) params->windowLog = srcLog;
|
||||
}
|
||||
|
||||
if (params->contentLog > params->windowLog+btPlus) params->contentLog = params->windowLog+btPlus; /* <= ZSTD_HC_CONTENTLOG_MAX */
|
||||
if (params->contentLog < ZSTD_HC_CONTENTLOG_MIN) params->contentLog = ZSTD_HC_CONTENTLOG_MIN;
|
||||
if (params->hashLog > ZSTD_HC_HASHLOG_MAX) params->hashLog = ZSTD_HC_HASHLOG_MAX;
|
||||
if (params->hashLog < ZSTD_HC_HASHLOG_MIN) params->hashLog = ZSTD_HC_HASHLOG_MIN;
|
||||
if (params->searchLog > ZSTD_HC_SEARCHLOG_MAX) params->searchLog = ZSTD_HC_SEARCHLOG_MAX;
|
||||
if (params->searchLog < ZSTD_HC_SEARCHLOG_MIN) params->searchLog = ZSTD_HC_SEARCHLOG_MIN;
|
||||
if (params->searchLength> ZSTD_HC_SEARCHLENGTH_MAX) params->searchLength = ZSTD_HC_SEARCHLENGTH_MAX;
|
||||
if (params->searchLength< ZSTD_HC_SEARCHLENGTH_MIN) params->searchLength = ZSTD_HC_SEARCHLENGTH_MIN;
|
||||
if ((U32)params->strategy>(U32)ZSTD_HC_btlazy2) params->strategy = ZSTD_HC_btlazy2;
|
||||
if (params->contentLog > params->windowLog+btPlus) params->contentLog = params->windowLog+btPlus; /* <= ZSTD_CONTENTLOG_MAX */
|
||||
if (params->contentLog < ZSTD_CONTENTLOG_MIN) params->contentLog = ZSTD_CONTENTLOG_MIN;
|
||||
if (params->hashLog > ZSTD_HASHLOG_MAX) params->hashLog = ZSTD_HASHLOG_MAX;
|
||||
if (params->hashLog < ZSTD_HASHLOG_MIN) params->hashLog = ZSTD_HASHLOG_MIN;
|
||||
if (params->searchLog > ZSTD_SEARCHLOG_MAX) params->searchLog = ZSTD_SEARCHLOG_MAX;
|
||||
if (params->searchLog < ZSTD_SEARCHLOG_MIN) params->searchLog = ZSTD_SEARCHLOG_MIN;
|
||||
if (params->searchLength> ZSTD_SEARCHLENGTH_MAX) params->searchLength = ZSTD_SEARCHLENGTH_MAX;
|
||||
if (params->searchLength< ZSTD_SEARCHLENGTH_MIN) params->searchLength = ZSTD_SEARCHLENGTH_MIN;
|
||||
if ((U32)params->strategy>(U32)ZSTD_btlazy2) params->strategy = ZSTD_btlazy2;
|
||||
}
|
||||
|
||||
|
||||
static size_t ZSTD_HC_resetCCtx_advanced (ZSTD_HC_CCtx* zc,
|
||||
ZSTD_HC_parameters params,
|
||||
static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
|
||||
ZSTD_parameters params,
|
||||
U64 srcSizeHint)
|
||||
{
|
||||
ZSTD_HC_validateParams(¶ms, srcSizeHint);
|
||||
ZSTD_validateParams(¶ms, srcSizeHint);
|
||||
|
||||
/* reserve table memory */
|
||||
{
|
||||
const U32 contentLog = params.strategy == ZSTD_HC_fast ? 1 : params.contentLog;
|
||||
const U32 contentLog = params.strategy == ZSTD_fast ? 1 : params.contentLog;
|
||||
const size_t tableSpace = ((1 << contentLog) + (1 << params.hashLog)) * sizeof(U32);
|
||||
const size_t neededSpace = tableSpace + WORKPLACESIZE;
|
||||
if (zc->workSpaceSize < neededSpace)
|
||||
@ -185,30 +184,30 @@ static size_t ZSTD_HC_resetCCtx_advanced (ZSTD_HC_CCtx* zc,
|
||||
***************************************/
|
||||
|
||||
static const U32 prime4bytes = 2654435761U;
|
||||
static U32 ZSTD_HC_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; }
|
||||
static size_t ZSTD_HC_hash4Ptr(const void* ptr, U32 h) { return ZSTD_HC_hash4(MEM_read32(ptr), h); }
|
||||
static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; }
|
||||
static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_read32(ptr), h); }
|
||||
|
||||
static const U64 prime5bytes = 889523592379ULL;
|
||||
static size_t ZSTD_HC_hash5(U64 u, U32 h) { return (size_t)((u * prime5bytes) << (64-40) >> (64-h)) ; }
|
||||
static size_t ZSTD_HC_hash5Ptr(const void* p, U32 h) { return ZSTD_HC_hash5(MEM_read64(p), h); }
|
||||
static size_t ZSTD_hash5(U64 u, U32 h) { return (size_t)((u * prime5bytes) << (64-40) >> (64-h)) ; }
|
||||
static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_read64(p), h); }
|
||||
|
||||
static const U64 prime6bytes = 227718039650203ULL;
|
||||
static size_t ZSTD_HC_hash6(U64 u, U32 h) { return (size_t)((u * prime6bytes) << (64-48) >> (64-h)) ; }
|
||||
static size_t ZSTD_HC_hash6Ptr(const void* p, U32 h) { return ZSTD_HC_hash6(MEM_read64(p), h); }
|
||||
static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)((u * prime6bytes) << (64-48) >> (64-h)) ; }
|
||||
static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_read64(p), h); }
|
||||
|
||||
static const U64 prime7bytes = 58295818150454627ULL;
|
||||
static size_t ZSTD_HC_hash7(U64 u, U32 h) { return (size_t)((u * prime7bytes) << (64-56) >> (64-h)) ; }
|
||||
static size_t ZSTD_HC_hash7Ptr(const void* p, U32 h) { return ZSTD_HC_hash7(MEM_read64(p), h); }
|
||||
static size_t ZSTD_hash7(U64 u, U32 h) { return (size_t)((u * prime7bytes) << (64-56) >> (64-h)) ; }
|
||||
static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_read64(p), h); }
|
||||
|
||||
static size_t ZSTD_HC_hashPtr(const void* p, U32 hBits, U32 mls)
|
||||
static size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
|
||||
{
|
||||
switch(mls)
|
||||
{
|
||||
default:
|
||||
case 4: return ZSTD_HC_hash4Ptr(p, hBits);
|
||||
case 5: return ZSTD_HC_hash5Ptr(p, hBits);
|
||||
case 6: return ZSTD_HC_hash6Ptr(p, hBits);
|
||||
case 7: return ZSTD_HC_hash7Ptr(p, hBits);
|
||||
case 4: return ZSTD_hash4Ptr(p, hBits);
|
||||
case 5: return ZSTD_hash5Ptr(p, hBits);
|
||||
case 6: return ZSTD_hash6Ptr(p, hBits);
|
||||
case 7: return ZSTD_hash7Ptr(p, hBits);
|
||||
}
|
||||
}
|
||||
|
||||
@ -217,7 +216,7 @@ static size_t ZSTD_HC_hashPtr(const void* p, U32 hBits, U32 mls)
|
||||
***************************************/
|
||||
|
||||
FORCE_INLINE
|
||||
size_t ZSTD_HC_compressBlock_fast_generic(ZSTD_HC_CCtx* ctx,
|
||||
size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* ctx,
|
||||
void* dst, size_t maxDstSize,
|
||||
const void* src, size_t srcSize,
|
||||
const U32 mls)
|
||||
@ -241,9 +240,9 @@ size_t ZSTD_HC_compressBlock_fast_generic(ZSTD_HC_CCtx* ctx,
|
||||
/* init */
|
||||
if (ip == base)
|
||||
{
|
||||
hashTable[ZSTD_HC_hashPtr(base+1, hBits, mls)] = 1;
|
||||
hashTable[ZSTD_HC_hashPtr(base+2, hBits, mls)] = 2;
|
||||
hashTable[ZSTD_HC_hashPtr(base+3, hBits, mls)] = 3;
|
||||
hashTable[ZSTD_hashPtr(base+1, hBits, mls)] = 1;
|
||||
hashTable[ZSTD_hashPtr(base+2, hBits, mls)] = 2;
|
||||
hashTable[ZSTD_hashPtr(base+3, hBits, mls)] = 3;
|
||||
ip = base+4;
|
||||
}
|
||||
ZSTD_resetSeqStore(seqStorePtr);
|
||||
@ -251,7 +250,7 @@ size_t ZSTD_HC_compressBlock_fast_generic(ZSTD_HC_CCtx* ctx,
|
||||
/* Main Search Loop */
|
||||
while (ip < ilimit) /* < instead of <=, because unconditionnal ZSTD_addPtr(ip+1) */
|
||||
{
|
||||
const size_t h = ZSTD_HC_hashPtr(ip, hBits, mls);
|
||||
const size_t h = ZSTD_hashPtr(ip, hBits, mls);
|
||||
const BYTE* match = base + hashTable[h];
|
||||
hashTable[h] = (U32)(ip-base);
|
||||
|
||||
@ -271,11 +270,11 @@ size_t ZSTD_HC_compressBlock_fast_generic(ZSTD_HC_CCtx* ctx,
|
||||
ZSTD_storeSeq(seqStorePtr, litLength, anchor, offsetCode, matchLength);
|
||||
|
||||
/* Fill Table */
|
||||
hashTable[ZSTD_HC_hashPtr(ip+1, hBits, mls)] = (U32)(ip+1-base);
|
||||
hashTable[ZSTD_hashPtr(ip+1, hBits, mls)] = (U32)(ip+1-base);
|
||||
ip += matchLength + MINMATCH;
|
||||
anchor = ip;
|
||||
if (ip < ilimit) /* same test as loop, for speed */
|
||||
hashTable[ZSTD_HC_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base);
|
||||
hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base);
|
||||
}
|
||||
}
|
||||
|
||||
@ -292,7 +291,7 @@ size_t ZSTD_HC_compressBlock_fast_generic(ZSTD_HC_CCtx* ctx,
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_HC_compressBlock_fast(ZSTD_HC_CCtx* ctx,
|
||||
size_t ZSTD_compressBlock_fast(ZSTD_CCtx* ctx,
|
||||
void* dst, size_t maxDstSize,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
@ -301,13 +300,13 @@ size_t ZSTD_HC_compressBlock_fast(ZSTD_HC_CCtx* ctx,
|
||||
{
|
||||
default:
|
||||
case 4 :
|
||||
return ZSTD_HC_compressBlock_fast_generic(ctx, dst, maxDstSize, src, srcSize, 4);
|
||||
return ZSTD_compressBlock_fast_generic(ctx, dst, maxDstSize, src, srcSize, 4);
|
||||
case 5 :
|
||||
return ZSTD_HC_compressBlock_fast_generic(ctx, dst, maxDstSize, src, srcSize, 5);
|
||||
return ZSTD_compressBlock_fast_generic(ctx, dst, maxDstSize, src, srcSize, 5);
|
||||
case 6 :
|
||||
return ZSTD_HC_compressBlock_fast_generic(ctx, dst, maxDstSize, src, srcSize, 6);
|
||||
return ZSTD_compressBlock_fast_generic(ctx, dst, maxDstSize, src, srcSize, 6);
|
||||
case 7 :
|
||||
return ZSTD_HC_compressBlock_fast_generic(ctx, dst, maxDstSize, src, srcSize, 7);
|
||||
return ZSTD_compressBlock_fast_generic(ctx, dst, maxDstSize, src, srcSize, 7);
|
||||
}
|
||||
}
|
||||
|
||||
@ -315,13 +314,13 @@ size_t ZSTD_HC_compressBlock_fast(ZSTD_HC_CCtx* ctx,
|
||||
/* *************************************
|
||||
* Binary Tree search
|
||||
***************************************/
|
||||
/** ZSTD_HC_insertBt1 : add one ptr to tree
|
||||
/** ZSTD_insertBt1 : add one ptr to tree
|
||||
@ip : assumed <= iend-8 */
|
||||
static U32 ZSTD_HC_insertBt1(ZSTD_HC_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares)
|
||||
static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares)
|
||||
{
|
||||
U32* const hashTable = zc->hashTable;
|
||||
const U32 hashLog = zc->params.hashLog;
|
||||
const size_t h = ZSTD_HC_hashPtr(ip, hashLog, mls);
|
||||
const size_t h = ZSTD_hashPtr(ip, hashLog, mls);
|
||||
U32* const bt = zc->contentTable;
|
||||
const U32 btLog = zc->params.contentLog - 1;
|
||||
const U32 btMask= (1 << btLog) - 1;
|
||||
@ -383,15 +382,15 @@ static U32 ZSTD_HC_insertBt1(ZSTD_HC_CCtx* zc, const BYTE* const ip, const U32 m
|
||||
|
||||
|
||||
FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */
|
||||
size_t ZSTD_HC_insertBtAndFindBestMatch (
|
||||
ZSTD_HC_CCtx* zc,
|
||||
size_t ZSTD_insertBtAndFindBestMatch (
|
||||
ZSTD_CCtx* zc,
|
||||
const BYTE* const ip, const BYTE* const iend,
|
||||
size_t* offsetPtr,
|
||||
U32 nbCompares, const U32 mls)
|
||||
{
|
||||
U32* const hashTable = zc->hashTable;
|
||||
const U32 hashLog = zc->params.hashLog;
|
||||
const size_t h = ZSTD_HC_hashPtr(ip, hashLog, mls);
|
||||
const size_t h = ZSTD_hashPtr(ip, hashLog, mls);
|
||||
U32* const bt = zc->contentTable;
|
||||
const U32 btLog = zc->params.contentLog - 1;
|
||||
const U32 btMask= (1 << btLog) - 1;
|
||||
@ -452,7 +451,7 @@ size_t ZSTD_HC_insertBtAndFindBestMatch (
|
||||
}
|
||||
|
||||
|
||||
static const BYTE* ZSTD_HC_updateTree(ZSTD_HC_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls)
|
||||
static const BYTE* ZSTD_updateTree(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls)
|
||||
{
|
||||
const BYTE* const base = zc->base;
|
||||
const U32 target = (U32)(ip - base);
|
||||
@ -460,8 +459,8 @@ static const BYTE* ZSTD_HC_updateTree(ZSTD_HC_CCtx* zc, const BYTE* const ip, co
|
||||
//size_t dummy;
|
||||
|
||||
for( ; idx < target ; )
|
||||
idx += ZSTD_HC_insertBt1(zc, base+idx, mls, iend, nbCompares);
|
||||
//ZSTD_HC_insertBtAndFindBestMatch(zc, base+idx, iend, &dummy, nbCompares, mls);
|
||||
idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares);
|
||||
//ZSTD_insertBtAndFindBestMatch(zc, base+idx, iend, &dummy, nbCompares, mls);
|
||||
|
||||
zc->nextToUpdate = idx;
|
||||
return base + idx;
|
||||
@ -470,25 +469,25 @@ static const BYTE* ZSTD_HC_updateTree(ZSTD_HC_CCtx* zc, const BYTE* const ip, co
|
||||
|
||||
/** Tree updater, providing best match */
|
||||
FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */
|
||||
size_t ZSTD_HC_BtFindBestMatch (
|
||||
ZSTD_HC_CCtx* zc,
|
||||
size_t ZSTD_BtFindBestMatch (
|
||||
ZSTD_CCtx* zc,
|
||||
const BYTE* const ip, const BYTE* const iLimit,
|
||||
size_t* offsetPtr,
|
||||
const U32 maxNbAttempts, const U32 mls)
|
||||
{
|
||||
const BYTE* nextToUpdate = ZSTD_HC_updateTree(zc, ip, iLimit, maxNbAttempts, mls);
|
||||
const BYTE* nextToUpdate = ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls);
|
||||
if (nextToUpdate > ip)
|
||||
{
|
||||
/* RLE data */
|
||||
*offsetPtr = 1;
|
||||
return ZSTD_count(ip, ip-1, iLimit);
|
||||
}
|
||||
return ZSTD_HC_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls);
|
||||
return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls);
|
||||
}
|
||||
|
||||
|
||||
FORCE_INLINE size_t ZSTD_HC_BtFindBestMatch_selectMLS (
|
||||
ZSTD_HC_CCtx* zc, /* Index table will be updated */
|
||||
FORCE_INLINE size_t ZSTD_BtFindBestMatch_selectMLS (
|
||||
ZSTD_CCtx* zc, /* Index table will be updated */
|
||||
const BYTE* ip, const BYTE* const iLimit,
|
||||
size_t* offsetPtr,
|
||||
const U32 maxNbAttempts, const U32 matchLengthSearch)
|
||||
@ -496,9 +495,9 @@ FORCE_INLINE size_t ZSTD_HC_BtFindBestMatch_selectMLS (
|
||||
switch(matchLengthSearch)
|
||||
{
|
||||
default :
|
||||
case 4 : return ZSTD_HC_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4);
|
||||
case 5 : return ZSTD_HC_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5);
|
||||
case 6 : return ZSTD_HC_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6);
|
||||
case 4 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4);
|
||||
case 5 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5);
|
||||
case 6 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6);
|
||||
}
|
||||
}
|
||||
|
||||
@ -510,7 +509,7 @@ FORCE_INLINE size_t ZSTD_HC_BtFindBestMatch_selectMLS (
|
||||
#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & mask]
|
||||
|
||||
/* Update chains up to ip (excluded) */
|
||||
static U32 ZSTD_HC_insertAndFindFirstIndex (ZSTD_HC_CCtx* zc, const BYTE* ip, U32 mls)
|
||||
static U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls)
|
||||
{
|
||||
U32* const hashTable = zc->hashTable;
|
||||
const U32 hashLog = zc->params.hashLog;
|
||||
@ -522,20 +521,20 @@ static U32 ZSTD_HC_insertAndFindFirstIndex (ZSTD_HC_CCtx* zc, const BYTE* ip, U
|
||||
|
||||
while(idx < target)
|
||||
{
|
||||
size_t h = ZSTD_HC_hashPtr(base+idx, hashLog, mls);
|
||||
size_t h = ZSTD_hashPtr(base+idx, hashLog, mls);
|
||||
NEXT_IN_CHAIN(idx, chainMask) = hashTable[h];
|
||||
hashTable[h] = idx;
|
||||
idx++;
|
||||
}
|
||||
|
||||
zc->nextToUpdate = target;
|
||||
return hashTable[ZSTD_HC_hashPtr(ip, hashLog, mls)];
|
||||
return hashTable[ZSTD_hashPtr(ip, hashLog, mls)];
|
||||
}
|
||||
|
||||
|
||||
FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */
|
||||
size_t ZSTD_HC_HcFindBestMatch (
|
||||
ZSTD_HC_CCtx* zc, /* Index table will be updated */
|
||||
size_t ZSTD_HcFindBestMatch (
|
||||
ZSTD_CCtx* zc, /* Index table will be updated */
|
||||
const BYTE* const ip, const BYTE* const iLimit,
|
||||
size_t* offsetPtr,
|
||||
const U32 maxNbAttempts, const U32 matchLengthSearch)
|
||||
@ -554,7 +553,7 @@ size_t ZSTD_HC_HcFindBestMatch (
|
||||
size_t ml=0;
|
||||
|
||||
/* HC4 match finder */
|
||||
matchIndex = ZSTD_HC_insertAndFindFirstIndex (zc, ip, matchLengthSearch);
|
||||
matchIndex = ZSTD_insertAndFindFirstIndex (zc, ip, matchLengthSearch);
|
||||
|
||||
while ((matchIndex>lowLimit) && (nbAttempts))
|
||||
{
|
||||
@ -596,8 +595,8 @@ size_t ZSTD_HC_HcFindBestMatch (
|
||||
}
|
||||
|
||||
|
||||
FORCE_INLINE size_t ZSTD_HC_HcFindBestMatch_selectMLS (
|
||||
ZSTD_HC_CCtx* zc, /* Index table will be updated */
|
||||
FORCE_INLINE size_t ZSTD_HcFindBestMatch_selectMLS (
|
||||
ZSTD_CCtx* zc, /* Index table will be updated */
|
||||
const BYTE* ip, const BYTE* const iLimit,
|
||||
size_t* offsetPtr,
|
||||
const U32 maxNbAttempts, const U32 matchLengthSearch)
|
||||
@ -605,16 +604,16 @@ FORCE_INLINE size_t ZSTD_HC_HcFindBestMatch_selectMLS (
|
||||
switch(matchLengthSearch)
|
||||
{
|
||||
default :
|
||||
case 4 : return ZSTD_HC_HcFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4);
|
||||
case 5 : return ZSTD_HC_HcFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5);
|
||||
case 6 : return ZSTD_HC_HcFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6);
|
||||
case 4 : return ZSTD_HcFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4);
|
||||
case 5 : return ZSTD_HcFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5);
|
||||
case 6 : return ZSTD_HcFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* common lazy function, to be inlined */
|
||||
FORCE_INLINE
|
||||
size_t ZSTD_HC_compressBlock_lazy_generic(ZSTD_HC_CCtx* ctx,
|
||||
size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
|
||||
void* dst, size_t maxDstSize, const void* src, size_t srcSize,
|
||||
const U32 searchMethod, const U32 deep) /* 0 : hc; 1 : bt */
|
||||
{
|
||||
@ -629,10 +628,10 @@ size_t ZSTD_HC_compressBlock_lazy_generic(ZSTD_HC_CCtx* ctx,
|
||||
const U32 maxSearches = 1 << ctx->params.searchLog;
|
||||
const U32 mls = ctx->params.searchLength;
|
||||
|
||||
typedef size_t (*searchMax_f)(ZSTD_HC_CCtx* zc, const BYTE* ip, const BYTE* iLimit,
|
||||
typedef size_t (*searchMax_f)(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iLimit,
|
||||
size_t* offsetPtr,
|
||||
U32 maxNbAttempts, U32 matchLengthSearch);
|
||||
searchMax_f searchMax = searchMethod ? ZSTD_HC_BtFindBestMatch_selectMLS : ZSTD_HC_HcFindBestMatch_selectMLS;
|
||||
searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS;
|
||||
|
||||
/* init */
|
||||
ZSTD_resetSeqStore(seqStorePtr);
|
||||
@ -750,23 +749,23 @@ size_t ZSTD_HC_compressBlock_lazy_generic(ZSTD_HC_CCtx* ctx,
|
||||
seqStorePtr, srcSize);
|
||||
}
|
||||
|
||||
size_t ZSTD_HC_compressBlock_btlazy2(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
size_t ZSTD_compressBlock_btlazy2(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
return ZSTD_HC_compressBlock_lazy_generic(ctx, dst, maxDstSize, src, srcSize, 1, 1);
|
||||
return ZSTD_compressBlock_lazy_generic(ctx, dst, maxDstSize, src, srcSize, 1, 1);
|
||||
}
|
||||
|
||||
size_t ZSTD_HC_compressBlock_lazy2(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
size_t ZSTD_compressBlock_lazy2(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
return ZSTD_HC_compressBlock_lazy_generic(ctx, dst, maxDstSize, src, srcSize, 0, 1);
|
||||
return ZSTD_compressBlock_lazy_generic(ctx, dst, maxDstSize, src, srcSize, 0, 1);
|
||||
}
|
||||
|
||||
size_t ZSTD_HC_compressBlock_lazy(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
size_t ZSTD_compressBlock_lazy(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
return ZSTD_HC_compressBlock_lazy_generic(ctx, dst, maxDstSize, src, srcSize, 0, 0);
|
||||
return ZSTD_compressBlock_lazy_generic(ctx, dst, maxDstSize, src, srcSize, 0, 0);
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_HC_compressBlock_greedy(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
size_t ZSTD_compressBlock_greedy(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
seqStore_t* seqStorePtr = &(ctx->seqStore);
|
||||
const BYTE* const istart = (const BYTE*)src;
|
||||
@ -817,7 +816,7 @@ size_t ZSTD_HC_compressBlock_greedy(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstS
|
||||
/* search */
|
||||
{
|
||||
size_t offset=999999;
|
||||
size_t matchLength = ZSTD_HC_HcFindBestMatch_selectMLS(ctx, ip, iend, &offset, maxSearches, mls);
|
||||
size_t matchLength = ZSTD_HcFindBestMatch_selectMLS(ctx, ip, iend, &offset, maxSearches, mls);
|
||||
if (matchLength < MINMATCH)
|
||||
{
|
||||
ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */
|
||||
@ -849,35 +848,35 @@ size_t ZSTD_HC_compressBlock_greedy(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstS
|
||||
}
|
||||
|
||||
|
||||
typedef size_t (*ZSTD_HC_blockCompressor) (ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
|
||||
typedef size_t (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
|
||||
|
||||
static ZSTD_HC_blockCompressor ZSTD_HC_selectBlockCompressor(ZSTD_HC_strategy strat)
|
||||
static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat)
|
||||
{
|
||||
switch(strat)
|
||||
{
|
||||
default :
|
||||
case ZSTD_HC_fast:
|
||||
return ZSTD_HC_compressBlock_fast;
|
||||
case ZSTD_HC_greedy:
|
||||
return ZSTD_HC_compressBlock_greedy;
|
||||
case ZSTD_HC_lazy:
|
||||
return ZSTD_HC_compressBlock_lazy;
|
||||
case ZSTD_HC_lazy2:
|
||||
return ZSTD_HC_compressBlock_lazy2;
|
||||
case ZSTD_HC_btlazy2:
|
||||
return ZSTD_HC_compressBlock_btlazy2;
|
||||
case ZSTD_fast:
|
||||
return ZSTD_compressBlock_fast;
|
||||
case ZSTD_greedy:
|
||||
return ZSTD_compressBlock_greedy;
|
||||
case ZSTD_lazy:
|
||||
return ZSTD_compressBlock_lazy;
|
||||
case ZSTD_lazy2:
|
||||
return ZSTD_compressBlock_lazy2;
|
||||
case ZSTD_btlazy2:
|
||||
return ZSTD_compressBlock_btlazy2;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_HC_compressBlock(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
size_t ZSTD_compressBlock(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
ZSTD_HC_blockCompressor blockCompressor = ZSTD_HC_selectBlockCompressor(ctx->params.strategy);
|
||||
ZSTD_blockCompressor blockCompressor = ZSTD_selectBlockCompressor(ctx->params.strategy);
|
||||
return blockCompressor(ctx, dst, maxDstSize, src, srcSize);
|
||||
}
|
||||
|
||||
|
||||
static size_t ZSTD_HC_compress_generic (ZSTD_HC_CCtx* ctxPtr,
|
||||
static size_t ZSTD_compress_generic (ZSTD_CCtx* ctxPtr,
|
||||
void* dst, size_t maxDstSize,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
@ -886,7 +885,7 @@ static size_t ZSTD_HC_compress_generic (ZSTD_HC_CCtx* ctxPtr,
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* op = ostart;
|
||||
const ZSTD_HC_blockCompressor blockCompressor = ZSTD_HC_selectBlockCompressor(ctxPtr->params.strategy);
|
||||
const ZSTD_blockCompressor blockCompressor = ZSTD_selectBlockCompressor(ctxPtr->params.strategy);
|
||||
|
||||
while (remaining)
|
||||
{
|
||||
@ -921,7 +920,7 @@ static size_t ZSTD_HC_compress_generic (ZSTD_HC_CCtx* ctxPtr,
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_HC_compressContinue (ZSTD_HC_CCtx* ctxPtr,
|
||||
size_t ZSTD_compressContinue (ZSTD_CCtx* ctxPtr,
|
||||
void* dst, size_t dstSize,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
@ -931,39 +930,39 @@ size_t ZSTD_HC_compressContinue (ZSTD_HC_CCtx* ctxPtr,
|
||||
if (ip != ctxPtr->end)
|
||||
{
|
||||
if (ctxPtr->end != NULL)
|
||||
ZSTD_HC_resetCCtx_advanced(ctxPtr, ctxPtr->params, srcSize);
|
||||
ZSTD_resetCCtx_advanced(ctxPtr, ctxPtr->params, srcSize);
|
||||
ctxPtr->base = ip;
|
||||
}
|
||||
|
||||
ctxPtr->end = ip + srcSize;
|
||||
return ZSTD_HC_compress_generic (ctxPtr, dst, dstSize, src, srcSize);
|
||||
return ZSTD_compress_generic (ctxPtr, dst, dstSize, src, srcSize);
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_HC_compressBegin_advanced(ZSTD_HC_CCtx* ctx,
|
||||
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* ctx,
|
||||
void* dst, size_t maxDstSize,
|
||||
const ZSTD_HC_parameters params,
|
||||
const ZSTD_parameters params,
|
||||
U64 srcSizeHint)
|
||||
{
|
||||
size_t errorCode;
|
||||
if (maxDstSize < 4) return ERROR(dstSize_tooSmall);
|
||||
errorCode = ZSTD_HC_resetCCtx_advanced(ctx, params, srcSizeHint);
|
||||
errorCode = ZSTD_resetCCtx_advanced(ctx, params, srcSizeHint);
|
||||
if (ZSTD_isError(errorCode)) return errorCode;
|
||||
MEM_writeLE32(dst, ZSTD_magicNumber); /* Write Header */
|
||||
return 4;
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint)
|
||||
size_t ZSTD_compressBegin(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint)
|
||||
{
|
||||
int tableID = ((srcSizeHint-1) > 128 KB); /* intentional underflow for 0 */
|
||||
if (compressionLevel<=0) compressionLevel = 1;
|
||||
if (compressionLevel > ZSTD_HC_MAX_CLEVEL) compressionLevel = ZSTD_HC_MAX_CLEVEL;
|
||||
return ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_HC_defaultParameters[tableID][compressionLevel], srcSizeHint);
|
||||
if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
|
||||
return ZSTD_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_defaultParameters[tableID][compressionLevel], srcSizeHint);
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_HC_compressEnd(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize)
|
||||
size_t ZSTD_compressEnd(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize)
|
||||
{
|
||||
BYTE* op = (BYTE*)dst;
|
||||
|
||||
@ -979,10 +978,10 @@ size_t ZSTD_HC_compressEnd(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize)
|
||||
return 3;
|
||||
}
|
||||
|
||||
size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx,
|
||||
size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
|
||||
void* dst, size_t maxDstSize,
|
||||
const void* src, size_t srcSize,
|
||||
ZSTD_HC_parameters params)
|
||||
ZSTD_parameters params)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* op = ostart;
|
||||
@ -991,46 +990,47 @@ size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx,
|
||||
/* correct params, to use less memory */
|
||||
{
|
||||
U32 srcLog = ZSTD_highbit((U32)srcSize-1) + 1;
|
||||
U32 contentBtPlus = (ctx->params.strategy == ZSTD_HC_btlazy2);
|
||||
U32 contentBtPlus = (ctx->params.strategy == ZSTD_btlazy2);
|
||||
if (params.windowLog > srcLog) params.windowLog = srcLog;
|
||||
if (params.contentLog > srcLog+contentBtPlus) params.contentLog = srcLog+contentBtPlus;
|
||||
}
|
||||
|
||||
/* Header */
|
||||
oSize = ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, params, srcSize);
|
||||
oSize = ZSTD_compressBegin_advanced(ctx, dst, maxDstSize, params, srcSize);
|
||||
if(ZSTD_isError(oSize)) return oSize;
|
||||
op += oSize;
|
||||
maxDstSize -= oSize;
|
||||
|
||||
/* body (compression) */
|
||||
ctx->base = (const BYTE*)src;
|
||||
oSize = ZSTD_HC_compress_generic (ctx, op, maxDstSize, src, srcSize);
|
||||
oSize = ZSTD_compress_generic (ctx, op, maxDstSize, src, srcSize);
|
||||
if(ZSTD_isError(oSize)) return oSize;
|
||||
op += oSize;
|
||||
maxDstSize -= oSize;
|
||||
|
||||
/* Close frame */
|
||||
oSize = ZSTD_HC_compressEnd(ctx, op, maxDstSize);
|
||||
oSize = ZSTD_compressEnd(ctx, op, maxDstSize);
|
||||
if(ZSTD_isError(oSize)) return oSize;
|
||||
op += oSize;
|
||||
|
||||
return (op - ostart);
|
||||
}
|
||||
|
||||
size_t ZSTD_HC_compressCCtx (ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
|
||||
size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
|
||||
{
|
||||
const int tableID = (srcSize > 128 KB);
|
||||
if (compressionLevel<=1) return ZSTD_compress(dst, maxDstSize, src, srcSize); /* fast mode */
|
||||
if (compressionLevel > ZSTD_HC_MAX_CLEVEL) compressionLevel = ZSTD_HC_MAX_CLEVEL;
|
||||
return ZSTD_HC_compress_advanced(ctx, dst, maxDstSize, src, srcSize, ZSTD_HC_defaultParameters[tableID][compressionLevel]);
|
||||
//if (compressionLevel<=1) return ZSTD_compress(dst, maxDstSize, src, srcSize); /* fast mode */
|
||||
if (compressionLevel < 1) compressionLevel = 1;
|
||||
if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
|
||||
return ZSTD_compress_advanced(ctx, dst, maxDstSize, src, srcSize, ZSTD_defaultParameters[tableID][compressionLevel]);
|
||||
}
|
||||
|
||||
size_t ZSTD_HC_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
|
||||
size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
|
||||
{
|
||||
size_t result;
|
||||
ZSTD_HC_CCtx ctxBody;
|
||||
ZSTD_CCtx ctxBody;
|
||||
memset(&ctxBody, 0, sizeof(ctxBody));
|
||||
result = ZSTD_HC_compressCCtx(&ctxBody, dst, maxDstSize, src, srcSize, compressionLevel);
|
||||
result = ZSTD_compressCCtx(&ctxBody, dst, maxDstSize, src, srcSize, compressionLevel);
|
||||
free(ctxBody.workSpace);
|
||||
return result;
|
||||
}
|
||||
1103
lib/zstd_decompress.c
Normal file
1103
lib/zstd_decompress.c
Normal file
File diff suppressed because it is too large
Load Diff
@ -46,12 +46,46 @@ extern "C" {
|
||||
* Includes
|
||||
***************************************/
|
||||
#include "zstd.h"
|
||||
#include "mem.h"
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Types
|
||||
***************************************/
|
||||
/** from faster to stronger */
|
||||
typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2 } ZSTD_strategy;
|
||||
|
||||
typedef struct
|
||||
{
|
||||
U32 windowLog; /* largest match distance : impact decompression buffer size */
|
||||
U32 contentLog; /* full search segment : larger == more compression, slower, more memory (useless for fast) */
|
||||
U32 hashLog; /* dispatch table : larger == more memory, faster*/
|
||||
U32 searchLog; /* nb of searches : larger == more compression, slower*/
|
||||
U32 searchLength; /* size of matches : larger == faster decompression */
|
||||
ZSTD_strategy strategy;
|
||||
} ZSTD_parameters;
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Advanced function
|
||||
***************************************/
|
||||
/** ZSTD_compress_advanced
|
||||
* Same as ZSTD_compressCCtx(), with fine-tune control of each compression parameter */
|
||||
size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
|
||||
void* dst, size_t maxDstSize,
|
||||
const void* src, size_t srcSize,
|
||||
ZSTD_parameters params);
|
||||
|
||||
/** ZSTD_validateParams
|
||||
correct params value to remain within authorized range
|
||||
srcSizeHint value is optional, select 0 if not known */
|
||||
void ZSTD_validateParams(ZSTD_parameters* params, U64 srcSizeHint);
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Streaming functions
|
||||
***************************************/
|
||||
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize);
|
||||
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint);
|
||||
size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
|
||||
size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t maxDstSize);
|
||||
|
||||
@ -77,6 +111,73 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
|
||||
#define ZSTD_magicNumber 0xFD2FB523 /* v0.3 (current)*/
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Pre-defined compression levels
|
||||
***************************************/
|
||||
#define ZSTD_MAX_CLEVEL 20
|
||||
#define ZSTD_WINDOWLOG_MAX 26
|
||||
#define ZSTD_WINDOWLOG_MIN 18
|
||||
#define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
|
||||
#define ZSTD_CONTENTLOG_MIN 4
|
||||
#define ZSTD_HASHLOG_MAX 28
|
||||
#define ZSTD_HASHLOG_MIN 4
|
||||
#define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1)
|
||||
#define ZSTD_SEARCHLOG_MIN 1
|
||||
#define ZSTD_SEARCHLENGTH_MAX 7
|
||||
#define ZSTD_SEARCHLENGTH_MIN 4
|
||||
|
||||
static const ZSTD_parameters ZSTD_defaultParameters[2][ZSTD_MAX_CLEVEL+1] = {
|
||||
{ /* for <= 128 KB */
|
||||
/* W, C, H, S, L, strat */
|
||||
{ 17, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */
|
||||
{ 17, 12, 13, 1, 6, ZSTD_fast }, /* level 1 */
|
||||
{ 17, 15, 16, 1, 5, ZSTD_fast }, /* level 2 */
|
||||
{ 17, 16, 17, 1, 5, ZSTD_fast }, /* level 3 */
|
||||
{ 17, 13, 15, 2, 4, ZSTD_greedy }, /* level 4 */
|
||||
{ 17, 15, 17, 3, 4, ZSTD_greedy }, /* level 5 */
|
||||
{ 17, 14, 17, 3, 4, ZSTD_lazy }, /* level 6 */
|
||||
{ 17, 16, 17, 4, 4, ZSTD_lazy }, /* level 7 */
|
||||
{ 17, 16, 17, 4, 4, ZSTD_lazy2 }, /* level 8 */
|
||||
{ 17, 17, 16, 5, 4, ZSTD_lazy2 }, /* level 9 */
|
||||
{ 17, 17, 16, 6, 4, ZSTD_lazy2 }, /* level 10 */
|
||||
{ 17, 17, 16, 7, 4, ZSTD_lazy2 }, /* level 11 */
|
||||
{ 17, 17, 16, 8, 4, ZSTD_lazy2 }, /* level 12 */
|
||||
{ 17, 18, 16, 4, 4, ZSTD_btlazy2 }, /* level 13 */
|
||||
{ 17, 18, 16, 5, 4, ZSTD_btlazy2 }, /* level 14 */
|
||||
{ 17, 18, 16, 6, 4, ZSTD_btlazy2 }, /* level 15 */
|
||||
{ 17, 18, 16, 7, 4, ZSTD_btlazy2 }, /* level 16 */
|
||||
{ 17, 18, 16, 8, 4, ZSTD_btlazy2 }, /* level 17 */
|
||||
{ 17, 18, 16, 9, 4, ZSTD_btlazy2 }, /* level 18 */
|
||||
{ 17, 18, 16, 10, 4, ZSTD_btlazy2 }, /* level 19 */
|
||||
{ 17, 18, 18, 12, 4, ZSTD_btlazy2 }, /* level 20 */
|
||||
},
|
||||
{ /* for > 128 KB */
|
||||
/* W, C, H, S, L, strat */
|
||||
{ 18, 12, 12, 1, 4, ZSTD_fast }, /* level 0 - never used */
|
||||
{ 18, 14, 14, 1, 7, ZSTD_fast }, /* level 1 - in fact redirected towards zstd fast */
|
||||
{ 19, 15, 16, 1, 6, ZSTD_fast }, /* level 2 */
|
||||
{ 20, 18, 20, 1, 6, ZSTD_fast }, /* level 3 */
|
||||
{ 21, 19, 21, 1, 6, ZSTD_fast }, /* level 4 */
|
||||
{ 20, 13, 18, 5, 5, ZSTD_greedy }, /* level 5 */
|
||||
{ 20, 17, 19, 3, 5, ZSTD_greedy }, /* level 6 */
|
||||
{ 21, 17, 20, 3, 5, ZSTD_lazy }, /* level 7 */
|
||||
{ 21, 19, 20, 3, 5, ZSTD_lazy }, /* level 8 */
|
||||
{ 21, 20, 20, 3, 5, ZSTD_lazy2 }, /* level 9 */
|
||||
{ 21, 19, 20, 4, 5, ZSTD_lazy2 }, /* level 10 */
|
||||
{ 22, 20, 22, 4, 5, ZSTD_lazy2 }, /* level 11 */
|
||||
{ 22, 20, 22, 5, 5, ZSTD_lazy2 }, /* level 12 */
|
||||
{ 22, 21, 22, 5, 5, ZSTD_lazy2 }, /* level 13 */
|
||||
{ 22, 22, 23, 5, 5, ZSTD_lazy2 }, /* level 14 */
|
||||
{ 23, 23, 23, 5, 5, ZSTD_lazy2 }, /* level 15 */
|
||||
{ 23, 21, 22, 5, 5, ZSTD_btlazy2 }, /* level 16 */
|
||||
{ 23, 24, 23, 4, 5, ZSTD_btlazy2 }, /* level 17 */
|
||||
{ 25, 24, 23, 5, 5, ZSTD_btlazy2 }, /* level 18 */
|
||||
{ 25, 26, 23, 5, 5, ZSTD_btlazy2 }, /* level 19 */
|
||||
{ 26, 27, 24, 6, 5, ZSTD_btlazy2 }, /* level 20 */
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Error management
|
||||
***************************************/
|
||||
|
||||
76
lib/zstdhc.h
76
lib/zstdhc.h
@ -1,76 +0,0 @@
|
||||
/*
|
||||
zstdhc - high compression variant
|
||||
Header File
|
||||
Copyright (C) 2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd source repository : http://www.zstd.net
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* *************************************
|
||||
* Includes
|
||||
***************************************/
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Simple function
|
||||
***************************************/
|
||||
/**
|
||||
ZSTD_HC_compress() :
|
||||
Compresses 'srcSize' bytes from buffer 'src' into buffer 'dst', of maximum size 'dstSize'.
|
||||
Destination buffer must be already allocated.
|
||||
Compression runs faster if maxDstSize >= ZSTD_compressBound(srcSize).
|
||||
@return : the number of bytes written into buffer 'dst'
|
||||
or an error code if it fails (which can be tested using ZSTD_isError())
|
||||
*/
|
||||
size_t ZSTD_HC_compress(void* dst, size_t maxDstSize,
|
||||
const void* src, size_t srcSize,
|
||||
int compressionLevel);
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Advanced functions
|
||||
***************************************/
|
||||
typedef struct ZSTD_HC_CCtx_s ZSTD_HC_CCtx; /* incomplete type */
|
||||
ZSTD_HC_CCtx* ZSTD_HC_createCCtx(void);
|
||||
size_t ZSTD_HC_freeCCtx(ZSTD_HC_CCtx* cctx);
|
||||
|
||||
/**
|
||||
ZSTD_HC_compressCCtx() :
|
||||
Same as ZSTD_compress(), but requires a ZSTD_HC_CCtx working space already allocated
|
||||
*/
|
||||
size_t ZSTD_HC_compressCCtx(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
@ -1,156 +0,0 @@
|
||||
/*
|
||||
zstdhc - high compression variant
|
||||
Header File - Experimental API, static linking only
|
||||
Copyright (C) 2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd source repository : http://www.zstd.net
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* *************************************
|
||||
* Includes
|
||||
***************************************/
|
||||
#include "mem.h"
|
||||
#include "zstdhc.h"
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Types
|
||||
***************************************/
|
||||
/** from faster to stronger */
|
||||
typedef enum { ZSTD_HC_fast, ZSTD_HC_greedy, ZSTD_HC_lazy, ZSTD_HC_lazy2, ZSTD_HC_btlazy2 } ZSTD_HC_strategy;
|
||||
|
||||
typedef struct
|
||||
{
|
||||
U32 windowLog; /* largest match distance : impact decompression buffer size */
|
||||
U32 contentLog; /* full search segment : larger == more compression, slower, more memory (useless for fast) */
|
||||
U32 hashLog; /* dispatch table : larger == more memory, faster*/
|
||||
U32 searchLog; /* nb of searches : larger == more compression, slower*/
|
||||
U32 searchLength; /* size of matches : larger == faster decompression */
|
||||
ZSTD_HC_strategy strategy;
|
||||
} ZSTD_HC_parameters;
|
||||
|
||||
/* parameters boundaries */
|
||||
#define ZSTD_HC_WINDOWLOG_MAX 26
|
||||
#define ZSTD_HC_WINDOWLOG_MIN 18
|
||||
#define ZSTD_HC_CONTENTLOG_MAX (ZSTD_HC_WINDOWLOG_MAX+1)
|
||||
#define ZSTD_HC_CONTENTLOG_MIN 4
|
||||
#define ZSTD_HC_HASHLOG_MAX 28
|
||||
#define ZSTD_HC_HASHLOG_MIN 4
|
||||
#define ZSTD_HC_SEARCHLOG_MAX (ZSTD_HC_CONTENTLOG_MAX-1)
|
||||
#define ZSTD_HC_SEARCHLOG_MIN 1
|
||||
#define ZSTD_HC_SEARCHLENGTH_MAX 7
|
||||
#define ZSTD_HC_SEARCHLENGTH_MIN 4
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Advanced function
|
||||
***************************************/
|
||||
/** ZSTD_HC_compress_advanced
|
||||
* Same as ZSTD_HC_compressCCtx(), with fine-tune control of each compression parameter */
|
||||
size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx,
|
||||
void* dst, size_t maxDstSize,
|
||||
const void* src, size_t srcSize,
|
||||
ZSTD_HC_parameters params);
|
||||
|
||||
/** ZSTD_HC_validateParams
|
||||
correct params value to remain within authorized range
|
||||
srcSizeHint value is optional, select 0 if not known */
|
||||
void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, U64 srcSizeHint);
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Streaming functions
|
||||
***************************************/
|
||||
size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint);
|
||||
size_t ZSTD_HC_compressContinue(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
|
||||
size_t ZSTD_HC_compressEnd(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize);
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Pre-defined compression levels
|
||||
***************************************/
|
||||
#define ZSTD_HC_MAX_CLEVEL 20
|
||||
static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[2][ZSTD_HC_MAX_CLEVEL+1] = {
|
||||
{ /* for <= 128 KB */
|
||||
/* W, C, H, S, L, strat */
|
||||
{ 17, 12, 12, 1, 4, ZSTD_HC_fast }, /* level 0 - never used */
|
||||
{ 17, 12, 13, 1, 6, ZSTD_HC_fast }, /* level 1 */
|
||||
{ 17, 15, 16, 1, 5, ZSTD_HC_fast }, /* level 2 */
|
||||
{ 17, 16, 17, 1, 5, ZSTD_HC_fast }, /* level 3 */
|
||||
{ 17, 13, 15, 2, 4, ZSTD_HC_greedy }, /* level 4 */
|
||||
{ 17, 15, 17, 3, 4, ZSTD_HC_greedy }, /* level 5 */
|
||||
{ 17, 14, 17, 3, 4, ZSTD_HC_lazy }, /* level 6 */
|
||||
{ 17, 16, 17, 4, 4, ZSTD_HC_lazy }, /* level 7 */
|
||||
{ 17, 16, 17, 4, 4, ZSTD_HC_lazy2 }, /* level 8 */
|
||||
{ 17, 17, 16, 5, 4, ZSTD_HC_lazy2 }, /* level 9 */
|
||||
{ 17, 17, 16, 6, 4, ZSTD_HC_lazy2 }, /* level 10 */
|
||||
{ 17, 17, 16, 7, 4, ZSTD_HC_lazy2 }, /* level 11 */
|
||||
{ 17, 17, 16, 8, 4, ZSTD_HC_lazy2 }, /* level 12 */
|
||||
{ 17, 18, 16, 4, 4, ZSTD_HC_btlazy2 }, /* level 13 */
|
||||
{ 17, 18, 16, 5, 4, ZSTD_HC_btlazy2 }, /* level 14 */
|
||||
{ 17, 18, 16, 6, 4, ZSTD_HC_btlazy2 }, /* level 15 */
|
||||
{ 17, 18, 16, 7, 4, ZSTD_HC_btlazy2 }, /* level 16 */
|
||||
{ 17, 18, 16, 8, 4, ZSTD_HC_btlazy2 }, /* level 17 */
|
||||
{ 17, 18, 16, 9, 4, ZSTD_HC_btlazy2 }, /* level 18 */
|
||||
{ 17, 18, 16, 10, 4, ZSTD_HC_btlazy2 }, /* level 19 */
|
||||
{ 17, 18, 18, 12, 4, ZSTD_HC_btlazy2 }, /* level 20 */
|
||||
},
|
||||
{ /* for > 128 KB */
|
||||
/* W, C, H, S, L, strat */
|
||||
{ 18, 12, 12, 1, 4, ZSTD_HC_fast }, /* level 0 - never used */
|
||||
{ 18, 14, 14, 1, 7, ZSTD_HC_fast }, /* level 1 - in fact redirected towards zstd fast */
|
||||
{ 19, 15, 16, 1, 6, ZSTD_HC_fast }, /* level 2 */
|
||||
{ 20, 18, 20, 1, 6, ZSTD_HC_fast }, /* level 3 */
|
||||
{ 21, 19, 21, 1, 6, ZSTD_HC_fast }, /* level 4 */
|
||||
{ 20, 13, 18, 5, 5, ZSTD_HC_greedy }, /* level 5 */
|
||||
{ 20, 17, 19, 3, 5, ZSTD_HC_greedy }, /* level 6 */
|
||||
{ 21, 17, 20, 3, 5, ZSTD_HC_lazy }, /* level 7 */
|
||||
{ 21, 19, 20, 3, 5, ZSTD_HC_lazy }, /* level 8 */
|
||||
{ 21, 20, 20, 3, 5, ZSTD_HC_lazy2 }, /* level 9 */
|
||||
{ 21, 19, 20, 4, 5, ZSTD_HC_lazy2 }, /* level 10 */
|
||||
{ 22, 20, 22, 4, 5, ZSTD_HC_lazy2 }, /* level 11 */
|
||||
{ 22, 20, 22, 5, 5, ZSTD_HC_lazy2 }, /* level 12 */
|
||||
{ 22, 21, 22, 5, 5, ZSTD_HC_lazy2 }, /* level 13 */
|
||||
{ 22, 22, 23, 5, 5, ZSTD_HC_lazy2 }, /* level 14 */
|
||||
{ 23, 23, 23, 5, 5, ZSTD_HC_lazy2 }, /* level 15 */
|
||||
{ 23, 21, 22, 5, 5, ZSTD_HC_btlazy2 }, /* level 16 */
|
||||
{ 23, 24, 23, 4, 5, ZSTD_HC_btlazy2 }, /* level 17 */
|
||||
{ 25, 24, 23, 5, 5, ZSTD_HC_btlazy2 }, /* level 18 */
|
||||
{ 25, 26, 23, 5, 5, ZSTD_HC_btlazy2 }, /* level 19 */
|
||||
{ 26, 27, 24, 6, 5, ZSTD_HC_btlazy2 }, /* level 20 */
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
@ -58,37 +58,37 @@ default: zstd
|
||||
|
||||
all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 paramgrill datagen
|
||||
|
||||
zstd: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
zstd: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
|
||||
xxhash.c bench.c fileio.c zstdcli.c legacy/fileio_legacy.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
zstd32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
zstd32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
|
||||
xxhash.c bench.c fileio.c zstdcli.c legacy/fileio_legacy.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fullbench : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
fullbench : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
|
||||
datagen.c fullbench.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fullbench32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
fullbench32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
|
||||
datagen.c fullbench.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fuzzer : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
fuzzer : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
|
||||
datagen.c xxhash.c fuzzer.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fuzzer32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
fuzzer32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
|
||||
datagen.c xxhash.c fuzzer.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
paramgrill : $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
paramgrill : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
|
||||
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
|
||||
datagen.c xxhash.c paramgrill.c
|
||||
$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
|
||||
|
||||
@ -61,7 +61,6 @@
|
||||
|
||||
#include "mem.h"
|
||||
#include "zstd.h"
|
||||
#include "zstdhc.h"
|
||||
#include "xxhash.h"
|
||||
|
||||
|
||||
@ -231,12 +230,6 @@ typedef struct
|
||||
|
||||
typedef size_t (*compressor_t) (void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel);
|
||||
|
||||
static size_t local_compress_fast (void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
|
||||
{
|
||||
(void)compressionLevel;
|
||||
return ZSTD_compress(dst, maxDstSize, src, srcSize);
|
||||
}
|
||||
|
||||
#define MIN(a,b) ((a)<(b) ? (a) : (b))
|
||||
|
||||
static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, int cLevel)
|
||||
@ -247,7 +240,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
|
||||
const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize);
|
||||
void* const compressedBuffer = malloc(maxCompressedSize);
|
||||
void* const resultBuffer = malloc(srcSize);
|
||||
const compressor_t compressor = (cLevel <= 1) ? local_compress_fast : ZSTD_HC_compress;
|
||||
const compressor_t compressor = ZSTD_compress;
|
||||
U64 crcOrig;
|
||||
|
||||
/* init */
|
||||
@ -413,7 +406,7 @@ static size_t BMK_findMaxMem(U64 requiredMem)
|
||||
return (size_t)(requiredMem - step);
|
||||
}
|
||||
|
||||
static int BMK_benchOneFile(char* inFileName, int cLevel)
|
||||
static int BMK_benchOneFile(const char* inFileName, int cLevel)
|
||||
{
|
||||
FILE* inFile;
|
||||
U64 inFileSize;
|
||||
@ -513,7 +506,7 @@ static int BMK_syntheticTest(int cLevel, double compressibility)
|
||||
}
|
||||
|
||||
|
||||
int BMK_benchFiles(char** fileNamesTable, unsigned nbFiles, unsigned cLevel)
|
||||
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, unsigned cLevel)
|
||||
{
|
||||
double compressibility = (double)g_compressibilityDefault / 100;
|
||||
|
||||
|
||||
@ -68,7 +68,6 @@
|
||||
#include "mem.h"
|
||||
#include "fileio.h"
|
||||
#include "zstd_static.h"
|
||||
#include "zstdhc_static.h"
|
||||
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
# include "zstd_legacy.h" /* legacy */
|
||||
@ -237,46 +236,6 @@ static U64 FIO_getFileSize(const char* infilename)
|
||||
}
|
||||
|
||||
|
||||
typedef void* (*FIO_createC) (void);
|
||||
static void* local_ZSTD_createCCtx(void) { return (void*) ZSTD_createCCtx(); }
|
||||
static void* local_ZSTD_HC_createCCtx(void) { return (void*) ZSTD_HC_createCCtx(); }
|
||||
|
||||
typedef size_t (*FIO_initC) (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint);
|
||||
static size_t local_ZSTD_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint)
|
||||
{
|
||||
(void)cLevel; (void)srcSizeHint;
|
||||
return ZSTD_compressBegin((ZSTD_CCtx*)ctx, dst, maxDstSize);
|
||||
}
|
||||
static size_t local_ZSTD_HC_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint)
|
||||
{
|
||||
return ZSTD_HC_compressBegin((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, cLevel, srcSizeHint);
|
||||
}
|
||||
|
||||
typedef size_t (*FIO_continueC) (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
|
||||
static size_t local_ZSTD_compressContinue (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
return ZSTD_compressContinue((ZSTD_CCtx*)ctx, dst, maxDstSize, src, srcSize);
|
||||
}
|
||||
static size_t local_ZSTD_HC_compressContinue (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
return ZSTD_HC_compressContinue((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, src, srcSize);
|
||||
}
|
||||
|
||||
typedef size_t (*FIO_endC) (void* ctx, void* dst, size_t maxDstSize);
|
||||
static size_t local_ZSTD_compressEnd (void* ctx, void* dst, size_t maxDstSize)
|
||||
{
|
||||
return ZSTD_compressEnd((ZSTD_CCtx*)ctx, dst, maxDstSize);
|
||||
}
|
||||
static size_t local_ZSTD_HC_compressEnd (void* ctx, void* dst, size_t maxDstSize)
|
||||
{
|
||||
return ZSTD_HC_compressEnd((ZSTD_HC_CCtx*)ctx, dst, maxDstSize);
|
||||
}
|
||||
|
||||
typedef void (*FIO_freeC) (void* ctx);
|
||||
static void local_ZSTD_freeCCtx(void* ctx) { ZSTD_freeCCtx((ZSTD_CCtx*)ctx); }
|
||||
static void local_ZSTD_HC_freeCCtx(void* ctx) { ZSTD_HC_freeCCtx((ZSTD_HC_CCtx*)ctx); }
|
||||
|
||||
|
||||
unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename, int cLevel)
|
||||
{
|
||||
U64 filesize = 0;
|
||||
@ -291,35 +250,14 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
|
||||
FILE* finput;
|
||||
FILE* foutput;
|
||||
size_t sizeCheck, cSize;
|
||||
void* ctx;
|
||||
FIO_createC createC=NULL;
|
||||
FIO_initC initC=NULL;
|
||||
FIO_continueC continueC = NULL;
|
||||
FIO_endC endC = NULL;
|
||||
FIO_freeC freeC = NULL;
|
||||
ZSTD_CCtx* ctx;
|
||||
|
||||
/* Init */
|
||||
if (cLevel <= 1)
|
||||
{
|
||||
createC = local_ZSTD_createCCtx;
|
||||
initC = local_ZSTD_compressBegin;
|
||||
continueC = local_ZSTD_compressContinue;
|
||||
endC = local_ZSTD_compressEnd;
|
||||
freeC = local_ZSTD_freeCCtx;
|
||||
}
|
||||
else
|
||||
{
|
||||
createC = local_ZSTD_HC_createCCtx;
|
||||
initC = local_ZSTD_HC_compressBegin;
|
||||
continueC = local_ZSTD_HC_compressContinue;
|
||||
endC = local_ZSTD_HC_compressEnd;
|
||||
freeC = local_ZSTD_HC_freeCCtx;
|
||||
}
|
||||
/* init */
|
||||
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
|
||||
filesize = FIO_getFileSize(input_filename);
|
||||
|
||||
/* Allocate Memory */
|
||||
ctx = createC();
|
||||
ctx = ZSTD_createCCtx();
|
||||
inBuff = (BYTE*)malloc(inBuffSize);
|
||||
outBuff = (BYTE*)malloc(outBuffSize);
|
||||
if (!inBuff || !outBuff || !ctx) EXM_THROW(21, "Allocation error : not enough memory");
|
||||
@ -327,7 +265,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
|
||||
inEnd = inBuff + inBuffSize;
|
||||
|
||||
/* Write Frame Header */
|
||||
cSize = initC(ctx, outBuff, outBuffSize, cLevel, filesize);
|
||||
cSize = ZSTD_compressBegin(ctx, outBuff, outBuffSize, cLevel, filesize);
|
||||
if (ZSTD_isError(cSize)) EXM_THROW(22, "Compression error : cannot create frame header");
|
||||
|
||||
sizeCheck = fwrite(outBuff, 1, cSize, foutput);
|
||||
@ -348,7 +286,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
|
||||
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20));
|
||||
|
||||
/* Compress Block */
|
||||
cSize = continueC(ctx, outBuff, outBuffSize, inSlot, inSize);
|
||||
cSize = ZSTD_compressContinue(ctx, outBuff, outBuffSize, inSlot, inSize);
|
||||
if (ZSTD_isError(cSize))
|
||||
EXM_THROW(24, "Compression error : %s ", ZSTD_getErrorName(cSize));
|
||||
|
||||
@ -362,7 +300,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
|
||||
}
|
||||
|
||||
/* End of Frame */
|
||||
cSize = endC(ctx, outBuff, outBuffSize);
|
||||
cSize = ZSTD_compressEnd(ctx, outBuff, outBuffSize);
|
||||
if (ZSTD_isError(cSize)) EXM_THROW(26, "Compression error : cannot create frame end");
|
||||
|
||||
sizeCheck = fwrite(outBuff, 1, cSize, foutput);
|
||||
@ -377,7 +315,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
|
||||
/* clean */
|
||||
free(inBuff);
|
||||
free(outBuff);
|
||||
freeC(ctx);
|
||||
ZSTD_freeCCtx(ctx);
|
||||
fclose(finput);
|
||||
if (fclose(foutput)) EXM_THROW(28, "Write error : cannot properly close %s", output_filename);
|
||||
|
||||
|
||||
@ -217,7 +217,7 @@ extern size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
|
||||
size_t local_ZSTD_compress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
|
||||
{
|
||||
(void)buff2;
|
||||
return ZSTD_compress(dst, dstSize, src, srcSize);
|
||||
return ZSTD_compress(dst, dstSize, src, srcSize, 1);
|
||||
}
|
||||
|
||||
size_t local_ZSTD_decompress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
|
||||
@ -325,12 +325,12 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
|
||||
switch(benchNb)
|
||||
{
|
||||
case 11:
|
||||
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize);
|
||||
g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1);
|
||||
break;
|
||||
case 31: /* ZSTD_decodeLiteralsBlock */
|
||||
{
|
||||
blockProperties_t bp;
|
||||
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
|
||||
g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
|
||||
ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp); // Get first block type
|
||||
if (bp.blockType != bt_compressed)
|
||||
{
|
||||
@ -349,7 +349,7 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
|
||||
const BYTE* ip = dstBuff;
|
||||
const BYTE* iend;
|
||||
size_t blockSize;
|
||||
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
|
||||
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
|
||||
ip += 4; // Jump magic Number
|
||||
blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); // Get first block type
|
||||
if (bp.blockType != bt_compressed)
|
||||
@ -380,7 +380,7 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
|
||||
case 102: /* local_decodeLiteralsForward */
|
||||
{
|
||||
blockProperties_t bp;
|
||||
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
|
||||
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, 1);
|
||||
g_cSize = ZSTD_getcBlockSize(dstBuff+7, dstBuffSize, &bp);
|
||||
memcpy(buff2, dstBuff+10, g_cSize);
|
||||
//srcSize = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize); // real speed
|
||||
|
||||
@ -47,7 +47,6 @@
|
||||
#include <sys/timeb.h> /* timeb */
|
||||
#include <string.h> /* strcmp */
|
||||
#include "zstd_static.h"
|
||||
#include "zstdhc_static.h"
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
#include "xxhash.h" /* XXH64 */
|
||||
#include "mem.h"
|
||||
@ -159,7 +158,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
|
||||
/* Basic tests */
|
||||
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
|
||||
result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), CNBuffer, COMPRESSIBLE_NOISE_LENGTH);
|
||||
result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
|
||||
if (ZSTD_isError(result)) goto _output_error;
|
||||
cSize = result;
|
||||
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
|
||||
@ -213,7 +212,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
sampleSize += 256 KB - 1;
|
||||
RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., randState);
|
||||
sampleSize += 96 KB;
|
||||
cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize);
|
||||
cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
|
||||
if (ZSTD_isError(cSize)) goto _output_error;
|
||||
result = ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize);
|
||||
if (ZSTD_isError(result)) goto _output_error;
|
||||
@ -265,11 +264,11 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
|
||||
U32 testNb = 0;
|
||||
U32 coreSeed = seed, lseed = 0;
|
||||
ZSTD_CCtx* ctx;
|
||||
ZSTD_HC_CCtx* hcctx;
|
||||
ZSTD_CCtx* hcctx;
|
||||
|
||||
/* allocation */
|
||||
ctx = ZSTD_createCCtx();
|
||||
hcctx = ZSTD_HC_createCCtx();
|
||||
hcctx = ZSTD_createCCtx();
|
||||
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
|
||||
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
|
||||
@ -332,8 +331,8 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
|
||||
#define MAX(a,b) ((a)>(b)?(a):(b))
|
||||
cLevelMod = MAX(1, 38 - (int)(MAX(9, sampleSizeLog) * 2)); /* use high compression levels with small samples, for speed */
|
||||
cLevel = (FUZ_rand(&lseed) % cLevelMod) +1;
|
||||
cSize = ZSTD_HC_compressCCtx(hcctx, cBuffer, cBufferSize, srcBuffer + sampleStart, sampleSize, cLevel);
|
||||
CHECK(ZSTD_isError(cSize), "ZSTD_HC_compressCCtx failed");
|
||||
cSize = ZSTD_compressCCtx(hcctx, cBuffer, cBufferSize, srcBuffer + sampleStart, sampleSize, cLevel);
|
||||
CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed");
|
||||
|
||||
/* compression failure test : too small dest buffer */
|
||||
if (cSize > 3)
|
||||
@ -344,10 +343,10 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
|
||||
static const U32 endMark = 0x4DC2B1A9;
|
||||
U32 endCheck;
|
||||
memcpy(dstBuffer+tooSmallSize, &endMark, 4);
|
||||
errorCode = ZSTD_HC_compressCCtx(hcctx, dstBuffer, tooSmallSize, srcBuffer + sampleStart, sampleSize, cLevel);
|
||||
CHECK(!ZSTD_isError(errorCode), "ZSTD_HC_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize);
|
||||
errorCode = ZSTD_compressCCtx(hcctx, dstBuffer, tooSmallSize, srcBuffer + sampleStart, sampleSize, cLevel);
|
||||
CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize);
|
||||
memcpy(&endCheck, dstBuffer+tooSmallSize, 4);
|
||||
CHECK(endCheck != endMark, "ZSTD_HC_compressCCtx : dst buffer overflow");
|
||||
CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow");
|
||||
}
|
||||
|
||||
/* successfull decompression tests*/
|
||||
@ -434,7 +433,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
|
||||
|
||||
_cleanup:
|
||||
ZSTD_freeCCtx(ctx);
|
||||
ZSTD_HC_freeCCtx(hcctx);
|
||||
ZSTD_freeCCtx(hcctx);
|
||||
free(cNoiseBuffer[0]);
|
||||
free(cNoiseBuffer[1]);
|
||||
free(cNoiseBuffer[2]);
|
||||
|
||||
@ -62,8 +62,7 @@
|
||||
#endif
|
||||
|
||||
#include "mem.h"
|
||||
#include "zstdhc_static.h"
|
||||
#include "zstd.h"
|
||||
#include "zstd_static.h"
|
||||
#include "datagen.h"
|
||||
#include "xxhash.h"
|
||||
|
||||
@ -122,8 +121,8 @@ static U32 g_rand = 1;
|
||||
static U32 g_singleRun = 0;
|
||||
static U32 g_target = 0;
|
||||
static U32 g_noSeed = 0;
|
||||
static const ZSTD_HC_parameters* g_seedParams = ZSTD_HC_defaultParameters[0];
|
||||
static ZSTD_HC_parameters g_params = { 0, 0, 0, 0, 0, ZSTD_HC_greedy };
|
||||
static const ZSTD_parameters* g_seedParams = ZSTD_defaultParameters[0];
|
||||
static ZSTD_parameters g_params = { 0, 0, 0, 0, 0, ZSTD_greedy };
|
||||
|
||||
void BMK_SetNbIterations(int nbLoops)
|
||||
{
|
||||
@ -270,8 +269,8 @@ typedef struct
|
||||
|
||||
static size_t BMK_benchParam(BMK_result_t* resultPtr,
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_HC_CCtx* ctx,
|
||||
const ZSTD_HC_parameters params)
|
||||
ZSTD_CCtx* ctx,
|
||||
const ZSTD_parameters params)
|
||||
{
|
||||
const size_t blockSize = g_blockSize ? g_blockSize : srcSize;
|
||||
const U32 nbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize);
|
||||
@ -284,7 +283,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
|
||||
U32 Hlog = params.hashLog;
|
||||
U32 Slog = params.searchLog;
|
||||
U32 Slength = params.searchLength;
|
||||
ZSTD_HC_strategy strat = params.strategy;
|
||||
ZSTD_strategy strat = params.strategy;
|
||||
char name[30] = { 0 };
|
||||
U64 crcOrig;
|
||||
|
||||
@ -358,7 +357,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
|
||||
while (BMK_GetMilliSpan(milliTime) < TIMELOOP)
|
||||
{
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++)
|
||||
blockTable[blockNb].cSize = ZSTD_HC_compress_advanced(ctx,
|
||||
blockTable[blockNb].cSize = ZSTD_compress_advanced(ctx,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
|
||||
blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize,
|
||||
params);
|
||||
@ -429,13 +428,13 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr,
|
||||
}
|
||||
|
||||
|
||||
const char* g_stratName[] = { "ZSTD_HC_fast ",
|
||||
"ZSTD_HC_greedy ",
|
||||
"ZSTD_HC_lazy ",
|
||||
"ZSTD_HC_lazy2 ",
|
||||
"ZSTD_HC_btlazy2" };
|
||||
const char* g_stratName[] = { "ZSTD_fast ",
|
||||
"ZSTD_greedy ",
|
||||
"ZSTD_lazy ",
|
||||
"ZSTD_lazy2 ",
|
||||
"ZSTD_btlazy2" };
|
||||
|
||||
static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_HC_parameters params, size_t srcSize)
|
||||
static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_parameters params, size_t srcSize)
|
||||
{
|
||||
DISPLAY("\r%79s\r", "");
|
||||
fprintf(f," {%3u,%3u,%3u,%3u,%3u, %s }, ",
|
||||
@ -447,11 +446,11 @@ static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_HC_pa
|
||||
}
|
||||
|
||||
|
||||
static U32 g_cSpeedTarget[ZSTD_HC_MAX_CLEVEL+1] = { 0 };
|
||||
static U32 g_cSpeedTarget[ZSTD_MAX_CLEVEL+1] = { 0 };
|
||||
|
||||
typedef struct {
|
||||
BMK_result_t result;
|
||||
ZSTD_HC_parameters params;
|
||||
ZSTD_parameters params;
|
||||
} winnerInfo_t;
|
||||
|
||||
static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize)
|
||||
@ -459,11 +458,11 @@ static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSi
|
||||
int cLevel;
|
||||
|
||||
fprintf(f, "\n /* Selected configurations : */ \n");
|
||||
fprintf(f, "#define ZSTD_HC_MAX_CLEVEL %2u \n", ZSTD_HC_MAX_CLEVEL);
|
||||
fprintf(f, "static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] = {\n");
|
||||
fprintf(f, "#define ZSTD_MAX_CLEVEL %2u \n", ZSTD_MAX_CLEVEL);
|
||||
fprintf(f, "static const ZSTD_parameters ZSTD_defaultParameters[ZSTD_MAX_CLEVEL+1] = {\n");
|
||||
fprintf(f, " /* W, C, H, S, L, strat */ \n");
|
||||
|
||||
for (cLevel=0; cLevel <= ZSTD_HC_MAX_CLEVEL; cLevel++)
|
||||
for (cLevel=0; cLevel <= ZSTD_MAX_CLEVEL; cLevel++)
|
||||
BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize);
|
||||
}
|
||||
|
||||
@ -477,9 +476,9 @@ static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSiz
|
||||
}
|
||||
|
||||
|
||||
static int BMK_seed(winnerInfo_t* winners, const ZSTD_HC_parameters params,
|
||||
static int BMK_seed(winnerInfo_t* winners, const ZSTD_parameters params,
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_HC_CCtx* ctx)
|
||||
ZSTD_CCtx* ctx)
|
||||
{
|
||||
BMK_result_t testResult;
|
||||
int better = 0;
|
||||
@ -487,7 +486,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_HC_parameters params,
|
||||
|
||||
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params);
|
||||
|
||||
for (cLevel = 1; cLevel <= ZSTD_HC_MAX_CLEVEL; cLevel++)
|
||||
for (cLevel = 1; cLevel <= ZSTD_MAX_CLEVEL; cLevel++)
|
||||
{
|
||||
if (testResult.cSpeed < g_cSpeedTarget[cLevel])
|
||||
continue; /* not fast enough for this level */
|
||||
@ -514,9 +513,9 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_HC_parameters params,
|
||||
double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed);
|
||||
|
||||
size_t W_CMemUsed = (1 << params.windowLog) + 4 * (1 << params.hashLog) +
|
||||
((params.strategy==ZSTD_HC_fast) ? 0 : 4 * (1 << params.contentLog));
|
||||
((params.strategy==ZSTD_fast) ? 0 : 4 * (1 << params.contentLog));
|
||||
size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + 4 * (1 << winners[cLevel].params.hashLog) +
|
||||
((winners[cLevel].params.strategy==ZSTD_HC_fast) ? 0 : 4 * (1 << winners[cLevel].params.contentLog));
|
||||
((winners[cLevel].params.strategy==ZSTD_fast) ? 0 : 4 * (1 << winners[cLevel].params.contentLog));
|
||||
double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed);
|
||||
double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);
|
||||
|
||||
@ -581,10 +580,10 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_HC_parameters params,
|
||||
|
||||
|
||||
/* nullified useless params, to ensure count stats */
|
||||
static ZSTD_HC_parameters* sanitizeParams(ZSTD_HC_parameters params)
|
||||
static ZSTD_parameters* sanitizeParams(ZSTD_parameters params)
|
||||
{
|
||||
g_params = params;
|
||||
if (params.strategy == ZSTD_HC_fast)
|
||||
if (params.strategy == ZSTD_fast)
|
||||
{
|
||||
g_params.contentLog = 0;
|
||||
g_params.searchLog = 0;
|
||||
@ -604,16 +603,16 @@ static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */
|
||||
#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
|
||||
|
||||
static void playAround(FILE* f, winnerInfo_t* winners,
|
||||
ZSTD_HC_parameters params,
|
||||
ZSTD_parameters params,
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_HC_CCtx* ctx)
|
||||
ZSTD_CCtx* ctx)
|
||||
{
|
||||
int nbVariations = 0;
|
||||
const int startTime = BMK_GetMilliStart();
|
||||
|
||||
while (BMK_GetMilliSpan(startTime) < g_maxVariationTime)
|
||||
{
|
||||
ZSTD_HC_parameters p = params;
|
||||
ZSTD_parameters p = params;
|
||||
U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1;
|
||||
if (nbVariations++ > g_maxNbVariations) break;
|
||||
|
||||
@ -643,16 +642,16 @@ static void playAround(FILE* f, winnerInfo_t* winners,
|
||||
case 9:
|
||||
p.searchLength--; break;
|
||||
case 10:
|
||||
p.strategy = (ZSTD_HC_strategy)(((U32)p.strategy)+1); break;
|
||||
p.strategy = (ZSTD_strategy)(((U32)p.strategy)+1); break;
|
||||
case 11:
|
||||
p.strategy = (ZSTD_HC_strategy)(((U32)p.strategy)-1); break;
|
||||
p.strategy = (ZSTD_strategy)(((U32)p.strategy)-1); break;
|
||||
}
|
||||
}
|
||||
|
||||
/* validate new conf */
|
||||
{
|
||||
ZSTD_HC_parameters saved = p;
|
||||
ZSTD_HC_validateParams(&p, g_blockSize ? g_blockSize : srcSize);
|
||||
ZSTD_parameters saved = p;
|
||||
ZSTD_validateParams(&p, g_blockSize ? g_blockSize : srcSize);
|
||||
if (memcmp(&p, &saved, sizeof(p))) continue; /* p was invalid */
|
||||
}
|
||||
|
||||
@ -675,19 +674,19 @@ static void playAround(FILE* f, winnerInfo_t* winners,
|
||||
static void BMK_selectRandomStart(
|
||||
FILE* f, winnerInfo_t* winners,
|
||||
const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_HC_CCtx* ctx)
|
||||
ZSTD_CCtx* ctx)
|
||||
{
|
||||
U32 id = (FUZ_rand(&g_rand) % (ZSTD_HC_MAX_CLEVEL+1));
|
||||
U32 id = (FUZ_rand(&g_rand) % (ZSTD_MAX_CLEVEL+1));
|
||||
if ((id==0) || (winners[id].params.windowLog==0))
|
||||
{
|
||||
/* totally random entry */
|
||||
ZSTD_HC_parameters p;
|
||||
p.contentLog = FUZ_rand(&g_rand) % (ZSTD_HC_CONTENTLOG_MAX+1 - ZSTD_HC_CONTENTLOG_MIN) + ZSTD_HC_CONTENTLOG_MIN;
|
||||
p.hashLog = FUZ_rand(&g_rand) % (ZSTD_HC_HASHLOG_MAX+1 - ZSTD_HC_HASHLOG_MIN) + ZSTD_HC_HASHLOG_MIN;
|
||||
p.searchLog = FUZ_rand(&g_rand) % (ZSTD_HC_SEARCHLOG_MAX+1 - ZSTD_HC_SEARCHLOG_MIN) + ZSTD_HC_SEARCHLOG_MIN;
|
||||
p.windowLog = FUZ_rand(&g_rand) % (ZSTD_HC_WINDOWLOG_MAX+1 - ZSTD_HC_WINDOWLOG_MIN) + ZSTD_HC_WINDOWLOG_MIN;
|
||||
p.searchLength=FUZ_rand(&g_rand) % (ZSTD_HC_SEARCHLENGTH_MAX+1 - ZSTD_HC_SEARCHLENGTH_MIN) + ZSTD_HC_SEARCHLENGTH_MIN;
|
||||
p.strategy = (ZSTD_HC_strategy) (FUZ_rand(&g_rand) % (ZSTD_HC_btlazy2+1));
|
||||
ZSTD_parameters p;
|
||||
p.contentLog = FUZ_rand(&g_rand) % (ZSTD_CONTENTLOG_MAX+1 - ZSTD_CONTENTLOG_MIN) + ZSTD_CONTENTLOG_MIN;
|
||||
p.hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN;
|
||||
p.searchLog = FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN) + ZSTD_SEARCHLOG_MIN;
|
||||
p.windowLog = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN;
|
||||
p.searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN;
|
||||
p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btlazy2+1));
|
||||
playAround(f, winners, p, srcBuffer, srcSize, ctx);
|
||||
}
|
||||
else
|
||||
@ -697,9 +696,9 @@ static void BMK_selectRandomStart(
|
||||
|
||||
static void BMK_benchMem(void* srcBuffer, size_t srcSize)
|
||||
{
|
||||
ZSTD_HC_CCtx* ctx = ZSTD_HC_createCCtx();
|
||||
ZSTD_HC_parameters params;
|
||||
winnerInfo_t winners[ZSTD_HC_MAX_CLEVEL+1];
|
||||
ZSTD_CCtx* ctx = ZSTD_createCCtx();
|
||||
ZSTD_parameters params;
|
||||
winnerInfo_t winners[ZSTD_MAX_CLEVEL+1];
|
||||
int i;
|
||||
const char* rfName = "grillResults.txt";
|
||||
FILE* f;
|
||||
@ -709,7 +708,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
|
||||
if (g_singleRun)
|
||||
{
|
||||
BMK_result_t testResult;
|
||||
ZSTD_HC_validateParams(&g_params, blockSize);
|
||||
ZSTD_validateParams(&g_params, blockSize);
|
||||
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params);
|
||||
DISPLAY("\n");
|
||||
return;
|
||||
@ -731,24 +730,24 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
|
||||
params.contentLog = 1;
|
||||
params.searchLog = 1;
|
||||
params.searchLength = 7;
|
||||
params.strategy = ZSTD_HC_fast;
|
||||
ZSTD_HC_validateParams(¶ms, blockSize);
|
||||
params.strategy = ZSTD_fast;
|
||||
ZSTD_validateParams(¶ms, blockSize);
|
||||
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params);
|
||||
g_cSpeedTarget[1] = (testResult.cSpeed * 15) >> 4;
|
||||
}
|
||||
|
||||
/* establish speed objectives (relative to level 1) */
|
||||
for (i=2; i<=ZSTD_HC_MAX_CLEVEL; i++)
|
||||
for (i=2; i<=ZSTD_MAX_CLEVEL; i++)
|
||||
g_cSpeedTarget[i] = (g_cSpeedTarget[i-1] * 25) >> 5;
|
||||
|
||||
/* populate initial solution */
|
||||
{
|
||||
const int tableID = (blockSize > 128 KB);
|
||||
const int maxSeeds = g_noSeed ? 1 : ZSTD_HC_MAX_CLEVEL;
|
||||
g_seedParams = ZSTD_HC_defaultParameters[tableID];
|
||||
const int maxSeeds = g_noSeed ? 1 : ZSTD_MAX_CLEVEL;
|
||||
g_seedParams = ZSTD_defaultParameters[tableID];
|
||||
for (i=1; i<=maxSeeds; i++)
|
||||
{
|
||||
const U32 btPlus = (params.strategy == ZSTD_HC_btlazy2);
|
||||
const U32 btPlus = (params.strategy == ZSTD_btlazy2);
|
||||
params = g_seedParams[i];
|
||||
params.windowLog = MIN(srcLog, params.windowLog);
|
||||
params.contentLog = MIN(params.windowLog+btPlus, params.contentLog);
|
||||
@ -775,7 +774,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
|
||||
|
||||
/* clean up*/
|
||||
fclose(f);
|
||||
ZSTD_HC_freeCCtx(ctx);
|
||||
ZSTD_freeCCtx(ctx);
|
||||
}
|
||||
|
||||
|
||||
@ -994,12 +993,12 @@ int main(int argc, char** argv)
|
||||
g_params.searchLength *= 10, g_params.searchLength += *argument++ - '0';
|
||||
continue;
|
||||
case 't': /* strategy */
|
||||
g_params.strategy = (ZSTD_HC_strategy)0;
|
||||
g_params.strategy = (ZSTD_strategy)0;
|
||||
argument++;
|
||||
while ((*argument>= '0') && (*argument<='9'))
|
||||
{
|
||||
g_params.strategy = (ZSTD_HC_strategy)((U32)g_params.strategy *10);
|
||||
g_params.strategy = (ZSTD_HC_strategy)((U32)g_params.strategy + *argument++ - '0');
|
||||
g_params.strategy = (ZSTD_strategy)((U32)g_params.strategy *10);
|
||||
g_params.strategy = (ZSTD_strategy)((U32)g_params.strategy + *argument++ - '0');
|
||||
}
|
||||
continue;
|
||||
case 'L':
|
||||
@ -1009,7 +1008,7 @@ int main(int argc, char** argv)
|
||||
while ((*argument>= '0') && (*argument<='9'))
|
||||
cLevel *= 10, cLevel += *argument++ - '0';
|
||||
if (cLevel < 1) cLevel = 1;
|
||||
if (cLevel > ZSTD_HC_MAX_CLEVEL) cLevel = ZSTD_HC_MAX_CLEVEL;
|
||||
if (cLevel > ZSTD_MAX_CLEVEL) cLevel = ZSTD_MAX_CLEVEL;
|
||||
g_params = g_seedParams[cLevel];
|
||||
continue;
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user