Merge remote-tracking branch 'refs/remotes/lz4/dev' into dev
This commit is contained in:
commit
cb95994499
23
lib/lz4hc.c
23
lib/lz4hc.c
@ -98,7 +98,7 @@ static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start)
|
||||
{
|
||||
MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
|
||||
MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
|
||||
hc4->nextToUpdate = hc4->nextToUpdateBT = 64 KB;
|
||||
hc4->nextToUpdate = 64 KB;
|
||||
hc4->base = start - 64 KB;
|
||||
hc4->end = start;
|
||||
hc4->dictBase = start - 64 KB;
|
||||
@ -486,6 +486,14 @@ _Search3:
|
||||
return (int) (((char*)op)-dest);
|
||||
}
|
||||
|
||||
static int LZ4HC_getSearchNum(int compressionLevel)
|
||||
{
|
||||
switch (compressionLevel) {
|
||||
default: return 0; /* unused */
|
||||
case 11: return 128;
|
||||
case 12: return 1<<10;
|
||||
}
|
||||
}
|
||||
|
||||
static int LZ4HC_compress_generic (
|
||||
LZ4HC_CCtx_internal* const ctx,
|
||||
@ -501,9 +509,9 @@ static int LZ4HC_compress_generic (
|
||||
if (compressionLevel > 9) {
|
||||
switch (compressionLevel) {
|
||||
case 10: return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (16-1), limit);
|
||||
case 11: ctx->searchNum = 128; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0);
|
||||
case 11: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0);
|
||||
default:
|
||||
case 12: ctx->searchNum = 1<<10; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1);
|
||||
case 12: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1);
|
||||
}
|
||||
}
|
||||
return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (compressionLevel-1), limit);
|
||||
@ -554,6 +562,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
|
||||
LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
|
||||
LZ4_streamHCPtr->internal_donotuse.base = NULL;
|
||||
LZ4_streamHCPtr->internal_donotuse.compressionLevel = (unsigned)compressionLevel;
|
||||
LZ4_streamHCPtr->internal_donotuse.searchNum = LZ4HC_getSearchNum(compressionLevel);
|
||||
}
|
||||
|
||||
int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize)
|
||||
@ -564,8 +573,11 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
|
||||
dictSize = 64 KB;
|
||||
}
|
||||
LZ4HC_init (ctxPtr, (const BYTE*)dictionary);
|
||||
if (dictSize >= 4) LZ4HC_Insert (ctxPtr, (const BYTE*)dictionary +(dictSize-3));
|
||||
ctxPtr->end = (const BYTE*)dictionary + dictSize;
|
||||
if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN)
|
||||
LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS);
|
||||
else
|
||||
if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);
|
||||
return dictSize;
|
||||
}
|
||||
|
||||
@ -585,7 +597,7 @@ static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBl
|
||||
ctxPtr->dictBase = ctxPtr->base;
|
||||
ctxPtr->base = newBlock - ctxPtr->dictLimit;
|
||||
ctxPtr->end = newBlock;
|
||||
ctxPtr->nextToUpdate = ctxPtr->nextToUpdateBT = ctxPtr->dictLimit; /* match referencing will resume from there */
|
||||
ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */
|
||||
}
|
||||
|
||||
static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr,
|
||||
@ -645,7 +657,6 @@ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictS
|
||||
streamPtr->dictLimit = endIndex - dictSize;
|
||||
streamPtr->lowLimit = endIndex - dictSize;
|
||||
if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit;
|
||||
if (streamPtr->nextToUpdateBT < streamPtr->dictLimit) streamPtr->nextToUpdateBT = streamPtr->dictLimit;
|
||||
}
|
||||
return dictSize;
|
||||
}
|
||||
|
@ -154,7 +154,6 @@ typedef struct
|
||||
uint32_t dictLimit; /* below that point, need extDict */
|
||||
uint32_t lowLimit; /* below that point, no more dict */
|
||||
uint32_t nextToUpdate; /* index from which to continue dictionary update */
|
||||
uint32_t nextToUpdateBT; /* index from which to continue binary tree update */
|
||||
uint32_t searchNum; /* only for optimal parser */
|
||||
uint32_t compressionLevel;
|
||||
} LZ4HC_CCtx_internal;
|
||||
@ -172,7 +171,6 @@ typedef struct
|
||||
unsigned int dictLimit; /* below that point, need extDict */
|
||||
unsigned int lowLimit; /* below that point, no more dict */
|
||||
unsigned int nextToUpdate; /* index from which to continue dictionary update */
|
||||
unsigned int nextToUpdateBT; /* index from which to continue binary tree update */
|
||||
unsigned int searchNum; /* only for optimal parser */
|
||||
unsigned int compressionLevel;
|
||||
} LZ4HC_CCtx_internal;
|
||||
|
49
lib/lz4opt.h
49
lib/lz4opt.h
@ -33,11 +33,6 @@
|
||||
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
|
||||
#define LZ4_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__)
|
||||
#define LZ4_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__)
|
||||
#define LZ4_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__)
|
||||
|
||||
|
||||
#define LZ4_OPT_NUM (1<<12)
|
||||
|
||||
|
||||
@ -133,7 +128,10 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches (
|
||||
if (matchLength > best_mlen) {
|
||||
best_mlen = matchLength;
|
||||
if (matches) {
|
||||
matches[mnum].off = (int)(ip - match);
|
||||
if (matchIndex >= dictLimit)
|
||||
matches[mnum].off = (int)(ip - match);
|
||||
else
|
||||
matches[mnum].off = (int)(ip - (base + matchIndex)); /* virtual matchpos */
|
||||
matches[mnum].len = (int)matchLength;
|
||||
mnum++;
|
||||
}
|
||||
@ -173,8 +171,7 @@ FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* cons
|
||||
{
|
||||
const BYTE* const base = ctx->base;
|
||||
const U32 target = (U32)(ip - base);
|
||||
U32 idx = ctx->nextToUpdateBT;
|
||||
|
||||
U32 idx = ctx->nextToUpdate;
|
||||
while(idx < target)
|
||||
idx += LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, 8, NULL, NULL);
|
||||
}
|
||||
@ -187,10 +184,10 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches (
|
||||
size_t best_mlen, LZ4HC_match_t* matches, const int fullUpdate)
|
||||
{
|
||||
int mnum = 0;
|
||||
if (ip < ctx->base + ctx->nextToUpdateBT) return 0; /* skipped area */
|
||||
if (ip < ctx->base + ctx->nextToUpdate) return 0; /* skipped area */
|
||||
if (fullUpdate) LZ4HC_updateBinTree(ctx, ip, iHighLimit);
|
||||
best_mlen = LZ4HC_BinTree_InsertAndGetAllMatches(ctx, ip, iHighLimit, best_mlen, matches, &mnum);
|
||||
ctx->nextToUpdateBT = (U32)(ip - ctx->base + best_mlen);
|
||||
ctx->nextToUpdate = (U32)(ip - ctx->base + best_mlen);
|
||||
return mnum;
|
||||
}
|
||||
|
||||
@ -202,7 +199,6 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches (
|
||||
opt[pos].off = (int)offset; \
|
||||
opt[pos].litlen = (int)litlen; \
|
||||
opt[pos].price = (int)price; \
|
||||
LZ4_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-source), pos, last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \
|
||||
}
|
||||
|
||||
|
||||
@ -219,7 +215,7 @@ static int LZ4HC_compress_optimal (
|
||||
{
|
||||
LZ4HC_optimal_t opt[LZ4_OPT_NUM + 1];
|
||||
LZ4HC_match_t matches[LZ4_OPT_NUM + 1];
|
||||
const BYTE *inr;
|
||||
const BYTE *inr = NULL;
|
||||
size_t res, cur, cur2;
|
||||
size_t i, llen, litlen, mlen, best_mlen, price, offset, best_off, match_num, last_pos;
|
||||
|
||||
@ -242,7 +238,6 @@ static int LZ4HC_compress_optimal (
|
||||
llen = ip - anchor;
|
||||
match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate);
|
||||
if (!match_num) { ip++; continue; }
|
||||
LZ4_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-source), match_num, last_pos);
|
||||
|
||||
if ((size_t)matches[match_num-1].len > sufficient_len) {
|
||||
best_mlen = matches[match_num-1].len;
|
||||
@ -256,7 +251,6 @@ static int LZ4HC_compress_optimal (
|
||||
for (i = 0; i < match_num; i++) {
|
||||
mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH;
|
||||
best_mlen = (matches[i].len < LZ4_OPT_NUM) ? matches[i].len : LZ4_OPT_NUM;
|
||||
LZ4_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-source), matches[i].len, matches[i].off, best_mlen, last_pos);
|
||||
while (mlen <= best_mlen) {
|
||||
litlen = 0;
|
||||
price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen);
|
||||
@ -276,29 +270,22 @@ static int LZ4HC_compress_optimal (
|
||||
litlen = opt[cur-1].litlen + 1;
|
||||
if (cur != litlen) {
|
||||
price = opt[cur - litlen].price + LZ4HC_literalsPrice(litlen);
|
||||
LZ4_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen);
|
||||
} else {
|
||||
price = LZ4HC_literalsPrice(llen + litlen) - LZ4HC_literalsPrice(llen);
|
||||
LZ4_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen);
|
||||
}
|
||||
} else {
|
||||
litlen = 1;
|
||||
price = opt[cur - 1].price + LZ4HC_literalsPrice(litlen);
|
||||
LZ4_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, LZ4HC_literalsPrice(litlen));
|
||||
}
|
||||
|
||||
mlen = 1;
|
||||
best_mlen = 0;
|
||||
LZ4_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price);
|
||||
if (cur > last_pos || price < (size_t)opt[cur].price)
|
||||
SET_PRICE(cur, mlen, best_mlen, litlen, price);
|
||||
|
||||
if (cur == last_pos || inr >= mflimit) break;
|
||||
LZ4_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen);
|
||||
|
||||
match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches, fullUpdate);
|
||||
LZ4_LOG_PARSER("%d: LZ4HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num);
|
||||
|
||||
if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) {
|
||||
best_mlen = matches[match_num-1].len;
|
||||
best_off = matches[match_num-1].off;
|
||||
@ -311,7 +298,6 @@ static int LZ4HC_compress_optimal (
|
||||
mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH;
|
||||
cur2 = cur;
|
||||
best_mlen = (cur2 + matches[i].len < LZ4_OPT_NUM) ? (size_t)matches[i].len : LZ4_OPT_NUM - cur2;
|
||||
LZ4_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-source), cur, cur2, matches[i].len, matches[i].off, best_mlen, last_pos);
|
||||
|
||||
while (mlen <= best_mlen) {
|
||||
if (opt[cur2].mlen == 1) {
|
||||
@ -326,7 +312,6 @@ static int LZ4HC_compress_optimal (
|
||||
price = opt[cur2].price + LZ4HC_sequencePrice(litlen, mlen);
|
||||
}
|
||||
|
||||
LZ4_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price);
|
||||
if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) { // || (((int)price == opt[cur2 + mlen].price) && (opt[cur2 + mlen-1].mlen == 1))) {
|
||||
SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price);
|
||||
}
|
||||
@ -340,12 +325,6 @@ static int LZ4HC_compress_optimal (
|
||||
cur = last_pos - best_mlen;
|
||||
|
||||
encode: /* cur, last_pos, best_mlen, best_off have to be set */
|
||||
for (i = 1; i <= last_pos; i++) {
|
||||
LZ4_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen);
|
||||
}
|
||||
|
||||
LZ4_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d\n", (int)(ip-source+cur), cur, last_pos, best_mlen, best_off);
|
||||
|
||||
opt[0].mlen = 1;
|
||||
while (1) {
|
||||
mlen = opt[cur].mlen;
|
||||
@ -358,26 +337,15 @@ encode: /* cur, last_pos, best_mlen, best_off have to be set */
|
||||
cur -= mlen;
|
||||
}
|
||||
|
||||
for (i = 0; i <= last_pos;) {
|
||||
LZ4_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen);
|
||||
i += opt[i].mlen;
|
||||
}
|
||||
|
||||
cur = 0;
|
||||
while (cur < last_pos) {
|
||||
LZ4_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d\n", (int)(ip-source+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen);
|
||||
mlen = opt[cur].mlen;
|
||||
if (mlen == 1) { ip++; cur++; continue; }
|
||||
offset = opt[cur].off;
|
||||
cur += mlen;
|
||||
|
||||
LZ4_LOG_ENCODE("%d: ENCODE literals=%d off=%d mlen=%d ", (int)(ip-(const BYTE*)source), (int)(ip-anchor), (int)(offset), (int)mlen);
|
||||
res = LZ4HC_encodeSequence(&ip, &op, &anchor, (int)mlen, ip - offset, limit, oend);
|
||||
LZ4_LOG_ENCODE("out=%d\n", (int)((char*)op - dest));
|
||||
|
||||
if (res) return 0;
|
||||
|
||||
LZ4_LOG_PARSER("%d: offset=%d\n", (int)(ip-source), offset);
|
||||
}
|
||||
}
|
||||
|
||||
@ -386,7 +354,6 @@ encode: /* cur, last_pos, best_mlen, best_off have to be set */
|
||||
if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */
|
||||
if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun > 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
|
||||
else *op++ = (BYTE)(lastRun<<ML_BITS);
|
||||
LZ4_LOG_ENCODE("%d: ENCODE_LAST literals=%d out=%d\n", (int)(ip-(const BYTE*)source), (int)(iend-anchor), (int)((char*)op-dest));
|
||||
memcpy(op, anchor, iend - anchor);
|
||||
op += iend-anchor;
|
||||
}
|
||||
|
@ -277,7 +277,20 @@ test-lz4-testmode: lz4 datagen
|
||||
$(LZ4) -fm file1-dne file2-dne && false || true
|
||||
$(LZ4) -fm file1-dne file2-dne && false || true
|
||||
|
||||
test-lz4: lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-sparse \
|
||||
test-lz4-opt-parser: lz4 datagen
|
||||
@echo "\n ---- test opt-parser ----"
|
||||
./datagen -g16KB | $(LZ4) -12 | $(LZ4) -t
|
||||
./datagen -P10 | $(LZ4) -12B4 | $(LZ4) -t
|
||||
./datagen -g256K | $(LZ4) -12B4D | $(LZ4) -t
|
||||
./datagen -g512K -P25 | $(LZ4) -12BD | $(LZ4) -t
|
||||
./datagen -g1M | $(LZ4) -12B5 | $(LZ4) -t
|
||||
./datagen -g2M -P99 | $(LZ4) -11B4D | $(LZ4) -t
|
||||
./datagen -g4M | $(LZ4) -11vq | $(LZ4) -qt
|
||||
./datagen -g8M | $(LZ4) -11B4 | $(LZ4) -t
|
||||
./datagen -g16M -P90 | $(LZ4) -11B5 | $(LZ4) -t
|
||||
./datagen -g32M -P10 | $(LZ4) -11B5D | $(LZ4) -t
|
||||
|
||||
test-lz4: lz4 datagen test-lz4-opt-parser test-lz4-basic test-lz4-multiple test-lz4-sparse \
|
||||
test-lz4-frame-concatenation test-lz4-testmode test-lz4-contentSize \
|
||||
test-lz4-hugefile
|
||||
|
||||
|
@ -303,6 +303,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
|
||||
int const blockStart = FUZ_rand(&randState) % (COMPRESSIBLE_NOISE_LENGTH - blockSize);
|
||||
int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
|
||||
int const dictSize = MIN(dictSizeRand, blockStart);
|
||||
int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1);
|
||||
char* const block = ((char*)CNBuffer) + blockStart;
|
||||
const char* dict = block - dictSize;
|
||||
int compressedSize, HCcompressedSize;
|
||||
@ -349,18 +350,18 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
|
||||
|
||||
/* Test compression HC */
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, 9);
|
||||
ret = LZ4_compress_HC(block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
|
||||
FUZ_CHECKTEST(ret==0, "LZ4_compressHC() failed");
|
||||
HCcompressedSize = ret;
|
||||
|
||||
/* Test compression HC using external state */
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, 9);
|
||||
ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, (int)compressedBufferSize, compressionLevel);
|
||||
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_withStateHC() failed");
|
||||
|
||||
/* Test compression using external state */
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 9);
|
||||
ret = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
|
||||
FUZ_CHECKTEST(ret==0, "LZ4_compress_withState() failed");
|
||||
|
||||
/* Test compression */
|
||||
@ -466,12 +467,12 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
|
||||
|
||||
/* Test HC compression with output size being exactly what's necessary (should work) */
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, 9);
|
||||
ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel);
|
||||
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput() failed despite sufficient space");
|
||||
|
||||
/* Test HC compression with output size being exactly what's necessary (should work) */
|
||||
FUZ_DISPLAYTEST;
|
||||
ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, 9);
|
||||
ret = LZ4_compress_HC_extStateHC(stateLZ4HC, block, compressedBuffer, blockSize, HCcompressedSize, compressionLevel);
|
||||
FUZ_CHECKTEST(ret==0, "LZ4_compressHC_limitedOutput_withStateHC() failed despite sufficient space");
|
||||
|
||||
/* Test compression with missing bytes into output buffer => must fail */
|
||||
@ -491,7 +492,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
|
||||
if (missingBytes >= HCcompressedSize) missingBytes = HCcompressedSize-1;
|
||||
missingBytes += !missingBytes; /* avoid special case missingBytes==0 */
|
||||
compressedBuffer[HCcompressedSize-missingBytes] = 0;
|
||||
ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, 9);
|
||||
ret = LZ4_compress_HC(block, compressedBuffer, blockSize, HCcompressedSize-missingBytes, compressionLevel);
|
||||
FUZ_CHECKTEST(ret, "LZ4_compressHC_limitedOutput should have failed (output buffer too small by %i byte)", missingBytes);
|
||||
FUZ_CHECKTEST(compressedBuffer[HCcompressedSize-missingBytes], "LZ4_compressHC_limitedOutput overran output buffer ! (%i missingBytes)", missingBytes)
|
||||
}
|
||||
@ -592,7 +593,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
|
||||
FUZ_DISPLAYTEST;
|
||||
dict -= (FUZ_rand(&randState) & 7); /* even bigger separation */
|
||||
if (dict < (char*)CNBuffer) dict = (char*)CNBuffer;
|
||||
LZ4_resetStreamHC (&LZ4dictHC, FUZ_rand(&randState) & 0x7);
|
||||
LZ4_resetStreamHC (&LZ4dictHC, compressionLevel);
|
||||
LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
|
||||
blockContinueCompressedSize = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, (int)compressedBufferSize);
|
||||
FUZ_CHECKTEST(blockContinueCompressedSize==0, "LZ4_compressHC_continue failed");
|
||||
@ -600,7 +601,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
|
||||
FUZ_DISPLAYTEST;
|
||||
LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
|
||||
ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
|
||||
FUZ_CHECKTEST(ret>0, "LZ4_compressHC_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer");
|
||||
FUZ_CHECKTEST(ret>0, "LZ4_compressHC_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize);
|
||||
|
||||
FUZ_DISPLAYTEST;
|
||||
LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
|
||||
@ -655,7 +656,7 @@ _output_error:
|
||||
#define testCompressedSize (128 KB)
|
||||
#define ringBufferSize (8 KB)
|
||||
|
||||
static void FUZ_unitTests(void)
|
||||
static void FUZ_unitTests(int compressionLevel)
|
||||
{
|
||||
const unsigned testNb = 0;
|
||||
const unsigned seed = 0;
|
||||
@ -750,7 +751,7 @@ static void FUZ_unitTests(void)
|
||||
|
||||
/* simple HC compression test */
|
||||
crcOrig = XXH64(testInput, testCompressedSize, 0);
|
||||
LZ4_resetStreamHC(&sHC, 0);
|
||||
LZ4_resetStreamHC(&sHC, compressionLevel);
|
||||
result = LZ4_compress_HC_continue(&sHC, testInput, testCompressed, testCompressedSize, testCompressedSize-1);
|
||||
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
|
||||
|
||||
@ -761,7 +762,7 @@ static void FUZ_unitTests(void)
|
||||
|
||||
/* simple dictionary HC compression test */
|
||||
crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0);
|
||||
LZ4_resetStreamHC(&sHC, 0);
|
||||
LZ4_resetStreamHC(&sHC, compressionLevel);
|
||||
LZ4_loadDictHC(&sHC, testInput, 64 KB);
|
||||
result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1);
|
||||
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result);
|
||||
@ -775,7 +776,7 @@ static void FUZ_unitTests(void)
|
||||
{ int result1, result2;
|
||||
int segSize = testCompressedSize / 2;
|
||||
crcOrig = XXH64(testInput + segSize, testCompressedSize, 0);
|
||||
LZ4_resetStreamHC(&sHC, 0);
|
||||
LZ4_resetStreamHC(&sHC, compressionLevel);
|
||||
LZ4_loadDictHC(&sHC, testInput, segSize);
|
||||
result1 = LZ4_compress_HC_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1);
|
||||
FUZ_CHECKTEST(result1==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result1);
|
||||
@ -792,7 +793,7 @@ static void FUZ_unitTests(void)
|
||||
|
||||
/* remote dictionary HC compression test */
|
||||
crcOrig = XXH64(testInput + 64 KB, testCompressedSize, 0);
|
||||
LZ4_resetStreamHC(&sHC, 0);
|
||||
LZ4_resetStreamHC(&sHC, compressionLevel);
|
||||
LZ4_loadDictHC(&sHC, testInput, 32 KB);
|
||||
result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1);
|
||||
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result);
|
||||
@ -813,7 +814,7 @@ static void FUZ_unitTests(void)
|
||||
int segSize = (FUZ_rand(&randState) & 8191);
|
||||
int segNb = 1;
|
||||
|
||||
LZ4_resetStreamHC(&sHC, 0);
|
||||
LZ4_resetStreamHC(&sHC, compressionLevel);
|
||||
LZ4_loadDictHC(&sHC, dict, dictSize);
|
||||
|
||||
XXH64_reset(&crcOrigState, 0);
|
||||
@ -859,7 +860,7 @@ static void FUZ_unitTests(void)
|
||||
|
||||
XXH64_reset(&xxhOrig, 0);
|
||||
XXH64_reset(&xxhNew, 0);
|
||||
LZ4_resetStreamHC(&sHC, 0);
|
||||
LZ4_resetStreamHC(&sHC, compressionLevel);
|
||||
LZ4_setStreamDecode(&decodeState, NULL, 0);
|
||||
|
||||
while (iNext + messageSize < testCompressedSize) {
|
||||
@ -901,7 +902,7 @@ static void FUZ_unitTests(void)
|
||||
|
||||
XXH64_reset(&xxhOrig, 0);
|
||||
XXH64_reset(&xxhNew, 0);
|
||||
LZ4_resetStreamHC(&sHC, 0);
|
||||
LZ4_resetStreamHC(&sHC, compressionLevel);
|
||||
LZ4_setStreamDecode(&decodeState, NULL, 0);
|
||||
|
||||
#define BSIZE1 65537
|
||||
@ -956,7 +957,7 @@ static void FUZ_unitTests(void)
|
||||
}
|
||||
}
|
||||
|
||||
printf("All unit tests completed successfully \n");
|
||||
printf("All unit tests completed successfully compressionLevel=%d \n", compressionLevel);
|
||||
return;
|
||||
_output_error:
|
||||
exit(1);
|
||||
@ -1102,7 +1103,7 @@ int main(int argc, const char** argv)
|
||||
|
||||
if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) printf("Compressibility : %i%%\n", proba);
|
||||
|
||||
if ((seedset==0) && (testNb==0)) FUZ_unitTests();
|
||||
if ((seedset==0) && (testNb==0)) { FUZ_unitTests(LZ4HC_CLEVEL_DEFAULT); FUZ_unitTests(LZ4HC_CLEVEL_OPT_MIN); }
|
||||
|
||||
if (nbTests<=0) nbTests=1;
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user