AuroraMiddleware/lz4
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #435 from lz4/destSize

Browse Source 
lz4opt is now compatible with _destSize variant
This commit is contained in:
Yann Collet 2018-01-06 06:57:56 +01:00 committed by GitHub
commit 54f3621907
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 87 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
lib/lz4hc.c
View File

@ -425,10 +425,7 @@ static int LZ4HC_compress_hashChain (
/* init */
*srcSizePtr = 0;
if (limit == limitedDestSize && maxOutputSize < 1) return 0; /* Impossible to store anything */
if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */
if (limit == limitedDestSize) oend -= LASTLITERALS; /* Hack for support limitations LZ4 decompressor */
if (limit == limitedDestSize) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */
if (inputSize < LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */
/* Main Loop */
@ -641,11 +638,12 @@ static int LZ4HC_compress_generic (
{ lz4opt,8192, LZ4_OPT_NUM }, /* 12==LZ4HC_CLEVEL_MAX */
};
if (limit == limitedDestSize && dstCapacity < 1) return 0; /* Impossible to store anything */
if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size (too large or negative) */
ctx->end += *srcSizePtr;
if (cLevel < 1) cLevel = LZ4HC_CLEVEL_DEFAULT; /* note : convention is different from lz4frame, maybe something to review */
cLevel = MIN(LZ4HC_CLEVEL_MAX, cLevel);
if (limit == limitedDestSize)
cLevel = MIN(LZ4HC_CLEVEL_OPT_MIN-1, cLevel); /* no limitedDestSize variant for lz4opt */
assert(cLevel >= 0);
assert(cLevel <= LZ4HC_CLEVEL_MAX);
{ cParams_t const cParam = clTable[cLevel];
@ -655,8 +653,8 @@ static int LZ4HC_compress_generic (
cParam.nbSearches, limit);
assert(cParam.strat == lz4opt);
return LZ4HC_compress_optimal(ctx,
src, dst, *srcSizePtr, dstCapacity, limit,
cParam.nbSearches, cParam.targetLength,
src, dst, srcSizePtr, dstCapacity,
cParam.nbSearches, cParam.targetLength, limit,
cLevel == LZ4HC_CLEVEL_MAX); /* ultra mode */
}
}

61
lib/lz4opt.h
View File

@ -97,11 +97,11 @@ static int LZ4HC_compress_optimal (
LZ4HC_CCtx_internal* ctx,
const char* const source,
char* dst,
int inputSize,
int* srcSizePtr,
int dstCapacity,
limitedOutput_directive limit,
int const nbSearches,
size_t sufficient_len,
limitedOutput_directive limit,
int const fullUpdate
)
{
@ -110,14 +110,17 @@ static int LZ4HC_compress_optimal (
const BYTE* ip = (const BYTE*) source;
const BYTE* anchor = ip;
const BYTE* const iend = ip + inputSize;
const BYTE* const iend = ip + *srcSizePtr;
const BYTE* const mflimit = iend - MFLIMIT;
const BYTE* const matchlimit = iend - LASTLITERALS;
BYTE* op = (BYTE*) dst;
BYTE* const oend = op + dstCapacity;
BYTE* opSaved = (BYTE*) dst;
BYTE* oend = op + dstCapacity;
/* init */
DEBUGLOG(5, "LZ4HC_compress_optimal");
*srcSizePtr = 0;
if (limit == limitedDestSize) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */
if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1;
/* Main Loop */
@ -134,8 +137,9 @@ static int LZ4HC_compress_optimal (
/* good enough solution : immediate encoding */
int const firstML = firstMatch.len;
const BYTE* const matchPos = ip - firstMatch.off;
opSaved = op;
if ( LZ4HC_encodeSequence(&ip, &op, &anchor, firstML, matchPos, limit, oend) ) /* updates ip, op and anchor */
return 0; /* error */
goto _dest_overflow;
continue;
}
@ -304,26 +308,47 @@ encode: /* cur, last_match_pos, best_mlen, best_off must be set */
rPos += ml;
assert(ml >= MINMATCH);
assert((offset >= 1) && (offset <= MAX_DISTANCE));
opSaved = op;
if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) ) /* updates ip, op and anchor */
return 0; /* error */
goto _dest_overflow;
} }
} /* while (ip < mflimit) */
_last_literals:
/* Encode Last Literals */
{ int lastRun = (int)(iend - anchor);
if ( (limit)
&& (((char*)op - dst) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)dstCapacity))
return 0; /* Check output limit */
if (lastRun >= (int)RUN_MASK) {
*op++=(RUN_MASK<<ML_BITS);
lastRun-=RUN_MASK;
for (; lastRun > 254 ; lastRun-=255) *op++ = 255;
*op++ = (BYTE) lastRun;
} else *op++ = (BYTE)(lastRun<<ML_BITS);
memcpy(op, anchor, iend - anchor);
op += iend-anchor;
{ size_t lastRunSize = (size_t)(iend - anchor); /* literals */
size_t litLength = (lastRunSize + 255 - RUN_MASK) / 255;
size_t const totalSize = 1 + litLength + lastRunSize;
if (limit == limitedDestSize) oend += LASTLITERALS; /* restore correct value */
if (limit && (op + totalSize > oend)) {
if (limit == limitedOutput) return 0; /* Check output limit */
/* adapt lastRunSize to fill 'dst' */
lastRunSize = (size_t)(oend - op) - 1;
litLength = (lastRunSize + 255 - RUN_MASK) / 255;
lastRunSize -= litLength;
}
ip = anchor + lastRunSize;
if (lastRunSize >= RUN_MASK) {
size_t accumulator = lastRunSize - RUN_MASK;
*op++ = (RUN_MASK << ML_BITS);
for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255;
*op++ = (BYTE) accumulator;
} else {
*op++ = (BYTE)(lastRunSize << ML_BITS);
}
memcpy(op, anchor, lastRunSize);
op += lastRunSize;
}
/* End */
*srcSizePtr = (int) (((const char*)ip) - source);
return (int) ((char*)op-dst);
_dest_overflow:
if (limit == limitedDestSize) {
op = opSaved; /* restore correct out pointer */
goto _last_literals;
}
return 0;
}

20
tests/Makefile
View File

@ -35,11 +35,12 @@ PRGDIR := ../programs
TESTDIR := versionsTest
PYTHON ?= python3
DEBUGFLAGS = -g -DLZ4_DEBUG=1
CFLAGS ?= -O3 # can select custom optimization flags. For example : CFLAGS=-O2 make
CFLAGS += -g -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
-Wpointer-arith -Wstrict-aliasing=1
CFLAGS += $(MOREFLAGS)
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
CPPFLAGS:= -I$(LZ4DIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_
FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
@ -77,21 +78,26 @@ lz4c unlz4 lz4cat: lz4
lz4c32: # create a 32-bits version for 32/64 interop tests
$(MAKE) -C $(PRGDIR) $@ CFLAGS="-m32 $(CFLAGS)"
fullbench : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/lz4frame.o $(LZ4DIR)/xxhash.o fullbench.c
%.o : $(LZ4DIR)/%.c $(LZ4DIR)/%.h
$(CC) -c $(CFLAGS) $(CPPFLAGS) $< -o $@
fullbench : lz4.o lz4hc.o lz4frame.o xxhash.o fullbench.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fullbench-lib: fullbench.c $(LZ4DIR)/xxhash.c
$(LZ4DIR)/liblz4.a:
$(MAKE) -C $(LZ4DIR) liblz4.a
$(CC) $(FLAGS) $^ -o $@$(EXT) $(LZ4DIR)/liblz4.a
fullbench-lib: fullbench.c $(LZ4DIR)/liblz4.a
$(CC) $(FLAGS) $^ -o $@$(EXT)
fullbench-dll: fullbench.c $(LZ4DIR)/xxhash.c
$(MAKE) -C $(LZ4DIR) liblz4
$(CC) $(FLAGS) $^ -o $@$(EXT) -DLZ4_DLL_IMPORT=1 $(LZ4DIR)/dll/liblz4.dll
fuzzer : $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/xxhash.o fuzzer.c
fuzzer : lz4.o lz4hc.o xxhash.o fuzzer.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
frametest: $(LZ4DIR)/lz4frame.o $(LZ4DIR)/lz4.o $(LZ4DIR)/lz4hc.o $(LZ4DIR)/xxhash.o frametest.c
frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
datagen : $(PRGDIR)/datagen.c datagencli.c

51
tests/fuzzer.c
View File

@ -345,10 +345,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data"); }
DISPLAYLEVEL(5, " OK \n");
}
else
} else {
DISPLAYLEVEL(5, " \n");
}
} }
/* Test compression HC destSize */
FUZ_DISPLAYTEST;
@ -359,11 +358,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(ctx==NULL, "LZ4_createHC() allocation failed");
compressedBuffer[targetSize] = endCheck;
ret = LZ4_compress_HC_destSize(ctx, block, compressedBuffer, &srcSize, targetSize, compressionLevel);
DISPLAYLEVEL(5, "LZ4_compress_HC_destSize(%i): destSize : %7i/%7i; content%7i/%7i ",
compressionLevel, ret, targetSize, srcSize, blockSize);
LZ4_freeHC(ctx);
FUZ_CHECKTEST(ret > targetSize, "LZ4_compress_HC_destSize() result larger than dst buffer !");
FUZ_CHECKTEST(compressedBuffer[targetSize] != endCheck, "LZ4_compress_HC_destSize() overwrite dst buffer !");
FUZ_CHECKTEST(srcSize > blockSize, "LZ4_compress_HC_destSize() fed more than src buffer !");
DISPLAYLEVEL(5, "destSize : %7i/%7i; content%7i/%7i ", ret, targetSize, srcSize, blockSize);
DISPLAYLEVEL(5, "LZ4_compress_HC_destSize(%i): destSize : %7i/%7i; content%7i/%7i ",
compressionLevel, ret, targetSize, srcSize, blockSize);
if (targetSize>0) {
/* check correctness */
U32 const crcBase = XXH32(block, srcSize, 0);
@ -380,10 +382,9 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_CHECKTEST(crcDec!=crcBase, "LZ4_decompress_safe() corrupted decoded data");
}
DISPLAYLEVEL(5, " OK \n");
}
else
} else {
DISPLAYLEVEL(5, " \n");
}
} }
/* Test compression HC */
FUZ_DISPLAYTEST;
@ -898,7 +899,6 @@ static void FUZ_unitTests(int compressionLevel)
}
dict = dst;
//dict = testInput + segStart;
dictSize = segSize;
dst += segSize + 1;
@ -972,28 +972,27 @@ static void FUZ_unitTests(int compressionLevel)
#define BSIZE2 16435
/* first block */
messageSize = BSIZE1;
XXH64_update(&xxhOrig, testInput + iNext, messageSize);
crcOrig = XXH64_digest(&xxhOrig);
messageSize = BSIZE1;
XXH64_update(&xxhOrig, testInput + iNext, messageSize);
crcOrig = XXH64_digest(&xxhOrig);
result = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
result = LZ4_compress_HC_continue(&sHC, testInput + iNext, testCompressed, messageSize, testCompressedSize-ringBufferSize);
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() compression failed");
result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed");
result = LZ4_decompress_safe_continue(&decodeState, testCompressed, testVerify + dNext, result, messageSize);
FUZ_CHECKTEST(result!=(int)messageSize, "64K D.ringBuffer : LZ4_decompress_safe() test failed");
XXH64_update(&xxhNew, testVerify + dNext, messageSize);
{ U64 const crcNew = XXH64_digest(&xxhNew);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); }
XXH64_update(&xxhNew, testVerify + dNext, messageSize);
{ U64 const crcNew = XXH64_digest(&xxhNew);
FUZ_CHECKTEST(crcOrig!=crcNew, "LZ4_decompress_safe() decompression corruption"); }
/* prepare next message */
dNext += messageSize;
totalMessageSize += messageSize;
messageSize = BSIZE2;
iNext = 132000;
memcpy(testInput + iNext, testInput + 8, messageSize);
if (dNext > dBufferSize) dNext = 0;
/* prepare next message */
dNext += messageSize;
totalMessageSize += messageSize;
messageSize = BSIZE2;
iNext = 132000;
memcpy(testInput + iNext, testInput + 8, messageSize);
if (dNext > dBufferSize) dNext = 0;
while (totalMessageSize < 9 MB) {
XXH64_update(&xxhOrig, testInput + iNext, messageSize);