AuroraMiddleware/lz4
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

frame content size support

Browse Source
This commit is contained in:
Yann Collet 2015-03-18 21:38:27 +01:00
parent 7d87d43e61
commit 7ee725645b
6 changed files with 220 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
NEWS
View File

@ -1,6 +1,7 @@
r128:
New : lz4 cli sparse file support
New : command -m, to compress multiple files in a single command
New : lz4frame supports frame content size
Fixed : Restored lz4hc compression ratio (was slightly lower since r124)
New : lz4frame supports skippable frames
Changed:Default "make install" directory is /usr/local

203
lib/lz4frame.c
View File

@ -33,7 +33,7 @@ You can contact the author at :
*/
/* LZ4F is a stand-alone API to create LZ4-compressed Frames
* in full conformance with specification v1.4.1.
* in full conformance with specification v1.5.0
* All related operations, including memory management, are handled by the library.
* */
@ -100,9 +100,10 @@ typedef unsigned long long U64;
#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U
#define LZ4F_MAGICNUMBER 0x184D2204U
#define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U
#define LZ4F_MAXHEADERFRAME_SIZE 7
#define LZ4F_MAXHEADERFRAME_SIZE 15
#define LZ4F_BLOCKSIZEID_DEFAULT max64KB
static const size_t minFHSize = 5;
static const U32 minHClevel = 3;
/**************************************
@ -111,23 +112,24 @@ static const U32 minHClevel = 3;
typedef struct
{
LZ4F_preferences_t prefs;
U32 version;
U32 cStage;
U32 version;
U32 cStage;
size_t maxBlockSize;
size_t maxBufferSize;
BYTE* tmpBuff;
BYTE* tmpIn;
size_t tmpInSize;
U64 totalInSize;
XXH32_state_t xxh;
void* lz4CtxPtr;
U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */
void* lz4CtxPtr;
U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */
} LZ4F_cctx_internal_t;
typedef struct
{
LZ4F_frameInfo_t frameInfo;
unsigned version;
unsigned dStage;
U32 version;
U32 dStage;
size_t maxBlockSize;
size_t maxBufferSize;
const BYTE* srcExpect;
@ -141,7 +143,7 @@ typedef struct
size_t tmpOutSize;
size_t tmpOutStart;
XXH32_state_t xxh;
BYTE header[8];
BYTE header[16];
} LZ4F_dctx_internal_t;
@ -152,7 +154,7 @@ typedef struct
static const char* LZ4F_errorStrings[] = { LZ4F_LIST_ERRORS(LZ4F_GENERATE_STRING) };
U32 LZ4F_isError(LZ4F_errorCode_t code)
unsigned LZ4F_isError(LZ4F_errorCode_t code)
{
return (code > (LZ4F_errorCode_t)(-ERROR_maxCode));
}
@ -180,14 +182,6 @@ static size_t LZ4F_getBlockSize(unsigned blockSizeID)
/* unoptimized version; solves endianess & alignment issues */
static void LZ4F_writeLE32 (BYTE* dstPtr, U32 value32)
{
dstPtr[0] = (BYTE)value32;
dstPtr[1] = (BYTE)(value32 >> 8);
dstPtr[2] = (BYTE)(value32 >> 16);
dstPtr[3] = (BYTE)(value32 >> 24);
}
static U32 LZ4F_readLE32 (const BYTE* srcPtr)
{
U32 value32 = srcPtr[0];
@ -197,10 +191,43 @@ static U32 LZ4F_readLE32 (const BYTE* srcPtr)
return value32;
}
static BYTE LZ4F_headerChecksum (const BYTE* header, size_t length)
static void LZ4F_writeLE32 (BYTE* dstPtr, U32 value32)
{
U32 xxh = XXH32(header, (U32)length, 0);
dstPtr[0] = (BYTE)value32;
dstPtr[1] = (BYTE)(value32 >> 8);
dstPtr[2] = (BYTE)(value32 >> 16);
dstPtr[3] = (BYTE)(value32 >> 24);
}
static U64 LZ4F_readLE64 (const BYTE* srcPtr)
{
U64 value64 = srcPtr[0];
value64 += (srcPtr[1]<<8);
value64 += (srcPtr[2]<<16);
value64 += (srcPtr[3]<<24);
value64 += ((U64)srcPtr[4]<<32);
value64 += ((U64)srcPtr[5]<<40);
value64 += ((U64)srcPtr[6]<<48);
value64 += ((U64)srcPtr[7]<<56);
return value64;
}
static void LZ4F_writeLE64 (BYTE* dstPtr, U64 value64)
{
dstPtr[0] = (BYTE)value64;
dstPtr[1] = (BYTE)(value64 >> 8);
dstPtr[2] = (BYTE)(value64 >> 16);
dstPtr[3] = (BYTE)(value64 >> 24);
dstPtr[4] = (BYTE)(value64 >> 32);
dstPtr[5] = (BYTE)(value64 >> 40);
dstPtr[6] = (BYTE)(value64 >> 48);
dstPtr[7] = (BYTE)(value64 >> 56);
}
static BYTE LZ4F_headerChecksum (const void* header, size_t length)
{
U32 xxh = XXH32(header, length, 0);
return (BYTE)(xxh >> 8);
}
@ -235,7 +262,7 @@ size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* prefere
prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize);
prefs.autoFlush = 1;
headerSize = 7; /* basic header size (no option) including magic number */
headerSize = 15; /* header size, including magic number and frame content size*/
streamSize = LZ4F_compressBound(srcSize, &prefs);
return headerSize + streamSize;
@ -269,7 +296,13 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuf
cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent allocation; works because autoflush==1 & stableSrc==1 */
if (preferencesPtr!=NULL) prefs = *preferencesPtr;
else memset(&prefs, 0, sizeof(prefs));
else
{
memset(&prefs, 0, sizeof(prefs));
prefs.frameInfo.frameOSize = (U64)srcSize;
}
if (prefs.frameInfo.frameOSize != 0)
prefs.frameInfo.frameOSize = (U64)srcSize; /* correct frame size if selected (!=0) */
if (prefs.compressionLevel < minHClevel)
{
@ -412,13 +445,22 @@ size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* ds
/* FLG Byte */
*dstPtr++ = ((1 & _2BITS) << 6) /* Version('01') */
+ ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) /* Block mode */
+ (char)((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2); /* Stream checksum */
+ (BYTE)((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) /* Frame checksum */
+ (BYTE)((cctxPtr->prefs.frameInfo.frameOSize > 0) << 3); /* Frame content size */
/* BD Byte */
*dstPtr++ = (char)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4);
*dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4);
/* Optional Frame content size field */
if (cctxPtr->prefs.frameInfo.frameOSize)
{
LZ4F_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.frameOSize);
dstPtr += 8;
cctxPtr->totalInSize = 0;
}
/* CRC Byte */
*dstPtr++ = LZ4F_headerChecksum(headerStart, 2);
*dstPtr = LZ4F_headerChecksum(headerStart, dstPtr - headerStart);
dstPtr++;
cctxPtr->cStage = 1; /* header written, wait for data block */
cctxPtr->cStage = 1; /* header written, now request input data block */
return (dstPtr - dstStart);
}
@ -608,8 +650,9 @@ size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* d
}
if (cctxPtr->prefs.frameInfo.contentChecksumFlag == contentChecksumEnabled)
XXH32_update(&(cctxPtr->xxh), srcBuffer, (unsigned)srcSize);
XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize);
cctxPtr->totalInSize += srcSize;
return dstPtr - dstStart;
}
@ -686,6 +729,12 @@ size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstB
cctxPtr->cStage = 0; /* state is now re-usable (with identical preferences) */
if (cctxPtr->prefs.frameInfo.frameOSize)
{
if (cctxPtr->prefs.frameInfo.frameOSize != cctxPtr->totalInSize)
return (size_t)-ERROR_frameSize_wrong;
}
return dstPtr - dstStart;
}
@ -735,7 +784,7 @@ typedef enum { dstage_getHeader=0, dstage_storeHeader,
dstage_getCBlock, dstage_storeCBlock, dstage_decodeCBlock,
dstage_decodeCBlock_intoDst, dstage_decodeCBlock_intoTmp, dstage_flushOut,
dstage_getSuffix, dstage_storeSuffix,
dstage_getSBlockSize, dstage_storeSBlockSize,
dstage_getSFrameSize, dstage_storeSFrameSize,
dstage_skipSkippable
} dStage_t;
@ -745,72 +794,84 @@ typedef enum { dstage_getHeader=0, dstage_storeHeader,
or an error code (testable with LZ4F_isError())
output : set internal values of dctx, such as
dctxPtr->frameInfo and dctxPtr->dStage.
*/
static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const void* srcVoidPtr, size_t srcSize)
{
BYTE FLG, BD, HC;
unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, dictFlag, blockSizeID;
unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, blockSizeID;
size_t bufferNeeded;
size_t frameHeaderSize;
const BYTE* srcPtr = (const BYTE*)srcVoidPtr;
/* need to decode header to get frameInfo */
if (srcSize < 7) return (size_t)-ERROR_GENERIC; /* minimal header size */
if (srcSize < minFHSize) return (size_t)-ERROR_GENERIC; /* minimal header size */
memset(&(dctxPtr->frameInfo), 0, sizeof(dctxPtr->frameInfo));
/* skippable frames */
if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START)
{
memset(&(dctxPtr->frameInfo), 0, sizeof(dctxPtr->frameInfo));
dctxPtr->frameInfo.frameType = skippableFrame;
if (srcVoidPtr == (void*)(dctxPtr->header))
{
memcpy(dctxPtr->header, dctxPtr->header + 4, srcSize-4);
dctxPtr->tmpInSize = srcSize-4;
dctxPtr->dStage = dstage_storeSBlockSize;
dctxPtr->tmpInSize = srcSize;
dctxPtr->tmpInTarget = 8;
dctxPtr->dStage = dstage_storeSFrameSize;
return srcSize;
}
else
{
dctxPtr->dStage = dstage_getSBlockSize;
dctxPtr->dStage = dstage_getSFrameSize;
return 4;
}
}
/* control magic number */
if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return (size_t)-ERROR_GENERIC;
srcPtr += 4;
if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return (size_t)-ERROR_frameType_unknown;
dctxPtr->frameInfo.frameType = LZ4F_frame;
/* Flags */
FLG = srcPtr[0];
FLG = srcPtr[4];
version = (FLG>>6) & _2BITS;
blockMode = (FLG>>5) & _1BIT;
blockChecksumFlag = (FLG>>4) & _1BIT;
contentSizeFlag = (FLG>>3) & _1BIT;
contentChecksumFlag = (FLG>>2) & _1BIT;
dictFlag = (FLG>>0) & _1BIT;
BD = srcPtr[1];
blockSizeID = (BD>>4) & _3BITS;
/* check */
HC = LZ4F_headerChecksum(srcPtr, 2);
if (HC != srcPtr[2]) return (size_t)-ERROR_GENERIC; /* Bad header checksum error */
/* Frame Header Size */
frameHeaderSize = contentSizeFlag ? 15 : 7;
if (srcSize < frameHeaderSize)
{
if (srcPtr != dctxPtr->header)
memcpy(dctxPtr->header, srcPtr, srcSize);
dctxPtr->tmpInSize = srcSize;
dctxPtr->tmpInTarget = frameHeaderSize;
dctxPtr->dStage = dstage_storeHeader;
return srcSize;
}
BD = srcPtr[5];
blockSizeID = (BD>>4) & _3BITS;
/* validate */
if (version != 1) return (size_t)-ERROR_GENERIC; /* Version Number, only supported value */
if (blockChecksumFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */
if (contentSizeFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */
if (((FLG>>1)&_1BIT) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bit */
if (dictFlag != 0) return (size_t)-ERROR_GENERIC; /* Only supported value for the time being */
if (((FLG>>0)&_2BITS) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bits */
if (((BD>>7)&_1BIT) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bit */
if (blockSizeID < 4) return (size_t)-ERROR_GENERIC; /* 4-7 only supported values for the time being */
if (((BD>>0)&_4BITS) != 0) return (size_t)-ERROR_GENERIC; /* Reserved bits */
/* check */
HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5);
if (HC != srcPtr[frameHeaderSize-1]) return (size_t)-ERROR_GENERIC; /* Bad header checksum error */
/* save */
dctxPtr->frameInfo.blockMode = (blockMode_t)blockMode;
dctxPtr->frameInfo.contentChecksumFlag = (contentChecksum_t)contentChecksumFlag;
dctxPtr->frameInfo.blockSizeID = (blockSizeID_t)blockSizeID;
dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID);
if (contentSizeFlag)
dctxPtr->frameInfo.frameOSize = LZ4F_readLE64(srcPtr+6);
/* init */
if (contentChecksumFlag) XXH32_reset(&(dctxPtr->xxh), 0);
@ -837,7 +898,7 @@ static size_t LZ4F_decodeHeader(LZ4F_dctx_internal_t* dctxPtr, const void* srcVo
dctxPtr->dStage = dstage_getCBlockSize;
return 7;
return frameHeaderSize;
}
@ -872,15 +933,14 @@ LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t decompressionCont
}
/* redirector, with common prototype */
static int LZ4F_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize)
{
(void)dictStart;
(void)dictSize;
(void)dictStart; (void)dictSize;
return LZ4_decompress_safe (source, dest, compressedSize, maxDecompressedSize);
}
static void LZ4F_updateDict(LZ4F_dctx_internal_t* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp)
{
if (dctxPtr->dictSize==0)
@ -1011,23 +1071,24 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
break;
}
dctxPtr->tmpInSize = 0;
dctxPtr->tmpInTarget = 7;
dctxPtr->dStage = dstage_storeHeader;
}
case dstage_storeHeader:
{
size_t sizeToCopy = 7 - dctxPtr->tmpInSize;
size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize;
if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr;
memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
dctxPtr->tmpInSize += sizeToCopy;
srcPtr += sizeToCopy;
if (dctxPtr->tmpInSize < 7)
if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget)
{
nextSrcSizeHint = (7 - dctxPtr->tmpInSize) + 4;
doAnotherStage = 0; /* not enough src data, wait to get some more */
nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + 4;
doAnotherStage = 0; /* not enough src data, ask for some more */
break;
}
LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, dctxPtr->header, 7);
LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget);
if (LZ4F_isError(errorCode)) return errorCode;
break;
}
@ -1292,7 +1353,7 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
break;
}
case dstage_getSBlockSize:
case dstage_getSFrameSize:
{
if ((srcEnd - srcPtr) >= 4)
{
@ -1302,32 +1363,34 @@ size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext,
else
{
/* not enough input to read cBlockSize field */
dctxPtr->tmpInSize = 0;
dctxPtr->dStage = dstage_storeSBlockSize;
dctxPtr->tmpInSize = 4;
dctxPtr->tmpInTarget = 8;
dctxPtr->dStage = dstage_storeSFrameSize;
}
}
if (dctxPtr->dStage == dstage_storeSBlockSize)
case dstage_storeSBlockSize:
if (dctxPtr->dStage == dstage_storeSFrameSize)
case dstage_storeSFrameSize:
{
size_t sizeToCopy = 4 - dctxPtr->tmpInSize;
size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize;
if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr;
memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy);
srcPtr += sizeToCopy;
dctxPtr->tmpInSize += sizeToCopy;
if (dctxPtr->tmpInSize < 4) /* not enough input to get full sBlockSize; wait for more */
if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) /* not enough input to get full sBlockSize; wait for more */
{
nextSrcSizeHint = 4 - dctxPtr->tmpInSize;
doAnotherStage=0;
nextSrcSizeHint = dctxPtr->tmpInTarget - dctxPtr->tmpInSize;
doAnotherStage = 0;
break;
}
selectedIn = dctxPtr->header;
selectedIn = dctxPtr->header + 4;
}
/* case dstage_decodeSBlockSize: */ /* no direct access */
{
size_t nextSBlockSize = LZ4F_readLE32(selectedIn);
dctxPtr->tmpInTarget = nextSBlockSize;
size_t SFrameSize = LZ4F_readLE32(selectedIn);
dctxPtr->frameInfo.frameOSize = SFrameSize;
dctxPtr->tmpInTarget = SFrameSize;
dctxPtr->dStage = dstage_skipSkippable;
break;
}

17
lib/lz4frame.h
View File

@ -68,18 +68,19 @@ typedef enum { noContentChecksum=0, contentChecksumEnabled } contentChecksum_t;
typedef enum { LZ4F_frame=0, skippableFrame } frameType_t;
typedef struct {
blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */
contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */
frameType_t frameType; /* LZ4F_frame, skippableFrame : 0 == default */
unsigned reserved[4];
blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */
contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */
frameType_t frameType; /* LZ4F_frame, skippableFrame ; 0 == default */
unsigned long long frameOSize; /* Size of uncompressed (original) content ; 0 == unknown */
unsigned reserved[2]; /* must be zero for forward compatibility */
} LZ4F_frameInfo_t;
typedef struct {
LZ4F_frameInfo_t frameInfo;
unsigned compressionLevel; /* 0 == default (fast mode); values above 16 count as 16 */
unsigned autoFlush; /* 1 == always flush : reduce need for tmp buffer */
unsigned reserved[4];
unsigned autoFlush; /* 1 == always flush (reduce need for tmp buffer) */
unsigned reserved[4]; /* must be zero for forward compatibility */
} LZ4F_preferences_t;
@ -131,7 +132,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* preferencesPtr);
/* LZ4F_compressBegin() :
* will write the frame header into dstBuffer.
* dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is 19 bytes.
* dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is 15 bytes.
* The LZ4F_preferences_t structure is optional : you can provide NULL as argument, all preferences will then be set to default.
* The result of the function is the number of bytes written into dstBuffer for the header
* or an error code (can be tested using LZ4F_isError())

2
lib/lz4frame_static.h
View File

@ -52,6 +52,8 @@ extern "C" {
ITEM(ERROR_compressionLevel_invalid) \
ITEM(ERROR_allocation_failed) \
ITEM(ERROR_srcSize_tooLarge) ITEM(ERROR_dstMaxSize_tooSmall) \
ITEM(ERROR_frameSize_wrong) \
ITEM(ERROR_frameType_unknown) \
ITEM(ERROR_wrongSrcPtr) \
ITEM(ERROR_decompressionFailed) \
ITEM(ERROR_checksum_invalid) \

76
programs/frametest.c
View File

@ -381,6 +381,57 @@ int basicTests(U32 seed, double compressibility)
if (LZ4F_isError(cSize)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
{
size_t errorCode;
BYTE* const ostart = (BYTE*)compressedBuffer;
BYTE* op = ostart;
LZ4F_compressionContext_t cctx;
errorCode = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION);
if (LZ4F_isError(errorCode)) goto _output_error;
DISPLAYLEVEL(3, "compress without frameSize : \n");
memset(&(prefs.frameInfo), 0, sizeof(prefs.frameInfo));
errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
if (LZ4F_isError(errorCode)) goto _output_error;
op += errorCode;
errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
op += errorCode;
errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart));
DISPLAYLEVEL(3, "compress with frameSize : \n");
prefs.frameInfo.frameOSize = testSize;
op = ostart;
errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
if (LZ4F_isError(errorCode)) goto _output_error;
op += errorCode;
errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
op += errorCode;
errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart));
DISPLAYLEVEL(3, "compress with wrong frameSize : \n");
prefs.frameInfo.frameOSize = testSize+1;
op = ostart;
errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
if (LZ4F_isError(errorCode)) goto _output_error;
op += errorCode;
errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
op += errorCode;
errorCode = LZ4F_compressEnd(cctx, op, testSize, NULL);
if (LZ4F_isError(errorCode)) { DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(errorCode)); }
else
goto _output_error;
errorCode = LZ4F_freeCompressionContext(cctx);
if (LZ4F_isError(errorCode)) goto _output_error;
}
DISPLAYLEVEL(3, "Skippable frame test : \n");
{
size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
@ -412,11 +463,13 @@ int basicTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)decodedBufferSize);
/* generate zero-size skippable frame */
DISPLAYLEVEL(3, "zero-size skippable frame\n");
ip = (BYTE*)compressedBuffer;
op = (BYTE*)decodedBuffer;
FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+1);
FUZ_writeLE32(ip+4, 0);
iend = ip+8;
DISPLAYLEVEL(3, "random segment sizes : \n");
while (ip < iend)
{
unsigned nbBits = FUZ_rand(&randState) % maxBits;
@ -428,8 +481,27 @@ int basicTests(U32 seed, double compressibility)
op += oSize;
ip += iSize;
}
DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)decodedBufferSize);
DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8));
DISPLAYLEVEL(3, "Skippable frame header complete in first call \n");
ip = (BYTE*)compressedBuffer;
op = (BYTE*)decodedBuffer;
FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+2);
FUZ_writeLE32(ip+4, 10);
iend = ip+18;
while (ip < iend)
{
size_t iSize = 10;
size_t oSize = 10;
if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
if (LZ4F_isError(errorCode)) goto _output_error;
op += oSize;
ip += iSize;
}
DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8));
/* release memory */
errorCode = LZ4F_freeDecompressionContext(dCtx);
if (LZ4F_isError(errorCode)) goto _output_error;
}

2
programs/lz4io.c
View File

@ -625,7 +625,7 @@ static unsigned long long decodeLZ4S(FILE* finput, FILE* foutput)
if (seekResult != 0) EXM_THROW(68, "1 GB skip error (sparse file)");
storedSkips -= 1 GB;
}
if (nb0T != seg0SizeT)
if (nb0T != seg0SizeT) /* not all 0s */
{
seekResult = fseek(foutput, storedSkips, SEEK_CUR);
if (seekResult) EXM_THROW(68, "Skip error (sparse file)");