AuroraMiddleware/lz4
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #383 from lz4/blockChecksum

Browse Source 
Block checksum
This commit is contained in:
Yann Collet 2017-08-15 16:50:04 -07:00 committed by GitHub
commit af9d72b7f6
10 changed files with 479 additions and 373 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
NEWS
View File

@ -4,6 +4,7 @@ cli : added GNU separator -- specifying that all following arguments are files
API : added LZ4_compress_HC_destSize(), by Oleg (@remittor)
API : added LZ4F_resetDecompressionContext()
API : lz4frame : negative compression levels trigger fast acceleration, request by Lawrence Chan
API : lz4frame : can control block checksum and dictionary ID
API : fix : expose obsolete decoding functions, reported by Chen Yufei
build : fix : static lib installation, by Ido Rosen
build : dragonFlyBSD, OpenBSD, NetBSD supported

25
doc/lz4frame_manual.html
View File

@ -62,6 +62,11 @@
LZ4F_OBSOLETE_ENUM(contentChecksumEnabled)
} LZ4F_contentChecksum_t;
</b></pre><BR>
<pre><b>typedef enum {
LZ4F_noBlockChecksum=0,
LZ4F_blockChecksumEnabled
} LZ4F_blockChecksum_t;
</b></pre><BR>
<pre><b>typedef enum {
LZ4F_frame=0,
LZ4F_skippableFrame
@ -70,16 +75,16 @@
</b></pre><BR>
<pre><b>typedef struct {
LZ4F_blockSizeID_t blockSizeID; </b>/* max64KB, max256KB, max1MB, max4MB ; 0 == default */<b>
LZ4F_blockMode_t blockMode; </b>/* blockLinked, blockIndependent ; 0 == default */<b>
LZ4F_contentChecksum_t contentChecksumFlag; </b>/* noContentChecksum, contentChecksumEnabled ; 0 == default */<b>
LZ4F_frameType_t frameType; </b>/* LZ4F_frame, skippableFrame ; 0 == default */<b>
LZ4F_blockMode_t blockMode; </b>/* LZ4F_blockLinked, LZ4F_blockIndependent ; 0 == default */<b>
LZ4F_contentChecksum_t contentChecksumFlag; </b>/* if enabled, frame is terminated with a 32-bits checksum of decompressed data ; 0 == disabled (default) */<b>
LZ4F_frameType_t frameType; </b>/* read-only field : LZ4F_frame or LZ4F_skippableFrame */<b>
unsigned long long contentSize; </b>/* Size of uncompressed content ; 0 == unknown */<b>
unsigned dictID; </b>/* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */<b>
unsigned reserved[1]; </b>/* must be zero for forward compatibility */<b>
LZ4F_blockChecksum_t blockChecksumFlag; </b>/* if enabled, each block is followed by a checksum of block's compressed data ; 0 == disabled (default) */<b>
} LZ4F_frameInfo_t;
</b><p> makes it possible to supply detailed frame parameters to the stream interface.
It's not required to set all fields, as long as the structure was initially memset() to zero.
All reserved fields must be set to zero.
</b><p> makes it possible to set or read frame parameters.
It's not required to set all fields, as long as the structure was initially memset() to zero.
For all fields, 0 sets it to default value
</p></pre><BR>
<pre><b>typedef struct {
@ -88,9 +93,9 @@
unsigned autoFlush; </b>/* 1 == always flush, to reduce usage of internal buffers */<b>
unsigned reserved[4]; </b>/* must be zero for forward compatibility */<b>
} LZ4F_preferences_t;
</b><p> makes it possible to supply detailed compression parameters to the stream interface.
It's not required to set all fields, as long as the structure was initially memset() to zero.
All reserved fields must be set to zero.
</b><p> makes it possible to supply detailed compression parameters to the stream interface.
It's not required to set all fields, as long as the structure was initially memset() to zero.
All reserved fields must be set to zero.
</p></pre><BR>
<a name="Chapter5"></a><h2>Simple compression function</h2><pre></pre>

6
examples/frameCompress.c
View File

@ -14,7 +14,7 @@
static const LZ4F_preferences_t lz4_preferences = {
{ LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame,
0 /* content size unknown */, 0 /* no dictID */ , { 0 } /* reserved */ },
0 /* content size unknown */, 0 /* no dictID */ , LZ4F_noBlockChecksum },
0, /* compression level */
0, /* autoflush */
{ 0, 0, 0, 0 }, /* reserved, must be set to 0 */
@ -266,7 +266,7 @@ int main(int argc, const char **argv) {
ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut);
if (ret) {
printf("compress : failed with code %zu\n", ret);
return ret;
return (int)ret;
}
printf("%s: %zu → %zu bytes, %.1f%%\n",
inpFilename, sizeIn, sizeOut,
@ -286,7 +286,7 @@ int main(int argc, const char **argv) {
ret = decompress_file(inpFp, outFp);
if (ret) {
printf("decompress : failed with code %zu\n", ret);
return ret;
return (int)ret;
}
printf("decompress : done\n");

751
lib/lz4frame.c
View File

File diff suppressed because it is too large Load Diff

25
lib/lz4frame.h
View File

@ -139,6 +139,11 @@ typedef enum {
LZ4F_OBSOLETE_ENUM(contentChecksumEnabled)
} LZ4F_contentChecksum_t;
typedef enum {
LZ4F_noBlockChecksum=0,
LZ4F_blockChecksumEnabled
} LZ4F_blockChecksum_t;
typedef enum {
LZ4F_frame=0,
LZ4F_skippableFrame
@ -153,23 +158,23 @@ typedef LZ4F_contentChecksum_t contentChecksum_t;
#endif
/*! LZ4F_frameInfo_t :
* makes it possible to supply detailed frame parameters to the stream interface.
* It's not required to set all fields, as long as the structure was initially memset() to zero.
* All reserved fields must be set to zero. */
* makes it possible to set or read frame parameters.
* It's not required to set all fields, as long as the structure was initially memset() to zero.
* For all fields, 0 sets it to default value */
typedef struct {
LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
LZ4F_blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */
LZ4F_contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */
LZ4F_frameType_t frameType; /* LZ4F_frame, skippableFrame ; 0 == default */
LZ4F_blockMode_t blockMode; /* LZ4F_blockLinked, LZ4F_blockIndependent ; 0 == default */
LZ4F_contentChecksum_t contentChecksumFlag; /* if enabled, frame is terminated with a 32-bits checksum of decompressed data ; 0 == disabled (default) */
LZ4F_frameType_t frameType; /* read-only field : LZ4F_frame or LZ4F_skippableFrame */
unsigned long long contentSize; /* Size of uncompressed content ; 0 == unknown */
unsigned dictID; /* Dictionary ID, sent by the compressor to help decoder select the correct dictionary; 0 == no dictID provided */
unsigned reserved[1]; /* must be zero for forward compatibility */
LZ4F_blockChecksum_t blockChecksumFlag; /* if enabled, each block is followed by a checksum of block's compressed data ; 0 == disabled (default) */
} LZ4F_frameInfo_t;
/*! LZ4F_preferences_t :
* makes it possible to supply detailed compression parameters to the stream interface.
* It's not required to set all fields, as long as the structure was initially memset() to zero.
* All reserved fields must be set to zero. */
* makes it possible to supply detailed compression parameters to the stream interface.
* It's not required to set all fields, as long as the structure was initially memset() to zero.
* All reserved fields must be set to zero. */
typedef struct {
LZ4F_frameInfo_t frameInfo;
int compressionLevel; /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */

2
lib/lz4frame_static.h
View File

@ -59,7 +59,7 @@ extern "C" {
ITEM(ERROR_contentChecksumFlag_invalid) \
ITEM(ERROR_compressionLevel_invalid) \
ITEM(ERROR_headerVersion_wrong) \
ITEM(ERROR_blockChecksum_unsupported) \
ITEM(ERROR_blockChecksum_invalid) \
ITEM(ERROR_reservedFlag_set) \
ITEM(ERROR_allocation_failed) \
ITEM(ERROR_srcSize_tooLarge) \

2
programs/lz4cli.c
View File

@ -144,7 +144,7 @@ static int usage_advanced(const char* exeName)
DISPLAY( " -l : compress using Legacy format (Linux kernel compression)\n");
DISPLAY( " -B# : Block size [4-7] (default : 7) \n");
DISPLAY( " -BD : Block dependency (improve compression ratio) \n");
/* DISPLAY( " -BX : enable block checksum (default:disabled)\n"); *//* Option currently inactive */
DISPLAY( " -BX : enable block checksum (default:disabled) \n");
DISPLAY( "--no-frame-crc : disable stream checksum (default:enabled) \n");
DISPLAY( "--content-size : compressed frame includes original size (default:not present)\n");
DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n");

11
programs/lz4io.c
View File

@ -173,17 +173,17 @@ int LZ4IO_setBlockMode(LZ4IO_blockMode_t blockMode)
return g_blockIndependence;
}
/* Default setting : no checksum */
int LZ4IO_setBlockChecksumMode(int xxhash)
/* Default setting : no block checksum */
int LZ4IO_setBlockChecksumMode(int enable)
{
g_blockChecksum = (xxhash != 0);
g_blockChecksum = (enable != 0);
return g_blockChecksum;
}
/* Default setting : checksum enabled */
int LZ4IO_setStreamChecksumMode(int xxhash)
int LZ4IO_setStreamChecksumMode(int enable)
{
g_streamChecksum = (xxhash != 0);
g_streamChecksum = (enable != 0);
return g_streamChecksum;
}
@ -455,6 +455,7 @@ static int LZ4IO_compressFilename_extRess(cRess_t ress, const char* srcFileName,
prefs.compressionLevel = compressionLevel;
prefs.frameInfo.blockMode = (LZ4F_blockMode_t)g_blockIndependence;
prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)g_blockSizeId;
prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)g_blockChecksum;
prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)g_streamChecksum;
if (g_contentSizeFlag) {
U64 const fileSize = UTIL_getFileSize(srcFileName);

1
tests/Makefile
View File

@ -263,6 +263,7 @@ test-lz4-basic: lz4 datagen unlz4 lz4cat
$(LZ4) -f tmp
cat tmp >> tmp.lz4
$(LZ4) -f tmp.lz4 # uncompress valid frame followed by invalid data
$(LZ4) -BX tmp -c -q | $(LZ4) -tv # test block checksum
@$(RM) tmp*
test-lz4-hugefile: lz4 datagen

28
tests/frametest.c
View File

@ -402,7 +402,7 @@ int basicTests(U32 seed, double compressibility)
CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "without checksum : ");
DISPLAYLEVEL(3, "without frame checksum : ");
prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
@ -416,7 +416,7 @@ int basicTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
}
DISPLAYLEVEL(3, "without checksum : ");
DISPLAYLEVEL(3, "without frame checksum : ");
prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
{ size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs);
DISPLAYLEVEL(4, "dstCapacity = %u ; ", (U32)dstCapacity)
@ -424,6 +424,29 @@ int basicTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
}
DISPLAYLEVEL(3, "LZ4F_compressFrame with block checksum : ");
memset(&prefs, 0, sizeof(prefs));
prefs.frameInfo.blockChecksumFlag = LZ4F_blockChecksumEnabled;
CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
DISPLAYLEVEL(3, "Decompress with block checksum : ");
{ size_t iSize = cSize;
size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
LZ4F_decompressionContext_t dctx;
CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
CHECK( LZ4F_decompress(dctx, decodedBuffer, &decodedSize, compressedBuffer, &iSize, NULL) );
if (decodedSize != testSize) goto _output_error;
if (iSize != cSize) goto _output_error;
{ U64 const crcDest = XXH64(decodedBuffer, decodedSize, 1);
U64 const crcSrc = XXH64(CNBuffer, testSize, 1);
if (crcDest != crcSrc) goto _output_error;
}
DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
CHECK( LZ4F_freeDecompressionContext(dctx) );
}
/* frame content size tests */
{ size_t cErr;
BYTE* const ostart = (BYTE*)compressedBuffer;
@ -771,6 +794,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
memset(&prefs, 0, sizeof(prefs));
prefs.frameInfo.blockMode = (LZ4F_blockMode_t)(FUZ_rand(&randState) & 1);
prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)(4 + (FUZ_rand(&randState) & 3));
prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)(FUZ_rand(&randState) & 1);
prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)(FUZ_rand(&randState) & 1);
prefs.frameInfo.contentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0;
prefs.autoFlush = neverFlush ? 0 : (FUZ_rand(&randState) & 7) == 2;