AuroraMiddleware/lz4
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

refactored frameCompress example

Browse Source 
to better reflect LZ4F API usage.
This commit is contained in:
Yann Collet 2018-01-31 14:33:16 -08:00
parent 8258f4d9cb
commit 87fb7a1d03
6 changed files with 120 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Makefile
View File

@ -70,7 +70,7 @@ lz4 lz4-release :
.PHONY: examples
examples: lib lz4
$(MAKE) -C $(EXDIR) test
$(MAKE) -C $(EXDIR) all
.PHONY: manuals
manuals:
@ -125,6 +125,7 @@ list:
.PHONY: test
test:
$(MAKE) -C $(TESTDIR) $@
$(MAKE) -C $(EXDIR) $@
clangtest: clean
clang -v

4
doc/lz4_manual.html
View File

@ -176,8 +176,8 @@ int LZ4_freeStream (LZ4_stream_t* streamPtr);
If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory !
Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
@return : size of compressed block
or 0 if there is an error (typically, compressed data cannot fit into 'dst')

12
doc/lz4frame_manual.html
View File

@ -101,8 +101,10 @@
<a name="Chapter5"></a><h2>Simple compression function</h2><pre></pre>
<pre><b>size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
</b><p> Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
</b><p> Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences.
`preferencesPtr` is optional. It can be replaced by NULL, in which case, the function will assume default preferences.
Note : this result is only usable with LZ4F_compressFrame().
It may also be used with LZ4F_compressUpdate() _if no flush() operation_ is performed.
</p></pre><BR>
@ -151,8 +153,10 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
</p></pre><BR>
<pre><b>size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr);
</b><p> Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario.
</b><p> Provides minimum dstCapacity for a given srcSize to guarantee operation success in worst case scenarios.
Estimation includes frame footer, which would be generated by LZ4F_compressEnd().
Estimation doesn't include frame header, already generated by LZ4F_compressBegin().
prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario.
Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.

146
examples/frameCompress.c
View File

@ -1,122 +1,133 @@
// LZ4frame API example : compress a file
// Based on sample code from Zbigniew Jędrzejewski-Szmek
/* LZ4frame API example : compress a file
* Based on sample code from Zbigniew Jędrzejewski-Szmek
*
* This example streams an input file into an output file
* using a bounded memory budget.
* Input is read in chunks of IN_CHUNK_SIZE */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <assert.h>
#include <lz4frame.h>
#define BUF_SIZE 16*1024
#define LZ4_HEADER_SIZE 19
#define LZ4_FOOTER_SIZE 4
#define IN_CHUNK_SIZE (16*1024)
static const LZ4F_preferences_t lz4_preferences = {
{ LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame,
0 /* content size unknown */, 0 /* no dictID */ , LZ4F_noBlockChecksum },
0, /* compression level */
0 /* unknown content size */, 0 /* no dictID */ , LZ4F_noBlockChecksum },
0, /* compression level; 0 == default */
0, /* autoflush */
{ 0, 0, 0, 0 }, /* reserved, must be set to 0 */
};
static size_t compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) {
size_t r=1; /* function result; 1 == error, default (early exit) */
LZ4F_compressionContext_t ctx;
char *src, *buf = NULL;
size_t size, count_in = 0, count_out, offset = 0, frame_size;
if (LZ4F_isError( LZ4F_createCompressionContext(&ctx, LZ4F_VERSION) )) {
printf("Failed to create context: error %zu\n", r);
return 1;
/* safe_fwrite() :
* performs fwrite(), ensure operation success, or immediately exit() */
static void safe_fwrite(void* buf, size_t eltSize, size_t nbElt, FILE* f)
{
size_t const writtenSize = fwrite(buf, eltSize, nbElt, f);
size_t const expectedSize = eltSize * nbElt; /* note : should check for overflow */
if (writtenSize < expectedSize) {
if (ferror(f)) /* note : ferror() must follow fwrite */
printf("Write failed\n");
else
printf("Short write\n");
exit(1);
}
}
src = malloc(BUF_SIZE);
static size_t
compress_file(FILE* in, FILE* out,
unsigned long long* size_in,
unsigned long long* size_out)
{
size_t result = 1; /* function result; 1 == error, default (early exit) */
unsigned long long count_in = 0, count_out;
/* init */
LZ4F_compressionContext_t ctx;
if (LZ4F_isError( LZ4F_createCompressionContext(&ctx, LZ4F_VERSION) )) {
printf("error: failed to create context \n");
return result;
}
char* outbuff = NULL;
void* const src = malloc(IN_CHUNK_SIZE);
if (!src) {
printf("Not enough memory\n");
goto cleanup;
}
frame_size = LZ4F_compressBound(BUF_SIZE, &lz4_preferences);
size = frame_size + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE;
buf = malloc(size);
if (!buf) {
size_t const outbufCapacity = LZ4F_compressBound(IN_CHUNK_SIZE, &lz4_preferences); /* large enough for any input <= IN_CHUNK_SIZE */
outbuff = malloc(outbufCapacity);
if (!outbuff) {
printf("Not enough memory\n");
goto cleanup;
}
{ size_t const headerSize = LZ4F_compressBegin(ctx, buf, size, &lz4_preferences);
/* write frame header */
assert(outbufCapacity >= LZ4F_HEADER_SIZE_MAX);
{ size_t const headerSize = LZ4F_compressBegin(ctx, outbuff, outbufCapacity, &lz4_preferences);
if (LZ4F_isError(headerSize)) {
printf("Failed to start compression: error %zu\n", headerSize);
goto cleanup;
}
offset = count_out = headerSize;
printf("Buffer size is %zu bytes, header size %zu bytes\n", size, headerSize);
count_out = headerSize;
printf("Buffer size is %zu bytes, header size %zu bytes\n", outbufCapacity, headerSize);
safe_fwrite(outbuff, 1, headerSize, out);
}
/* stream file */
for (;;) {
size_t const readSize = fread(src, 1, BUF_SIZE, in);
if (readSize == 0)
break;
size_t const readSize = fread(src, 1, outbufCapacity, in);
if (readSize == 0) break;
count_in += readSize;
{ size_t const compressedSize = LZ4F_compressUpdate(ctx, buf + offset, size - offset, src, readSize, NULL);
size_t const compressedSize = LZ4F_compressUpdate(ctx,
outbuff, outbufCapacity,
src, readSize,
NULL);
if (LZ4F_isError(compressedSize)) {
printf("Compression failed: error %zu\n", compressedSize);
goto cleanup;
}
offset += compressedSize;
printf("Writing %zu bytes\n", compressedSize);
safe_fwrite(outbuff, 1, compressedSize, out);
count_out += compressedSize;
}
if (size - offset < frame_size + LZ4_FOOTER_SIZE) {
size_t writtenSize;
printf("Writing %zu bytes\n", offset);
writtenSize = fwrite(buf, 1, offset, out);
if (writtenSize < offset) {
if (ferror(out)) /* note : ferror() must follow fwrite */
printf("Write failed\n");
else
printf("Short write\n");
goto cleanup;
}
offset = 0;
}
}
{ size_t const compressedSize = LZ4F_compressEnd(ctx, buf + offset, size - offset, NULL);
/* flush whatever remains within internal buffers */
{ size_t const compressedSize = LZ4F_compressEnd(ctx,
outbuff, outbufCapacity,
NULL);
if (LZ4F_isError(compressedSize)) {
printf("Failed to end compression: error %zu\n", compressedSize);
goto cleanup;
}
offset += compressedSize;
printf("Writing %zu bytes\n", compressedSize);
safe_fwrite(outbuff, 1, compressedSize, out);
count_out += compressedSize;
}
printf("Writing %zu bytes\n", offset);
{ size_t const writtenSize = fwrite(buf, 1, offset, out);
if (writtenSize < offset) {
if (ferror(out))
printf("Write failed\n");
else
printf("Short write\n");
goto cleanup;
} }
*size_in = count_in;
*size_out = count_out;
r = 0; /* success */
result = 0; /* success */
cleanup:
LZ4F_freeCompressionContext(ctx); /* supports free on NULL */
free(src);
free(buf);
return r;
free(outbuff);
return result;
}
static size_t get_block_size(const LZ4F_frameInfo_t* info) {
switch (info->blockSizeID) {
case LZ4F_default:
@ -131,7 +142,7 @@ static size_t get_block_size(const LZ4F_frameInfo_t* info) {
}
static size_t decompress_file(FILE* in, FILE* out) {
void* const src = malloc(BUF_SIZE);
void* const src = malloc(IN_CHUNK_SIZE);
void* dst = NULL;
size_t dstCapacity = 0;
LZ4F_dctx* dctx = NULL;
@ -148,7 +159,7 @@ static size_t decompress_file(FILE* in, FILE* out) {
/* Decompression */
while (ret != 0) {
/* Load more input */
size_t srcSize = fread(src, 1, BUF_SIZE, in);
size_t srcSize = fread(src, 1, IN_CHUNK_SIZE, in);
const void* srcPtr = src;
const void* const srcEnd = srcPtr + srcSize;
if (srcSize == 0 || ferror(in)) {
@ -215,6 +226,7 @@ cleanup:
return LZ4F_freeDecompressionContext(dctx); /* note : free works on NULL */
}
int compare(FILE* fp0, FILE* fp1)
{
int result = 0;
@ -238,6 +250,7 @@ int compare(FILE* fp0, FILE* fp1)
return result;
}
int main(int argc, const char **argv) {
char inpFilename[256] = { 0 };
char lz4Filename[256] = { 0 };
@ -259,8 +272,8 @@ int main(int argc, const char **argv) {
/* compress */
{ FILE* const inpFp = fopen(inpFilename, "rb");
FILE* const outFp = fopen(lz4Filename, "wb");
size_t sizeIn = 0;
size_t sizeOut = 0;
unsigned long long sizeIn = 0;
unsigned long long sizeOut = 0;
size_t ret;
printf("compress : %s -> %s\n", inpFilename, lz4Filename);
@ -270,7 +283,8 @@ int main(int argc, const char **argv) {
return (int)ret;
}
printf("%s: %zu → %zu bytes, %.1f%%\n",
inpFilename, sizeIn, sizeOut,
inpFilename,
(size_t)sizeIn, (size_t)sizeOut, /* might overflow */
(double)sizeOut / sizeIn * 100);
printf("compress : done\n");

3
lib/lz4frame.c
View File

@ -726,7 +726,8 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize)) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
memset(&cOptionsNull, 0, sizeof(cOptionsNull));
if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull;

12
lib/lz4frame.h
View File

@ -189,8 +189,10 @@ LZ4FLIB_API int LZ4F_compressionLevel_max(void);
* Simple compression function
***********************************/
/*! LZ4F_compressFrameBound() :
* Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
* Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
* Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences.
* `preferencesPtr` is optional. It can be replaced by NULL, in which case, the function will assume default preferences.
* Note : this result is only usable with LZ4F_compressFrame().
* It may also be used with LZ4F_compressUpdate() _if no flush() operation_ is performed.
*/
LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
@ -235,7 +237,7 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
/*---- Compression ----*/
#define LZ4F_HEADER_SIZE_MAX 19
#define LZ4F_HEADER_SIZE_MAX 19 /* LZ4 Frame header size can vary from 7 to 19 bytes */
/*! LZ4F_compressBegin() :
* will write the frame header into dstBuffer.
* dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
@ -248,7 +250,9 @@ LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx,
const LZ4F_preferences_t* prefsPtr);
/*! LZ4F_compressBound() :
* Provides minimum dstCapacity for a given srcSize to guarantee operation success in worst case situations.
* Provides minimum dstCapacity for a given srcSize to guarantee operation success in worst case scenarios.
* Estimation includes frame footer, which would be generated by LZ4F_compressEnd().
* Estimation doesn't include frame header, already generated by LZ4F_compressBegin().
* prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario.
* Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
* When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.