From 7aad356ffded1306492d3b5326ab2c014ba701ed Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 11 Nov 2016 11:31:52 -0800 Subject: [PATCH] Add frame decompression example --- examples/frameCompress.c | 263 ++++++++++++++++++++++++++++++--------- 1 file changed, 206 insertions(+), 57 deletions(-) diff --git a/examples/frameCompress.c b/examples/frameCompress.c index b52b042..b7a3576 100644 --- a/examples/frameCompress.c +++ b/examples/frameCompress.c @@ -8,7 +8,7 @@ #include -#define BUF_SIZE (16*1024) +#define BUF_SIZE 16*1024 #define LZ4_HEADER_SIZE 19 #define LZ4_FOOTER_SIZE 4 @@ -19,7 +19,7 @@ static const LZ4F_preferences_t lz4_preferences = { { 0, 0, 0, 0 }, /* reserved, must be set to 0 */ }; -static int compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) { +static size_t compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) { LZ4F_errorCode_t r; LZ4F_compressionContext_t ctx; char *src, *buf = NULL; @@ -27,14 +27,14 @@ static int compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) r = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); if (LZ4F_isError(r)) { - printf("Failed to create context: error %zu", r); + printf("Failed to create context: error %zu\n", r); return 1; } r = 1; src = malloc(BUF_SIZE); if (!src) { - printf("Not enough memory"); + printf("Not enough memory\n"); goto cleanup; } @@ -42,13 +42,13 @@ static int compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) size = frame_size + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE; buf = malloc(size); if (!buf) { - printf("Not enough memory"); + printf("Not enough memory\n"); goto cleanup; } n = offset = count_out = LZ4F_compressBegin(ctx, buf, size, &lz4_preferences); if (LZ4F_isError(n)) { - printf("Failed to start compression: error %zu", n); + printf("Failed to start compression: error %zu\n", n); goto cleanup; } @@ -62,7 +62,7 @@ static int compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) n = LZ4F_compressUpdate(ctx, buf + offset, size - offset, src, k, NULL); if (LZ4F_isError(n)) { - printf("Compression failed: error %zu", n); + printf("Compression failed: error %zu\n", n); goto cleanup; } @@ -74,9 +74,9 @@ static int compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) k = fwrite(buf, 1, offset, out); if (k < offset) { if (ferror(out)) - printf("Write failed"); + printf("Write failed\n"); else - printf("Short write"); + printf("Short write\n"); goto cleanup; } @@ -86,7 +86,7 @@ static int compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) n = LZ4F_compressEnd(ctx, buf + offset, size - offset, NULL); if (LZ4F_isError(n)) { - printf("Failed to end compression: error %zu", n); + printf("Failed to end compression: error %zu\n", n); goto cleanup; } @@ -97,9 +97,9 @@ static int compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) k = fwrite(buf, 1, offset, out); if (k < offset) { if (ferror(out)) - printf("Write failed"); + printf("Write failed\n"); else - printf("Short write"); + printf("Short write\n"); goto cleanup; } @@ -114,56 +114,205 @@ static int compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) return r; } -static int compress(const char *input, const char *output) { - char *tmp = NULL; - FILE *in = NULL, *out = NULL; - size_t size_in = 0, size_out = 0; - int r = 1; - - if (!output) { - size_t len = strlen(input); - - output = tmp = malloc(len + 5); - if (!tmp) { - printf("Not enough memory"); - return 1; - } - strcpy(tmp, input); - strcpy(tmp + len, ".lz4"); +static size_t get_block_size(const LZ4F_frameInfo_t* info) { + switch (info->blockSizeID) { + case LZ4F_max64KB: return 1 << 16; + case LZ4F_max256KB: return 1 << 18; + case LZ4F_max1MB: return 1 << 20; + case LZ4F_max4MB: return 1 << 22; + default: + printf("Impossible unless more block sizes are allowed\n"); + exit(1); } - - in = fopen(input, "rb"); - if (!in) { - fprintf(stderr, "Failed to open input file %s: %s\n", input, strerror(errno)); - goto cleanup; - } - - out = fopen(output, "wb"); - if (!out) { - fprintf(stderr, "Failed to open output file %s: %s\n", output, strerror(errno)); - goto cleanup; - } - - r = compress_file(in, out, &size_in, &size_out); - if (r == 0) - printf("%s: %zu → %zu bytes, %.1f%%\n", - input, size_in, size_out, - (double)size_out / size_in * 100); - cleanup: - if (in) - fclose(in); - if (out) - fclose(out); - free(tmp); - return r; } +static size_t decompress_file(FILE *in, FILE *out) { + void * const src = malloc(BUF_SIZE); + void *dst = NULL; + void *srcPtr = src; + void *srcEnd = src; + size_t srcSize = 0; + size_t dstSize = 0; + size_t dstCapacity = 0; + LZ4F_dctx *dctx = NULL; + size_t ret; + + /* Initialization */ + ret = LZ4F_createDecompressionContext(&dctx, 100); + if (LZ4F_isError(ret)) { + printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(ret)); + goto cleanup; + } + + /* Decompression */ + ret = 1; + while (ret != 0) { + /* INVARIANT: At this point srcPtr == srcEnd */ + /* Load more input */ + srcSize = fread(src, 1, BUF_SIZE, in); + if (srcSize == 0 || ferror(in)) { + printf("Decompress: not enough input or error reading file\n"); + goto cleanup; + } + srcPtr = src; + srcEnd = srcPtr + srcSize; + /* Allocate destination buffer if it isn't already */ + if (!dst) { + LZ4F_frameInfo_t info; + ret = LZ4F_getFrameInfo(dctx, &info, src, &srcSize); + if (LZ4F_isError(ret)) { + printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(ret)); + goto cleanup; + } + /* Allocating enough space for an entire block isn't necessary for + * correctness, but it allows some memcpy's to be elided. + */ + dstCapacity = get_block_size(&info); + dst = malloc(dstCapacity); + srcPtr += srcSize; + srcSize = srcEnd - srcPtr; + } + /* Decompress: + * Continue while there is more input to read and the frame isn't over. + * If srcPtr == srcEnd then we know that there is no more output left in the + * internal buffer left to flush. + */ + while (srcPtr != srcEnd && ret != 0) { + /* INVARIANT: Any data left in dst has already been written */ + dstSize = dstCapacity; + ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL); + if (LZ4F_isError(ret)) { + printf("Decompression error: %s\n", LZ4F_getErrorName(ret)); + goto cleanup; + } + /* Flush output */ + if (dstSize != 0){ + size_t written = fwrite(dst, 1, dstSize, out); + printf("Writing %zu bytes\n", dstSize); + if (written != dstSize) { + printf("Decompress: Failed to write to file\n"); + ret = 1; + goto cleanup; + } + } + /* Update input */ + srcPtr += srcSize; + srcSize = srcEnd - srcPtr; + } + } + /* Check that there isn't trailing input data after the frame. + * It is valid to have multiple frames in the same file, but this example + * doesn't support it. + */ + ret = fread(src, 1, 1, in); + if (ret != 0 || !feof(in)) { + printf("Decompress: Trailing data left in file after frame\n"); + goto cleanup; + } + +cleanup: + free(src); + if (dst) free(dst); + if (dctx) ret = LZ4F_freeDecompressionContext(dctx); + return ret; +} + +int compare(FILE* fp0, FILE* fp1) +{ + int result = 0; + + while(0 == result) { + char b0[1024]; + char b1[1024]; + const size_t r0 = fread(b0, 1, sizeof(b0), fp0); + const size_t r1 = fread(b1, 1, sizeof(b1), fp1); + + result = (int) r0 - (int) r1; + + if (0 == r0 || 0 == r1) { + break; + } + if (0 == result) { + result = memcmp(b0, b1, r0); + } + } + + return result; +} int main(int argc, char **argv) { - if (argc < 2 || argc > 3) { - fprintf(stderr, "Syntax: %s \n", argv[0]); - return EXIT_FAILURE; + char inpFilename[256] = { 0 }; + char lz4Filename[256] = { 0 }; + char decFilename[256] = { 0 }; + + if(argc < 2) { + printf("Please specify input filename\n"); + return 0; } - return compress(argv[1], argv[2]); + snprintf(inpFilename, 256, "%s", argv[1]); + snprintf(lz4Filename, 256, "%s.lz4", argv[1]); + snprintf(decFilename, 256, "%s.lz4.dec", argv[1]); + + printf("inp = [%s]\n", inpFilename); + printf("lz4 = [%s]\n", lz4Filename); + printf("dec = [%s]\n", decFilename); + + /* compress */ + { + FILE* inpFp = fopen(inpFilename, "rb"); + FILE* outFp = fopen(lz4Filename, "wb"); + size_t sizeIn = 0; + size_t sizeOut = 0; + size_t ret; + + printf("compress : %s -> %s\n", inpFilename, lz4Filename); + ret = compress_file(inpFp, outFp, &sizeIn, &sizeOut); + if (ret) { + printf("compress : failed with code %zu\n", ret); + return ret; + } + printf("%s: %zu → %zu bytes, %.1f%%\n", + inpFilename, sizeIn, sizeOut, + (double)sizeOut / sizeIn * 100); + printf("compress : done\n"); + + fclose(outFp); + fclose(inpFp); + } + + /* decompress */ + { + FILE* inpFp = fopen(lz4Filename, "rb"); + FILE* outFp = fopen(decFilename, "wb"); + size_t ret; + + printf("decompress : %s -> %s\n", lz4Filename, decFilename); + ret = decompress_file(inpFp, outFp); + if (ret) { + printf("decompress : failed with code %zu\n", ret); + return ret; + } + printf("decompress : done\n"); + + fclose(outFp); + fclose(inpFp); + } + + /* verify */ + { + FILE* inpFp = fopen(inpFilename, "rb"); + FILE* decFp = fopen(decFilename, "rb"); + + printf("verify : %s <-> %s\n", inpFilename, decFilename); + const int cmp = compare(inpFp, decFp); + if(0 == cmp) { + printf("verify : OK\n"); + } else { + printf("verify : NG\n"); + } + + fclose(decFp); + fclose(inpFp); + } }