refactored frameCompress example

to better reflect LZ4F API usage.

Makefile

@@ -70,7 +70,7 @@ lz4 lz4-release :
 
 .PHONY: examples
 examples: lib lz4
-	$(MAKE) -C $(EXDIR) test
+	$(MAKE) -C $(EXDIR) all
 
 .PHONY: manuals
 manuals:
@@ -125,6 +125,7 @@ list:
 .PHONY: test
 test:
 	$(MAKE) -C $(TESTDIR) $@
+	$(MAKE) -C $(EXDIR) $@
 
 clangtest: clean
 	clang -v

doc/lz4_manual.html

@@ -176,8 +176,8 @@ int           LZ4_freeStream (LZ4_stream_t* streamPtr);
   If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
 
   Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory !
-  Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
+  Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB.
-  Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
+  Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
 
  @return : size of compressed block
            or 0 if there is an error (typically, compressed data cannot fit into 'dst')

doc/lz4frame_manual.html

@@ -101,8 +101,10 @@
 <a name="Chapter5"></a><h2>Simple compression function</h2><pre></pre>
 
 <pre><b>size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
-</b><p>  Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
+</b><p>  Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences.
-  Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
+ `preferencesPtr` is optional. It can be replaced by NULL, in which case, the function will assume default preferences.
+  Note : this result is only usable with LZ4F_compressFrame().
+         It may also be used with LZ4F_compressUpdate() _if no flush() operation_ is performed.
  
 </p></pre><BR>
 
@@ -151,8 +153,10 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
 </p></pre><BR>
 
 <pre><b>size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr);
-</b><p> Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
+</b><p>  Provides minimum dstCapacity for a given srcSize to guarantee operation success in worst case scenarios.
- prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario.
+  Estimation includes frame footer, which would be generated by LZ4F_compressEnd().
+  Estimation doesn't include frame header, already generated by LZ4F_compressBegin().
+  prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario.
   Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
   When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
  

examples/frameCompress.c

@@ -1,122 +1,133 @@
-// LZ4frame API example : compress a file
+/* LZ4frame API example : compress a file
-// Based on sample code from Zbigniew Jędrzejewski-Szmek
+ * Based on sample code from Zbigniew Jędrzejewski-Szmek
+ *
+ * This example streams an input file into an output file
+ * using a bounded memory budget.
+ * Input is read in chunks of IN_CHUNK_SIZE */
 
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <errno.h>
+#include <assert.h>
 
 #include <lz4frame.h>
 
-#define BUF_SIZE 16*1024
+
-#define LZ4_HEADER_SIZE 19
+#define IN_CHUNK_SIZE  (16*1024)
-#define LZ4_FOOTER_SIZE 4
 
 static const LZ4F_preferences_t lz4_preferences = {
     { LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame,
-      0 /* content size unknown */, 0 /* no dictID */ , LZ4F_noBlockChecksum },
+      0 /* unknown content size */, 0 /* no dictID */ , LZ4F_noBlockChecksum },
-    0,   /* compression level */
+    0,   /* compression level; 0 == default */
     0,   /* autoflush */
     { 0, 0, 0, 0 },  /* reserved, must be set to 0 */
 };
 
-static size_t compress_file(FILE *in, FILE *out, size_t *size_in, size_t *size_out) {
-    size_t r=1;  /* function result; 1 == error, default (early exit) */
-    LZ4F_compressionContext_t ctx;
-    char *src, *buf = NULL;
-    size_t size, count_in = 0, count_out, offset = 0, frame_size;
 
-    if (LZ4F_isError( LZ4F_createCompressionContext(&ctx, LZ4F_VERSION) )) {
+/* safe_fwrite() :
-        printf("Failed to create context: error %zu\n", r);
+ * performs fwrite(), ensure operation success, or immediately exit() */
-        return 1;
+static void safe_fwrite(void* buf, size_t eltSize, size_t nbElt, FILE* f)
+{
+    size_t const writtenSize = fwrite(buf, eltSize, nbElt, f);
+    size_t const expectedSize = eltSize * nbElt;   /* note : should check for overflow */
+    if (writtenSize < expectedSize) {
+        if (ferror(f))  /* note : ferror() must follow fwrite */
+            printf("Write failed\n");
+        else
+            printf("Short write\n");
+        exit(1);
+    }
 }
 
-    src = malloc(BUF_SIZE);
+
+static size_t
+compress_file(FILE* in, FILE* out,
+              unsigned long long* size_in,
+              unsigned long long* size_out)
+{
+    size_t result = 1;  /* function result; 1 == error, default (early exit) */
+    unsigned long long count_in = 0, count_out;
+
+    /* init */
+    LZ4F_compressionContext_t ctx;
+    if (LZ4F_isError( LZ4F_createCompressionContext(&ctx, LZ4F_VERSION) )) {
+        printf("error: failed to create context \n");
+        return result;
+    }
+
+    char* outbuff = NULL;
+    void* const src = malloc(IN_CHUNK_SIZE);
     if (!src) {
         printf("Not enough memory\n");
         goto cleanup;
     }
 
-    frame_size = LZ4F_compressBound(BUF_SIZE, &lz4_preferences);
+    size_t const outbufCapacity = LZ4F_compressBound(IN_CHUNK_SIZE, &lz4_preferences);   /* large enough for any input <= IN_CHUNK_SIZE */
-    size = frame_size + LZ4_HEADER_SIZE + LZ4_FOOTER_SIZE;
+    outbuff = malloc(outbufCapacity);
-    buf = malloc(size);
+    if (!outbuff) {
-    if (!buf) {
         printf("Not enough memory\n");
         goto cleanup;
     }
 
-    {   size_t const headerSize = LZ4F_compressBegin(ctx, buf, size, &lz4_preferences);
+    /* write frame header */
+    assert(outbufCapacity >= LZ4F_HEADER_SIZE_MAX);
+    {   size_t const headerSize = LZ4F_compressBegin(ctx, outbuff, outbufCapacity, &lz4_preferences);
         if (LZ4F_isError(headerSize)) {
             printf("Failed to start compression: error %zu\n", headerSize);
             goto cleanup;
         }
-        offset = count_out = headerSize;
+        count_out = headerSize;
-        printf("Buffer size is %zu bytes, header size %zu bytes\n", size, headerSize);
+        printf("Buffer size is %zu bytes, header size %zu bytes\n", outbufCapacity, headerSize);
+        safe_fwrite(outbuff, 1, headerSize, out);
     }
 
-
+    /* stream file */
     for (;;) {
-        size_t const readSize = fread(src, 1, BUF_SIZE, in);
+        size_t const readSize = fread(src, 1, outbufCapacity, in);
-        if (readSize == 0)
+        if (readSize == 0) break;
-            break;
         count_in += readSize;
 
-        {   size_t const compressedSize = LZ4F_compressUpdate(ctx, buf + offset, size - offset, src, readSize, NULL);
+        size_t const compressedSize = LZ4F_compressUpdate(ctx,
+                                                outbuff, outbufCapacity,
+                                                src, readSize,
+                                                NULL);
         if (LZ4F_isError(compressedSize)) {
             printf("Compression failed: error %zu\n", compressedSize);
             goto cleanup;
         }
-            offset += compressedSize;
+
+        printf("Writing %zu bytes\n", compressedSize);
+        safe_fwrite(outbuff, 1, compressedSize, out);
         count_out += compressedSize;
     }
 
-        if (size - offset < frame_size + LZ4_FOOTER_SIZE) {
+    /* flush whatever remains within internal buffers */
-            size_t writtenSize;
+    {   size_t const compressedSize = LZ4F_compressEnd(ctx,
-            printf("Writing %zu bytes\n", offset);
+                                            outbuff, outbufCapacity,
-
+                                            NULL);
-            writtenSize = fwrite(buf, 1, offset, out);
-            if (writtenSize < offset) {
-                if (ferror(out))  /* note : ferror() must follow fwrite */
-                    printf("Write failed\n");
-                else
-                    printf("Short write\n");
-                goto cleanup;
-            }
-
-            offset = 0;
-        }
-    }
-
-    {   size_t const compressedSize = LZ4F_compressEnd(ctx, buf + offset, size - offset, NULL);
         if (LZ4F_isError(compressedSize)) {
             printf("Failed to end compression: error %zu\n", compressedSize);
             goto cleanup;
         }
-        offset += compressedSize;
+
+        printf("Writing %zu bytes\n", compressedSize);
+        safe_fwrite(outbuff, 1, compressedSize, out);
         count_out += compressedSize;
     }
 
-    printf("Writing %zu bytes\n", offset);
-    {   size_t const writtenSize = fwrite(buf, 1, offset, out);
-        if (writtenSize < offset) {
-            if (ferror(out))
-                printf("Write failed\n");
-            else
-                printf("Short write\n");
-            goto cleanup;
-    }   }
-
     *size_in = count_in;
     *size_out = count_out;
-    r = 0;   /* success */
+    result = 0;   /* success */
 
  cleanup:
     LZ4F_freeCompressionContext(ctx);   /* supports free on NULL */
     free(src);
-    free(buf);
+    free(outbuff);
-    return r;
+    return result;
 }
 
+
 static size_t get_block_size(const LZ4F_frameInfo_t* info) {
     switch (info->blockSizeID) {
         case LZ4F_default:
@@ -131,7 +142,7 @@ static size_t get_block_size(const LZ4F_frameInfo_t* info) {
 }
 
 static size_t decompress_file(FILE* in, FILE* out) {
-    void* const src = malloc(BUF_SIZE);
+    void* const src = malloc(IN_CHUNK_SIZE);
     void* dst = NULL;
     size_t dstCapacity = 0;
     LZ4F_dctx* dctx = NULL;
@@ -148,7 +159,7 @@ static size_t decompress_file(FILE* in, FILE* out) {
     /* Decompression */
     while (ret != 0) {
         /* Load more input */
-        size_t srcSize = fread(src, 1, BUF_SIZE, in);
+        size_t srcSize = fread(src, 1, IN_CHUNK_SIZE, in);
         const void* srcPtr = src;
         const void* const srcEnd = srcPtr + srcSize;
         if (srcSize == 0 || ferror(in)) {
@@ -215,6 +226,7 @@ cleanup:
     return LZ4F_freeDecompressionContext(dctx);   /* note : free works on NULL */
 }
 
+
 int compare(FILE* fp0, FILE* fp1)
 {
     int result = 0;
@@ -238,6 +250,7 @@ int compare(FILE* fp0, FILE* fp1)
     return result;
 }
 
+
 int main(int argc, const char **argv) {
     char inpFilename[256] = { 0 };
     char lz4Filename[256] = { 0 };
@@ -259,8 +272,8 @@ int main(int argc, const char **argv) {
     /* compress */
     {   FILE* const inpFp = fopen(inpFilename, "rb");
         FILE* const outFp = fopen(lz4Filename, "wb");
-        size_t sizeIn = 0;
+        unsigned long long sizeIn = 0;
-        size_t sizeOut = 0;
+        unsigned long long sizeOut = 0;
         size_t ret;
 
         printf("compress : %s -> %s\n", inpFilename, lz4Filename);
@@ -270,7 +283,8 @@ int main(int argc, const char **argv) {
             return (int)ret;
         }
         printf("%s: %zu → %zu bytes, %.1f%%\n",
-            inpFilename, sizeIn, sizeOut,
+            inpFilename,
+            (size_t)sizeIn, (size_t)sizeOut,  /* might overflow */
             (double)sizeOut / sizeIn * 100);
         printf("compress : done\n");
 

lib/lz4frame.c

@@ -726,7 +726,8 @@ size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr,
 
 
     if (cctxPtr->cStage != 1) return err0r(LZ4F_ERROR_GENERIC);
-    if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize)) return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
+    if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize))
+        return err0r(LZ4F_ERROR_dstMaxSize_tooSmall);
     memset(&cOptionsNull, 0, sizeof(cOptionsNull));
     if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull;
 

lib/lz4frame.h

@@ -189,8 +189,10 @@ LZ4FLIB_API int LZ4F_compressionLevel_max(void);
 *  Simple compression function
 ***********************************/
 /*! LZ4F_compressFrameBound() :
- *  Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences.
+ *  Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences.
- *  Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression.
+ * `preferencesPtr` is optional. It can be replaced by NULL, in which case, the function will assume default preferences.
+ *  Note : this result is only usable with LZ4F_compressFrame().
+ *         It may also be used with LZ4F_compressUpdate() _if no flush() operation_ is performed.
  */
 LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
 
@@ -235,7 +237,7 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
 
 /*----    Compression    ----*/
 
-#define LZ4F_HEADER_SIZE_MAX 19
+#define LZ4F_HEADER_SIZE_MAX 19   /* LZ4 Frame header size can vary from 7 to 19 bytes */
 /*! LZ4F_compressBegin() :
  *  will write the frame header into dstBuffer.
  *  dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes.
@@ -248,7 +250,9 @@ LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx,
                                       const LZ4F_preferences_t* prefsPtr);
 
 /*! LZ4F_compressBound() :
- *  Provides minimum dstCapacity for a given srcSize to guarantee operation success in worst case situations.
+ *  Provides minimum dstCapacity for a given srcSize to guarantee operation success in worst case scenarios.
+ *  Estimation includes frame footer, which would be generated by LZ4F_compressEnd().
+ *  Estimation doesn't include frame header, already generated by LZ4F_compressBegin().
  *  prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario.
  *  Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
  *  When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.