Merge pull request #1456 from yijinfb/example-ZSTD_compressCCtx

Add multiple_simple_compression.c (demonstrates the use of ZSTD_compressCCtx(...)) to examples directory.

examples/.gitignore

@@ -1,6 +1,7 @@
 #build
 simple_compression
 simple_decompression
+multiple_simple_compression
 dictionary_compression
 dictionary_decompression
 streaming_compression

examples/Makefile

@@ -17,6 +17,7 @@ LIB = ../lib/libzstd.a
 default: all
 
 all: simple_compression simple_decompression \
+	multiple_simple_compression\
 	dictionary_compression dictionary_decompression \
 	streaming_compression streaming_decompression \
 	multiple_streaming_compression streaming_memory_usage
@@ -24,33 +25,37 @@ all: simple_compression simple_decompression \
 $(LIB) :
 	$(MAKE) -C ../lib libzstd.a
 
-simple_compression : simple_compression.c $(LIB)
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+simple_compression : simple_compression.c utils.h $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
-simple_decompression : simple_decompression.c $(LIB)
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+simple_decompression : simple_decompression.c utils.h $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
-dictionary_compression : dictionary_compression.c $(LIB)
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+multiple_simple_compression : multiple_simple_compression.c utils.h $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
-dictionary_decompression : dictionary_decompression.c $(LIB)
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+dictionary_compression : dictionary_compression.c utils.h $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
-streaming_compression : streaming_compression.c $(LIB)
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+dictionary_decompression : dictionary_decompression.c utils.h $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
-multiple_streaming_compression : multiple_streaming_compression.c $(LIB)
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+streaming_compression : streaming_compression.c utils.h $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
-streaming_decompression : streaming_decompression.c $(LIB)
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+multiple_streaming_compression : multiple_streaming_compression.c utils.h $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
+
+streaming_decompression : streaming_decompression.c utils.h $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
 streaming_memory_usage : streaming_memory_usage.c $(LIB)
-	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+	$(CC) $(CPPFLAGS) $(CFLAGS) $< $(LIB) $(LDFLAGS) -o $@
 
 clean:
 	@rm -f core *.o tmp* result* *.zst \
         simple_compression simple_decompression \
+        multiple_simple_compression \
         dictionary_compression dictionary_decompression \
         streaming_compression streaming_decompression \
         multiple_streaming_compression streaming_memory_usage
@@ -62,6 +67,7 @@ test: all
 	@echo -- Simple compression tests
 	./simple_compression tmp
 	./simple_decompression tmp.zst
+	./multiple_simple_compression *.c
 	./streaming_decompression tmp.zst > /dev/null
 	@echo -- Streaming memory usage
 	./streaming_memory_usage

examples/README.md

@@ -11,8 +11,14 @@ Zstandard library : usage examples
   Result remains in memory.
   Introduces usage of : `ZSTD_decompress()`
 
+- [Multiple simple compression](multiple_simple_compression.c) :
+  Compress multiple files (in simple mode) in a single command line.
+  Demonstrates memory preservation technique that
+  minimizes malloc()/free() calls by re-using existing resources.
+  Introduces usage of : `ZSTD_compressCCtx()`
+
 - [Streaming memory usage](streaming_memory_usage.c) :
-  Provides amount of memory used by streaming context
+  Provides amount of memory used by streaming context.
   Introduces usage of : `ZSTD_sizeof_CStream()`
 
 - [Streaming compression](streaming_compression.c) :
@@ -20,7 +26,7 @@ Zstandard library : usage examples
   Introduces usage of : `ZSTD_compressStream()`
 
 - [Multiple Streaming compression](multiple_streaming_compression.c) :
-  Compress multiple files in a single command line.
+  Compress multiple files (in streaming mode) in a single command line.
   Introduces memory usage preservation technique,
   reducing impact of malloc()/free() and memset() by re-using existing resources.
 

examples/dictionary_compression.c

@@ -21,7 +21,7 @@ static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel)
 {
     size_t dictSize;
     printf("loading dictionary %s \n", dictFileName);
-    void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize);
+    void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize);
     ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, cLevel);
     if (!cdict) {
         fprintf(stderr, "ZSTD_createCDict error \n");
@@ -35,7 +35,7 @@ static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel)
 static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdict)
 {
     size_t fSize;
-    void* const fBuff = loadFile_orDie(fname, &fSize);
+    void* const fBuff = mallocAndLoadFile_orDie(fname, &fSize);
     size_t const cBuffSize = ZSTD_compressBound(fSize);
     void* const cBuff = malloc_orDie(cBuffSize);
 

examples/dictionary_decompression.c

@@ -25,7 +25,7 @@ static ZSTD_DDict* createDict_orDie(const char* dictFileName)
 {
     size_t dictSize;
     printf("loading dictionary %s \n", dictFileName);
-    void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize);
+    void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize);
     ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize);
     if (ddict==NULL) { fprintf(stderr, "ZSTD_createDDict error \n"); exit(5); }
     free(dictBuffer);
@@ -35,7 +35,7 @@ static ZSTD_DDict* createDict_orDie(const char* dictFileName)
 static void decompress(const char* fname, const ZSTD_DDict* ddict)
 {
     size_t cSize;
-    void* const cBuff = loadFile_orDie(fname, &cSize);
+    void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize);
     unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
     if (rSize==ZSTD_CONTENTSIZE_ERROR) {
         fprintf(stderr, "%s : it was not compressed by zstd.\n", fname);

examples/multiple_simple_compression.c

@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include <stdlib.h>    // malloc, free, exit
+#include <stdio.h>     // fprintf, perror, fopen, etc.
+#include <string.h>    // strlen, strcat, memset, strerror
+#include <errno.h>     // errno
+#include <sys/stat.h>  // stat
+#include <zstd.h>      // presumes zstd library is installed
+#include "utils.h"
+
+typedef struct {
+    void* fBuffer;
+    void* cBuffer;
+    size_t fBufferSize;
+    size_t cBufferSize;
+    ZSTD_CCtx* cctx;
+} resources;
+
+/*
+ * allocate memory for buffers big enough to compress all files
+ * as well as memory for output file name (ofn)
+ */
+static resources createResources_orDie(int argc, const char** argv, char **ofn, int* ofnBufferLen)
+{
+    size_t maxFilenameLength=0;
+    size_t maxFileSize = 0;
+
+    int argNb;
+    for (argNb = 1; argNb < argc; argNb++) {
+      const char* const filename = argv[argNb];
+      size_t const filenameLength = strlen(filename);
+      size_t const fileSize = fsize_orDie(filename);
+
+      if (filenameLength > maxFilenameLength) maxFilenameLength = filenameLength;
+      if (fileSize > maxFileSize) maxFileSize = fileSize;
+    }
+
+    resources ress;
+    ress.fBufferSize = maxFileSize;
+    ress.cBufferSize = ZSTD_compressBound(maxFileSize);
+
+    *ofnBufferLen = maxFilenameLength + 5;
+    *ofn = (char*)malloc_orDie(*ofnBufferLen);
+    ress.fBuffer = malloc_orDie(ress.fBufferSize);
+    ress.cBuffer = malloc_orDie(ress.cBufferSize);
+    ress.cctx = ZSTD_createCCtx();
+    if (ress.cctx==NULL) { fprintf(stderr, "ZSTD_createCCtx() error \n"); exit(10); }
+    return ress;
+}
+
+static void freeResources(resources ress, char *outFilename)
+{
+    free(ress.fBuffer);
+    free(ress.cBuffer);
+    ZSTD_freeCCtx(ress.cctx);   /* never fails */
+    free(outFilename);
+}
+
+/* compress with pre-allocated context (ZSTD_CCtx) and input/output buffers*/
+static void compressFile_orDie(resources ress, const char* fname, const char* oname)
+{
+    size_t fSize = loadFile_orDie(fname, ress.fBuffer, ress.fBufferSize);
+
+    size_t const cSize = ZSTD_compressCCtx(ress.cctx, ress.cBuffer, ress.cBufferSize, ress.fBuffer, fSize, 1);
+    if (ZSTD_isError(cSize)) {
+        fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize));
+        exit(8);
+    }
+
+    saveFile_orDie(oname, ress.cBuffer, cSize);
+
+    /* success */
+    // printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
+}
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc<2) {
+        printf("wrong arguments\n");
+        printf("usage:\n");
+        printf("%s FILE(s)\n", exeName);
+        return 1;
+    }
+
+    /* memory allocation for outFilename and resources */
+    char* outFilename;
+    int outFilenameBufferLen;
+    resources const ress = createResources_orDie(argc, argv, &outFilename, &outFilenameBufferLen); 
+
+    /* compress files with shared context, input and output buffers */
+    int argNb;
+    for (argNb = 1; argNb < argc; argNb++) {
+        const char* const inFilename = argv[argNb];
+        int inFilenameLen = strlen(inFilename);
+        assert(inFilenameLen + 5 <= outFilenameBufferLen);
+        memcpy(outFilename, inFilename, inFilenameLen);
+        memcpy(outFilename+inFilenameLen, ".zst", 5);
+        compressFile_orDie(ress, inFilename, outFilename);
+    }
+
+    /* free momery */
+    freeResources(ress,outFilename);
+
+    printf("compressed %i files \n", argc-1);
+
+    return 0;
+}

examples/simple_compression.c

@@ -19,7 +19,7 @@
 static void compress_orDie(const char* fname, const char* oname)
 {
     size_t fSize;
-    void* const fBuff = loadFile_orDie(fname, &fSize);
+    void* const fBuff = mallocAndLoadFile_orDie(fname, &fSize);
     size_t const cBuffSize = ZSTD_compressBound(fSize);
     void* const cBuff = malloc_orDie(cBuffSize);
 

examples/simple_decompression.c

@@ -20,7 +20,7 @@
 static void decompress(const char* fname)
 {
     size_t cSize;
-    void* const cBuff = loadFile_orDie(fname, &cSize);
+    void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize);
     unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
     if (rSize==ZSTD_CONTENTSIZE_ERROR) {
         fprintf(stderr, "%s : it was not compressed by zstd.\n", fname);

examples/utils.h

@@ -18,6 +18,7 @@
 #include <stdio.h>     // fprintf, perror, fopen, etc.
 #include <string.h>    // strlen, strcat, memset, strerror
 #include <errno.h>     // errno
+#include <assert.h>    // assert
 #include <sys/stat.h>  // stat
 
 /*
@@ -40,13 +41,26 @@ typedef enum {
  *
  * @return The size of a given file path.
  */
-static off_t fsize_orDie(const char *filename)
+static size_t fsize_orDie(const char *filename)
 {
     struct stat st;
-    if (stat(filename, &st) == 0) return st.st_size;
+    if (stat(filename, &st) != 0) {
         /* error */
         perror(filename);
         exit(ERROR_fsize);
+    }
+
+    off_t const fileSize = st.st_size;
+    size_t const size = (size_t)fileSize;
+    /* 1. fileSize should be non-negative,
+     * 2. if off_t -> size_t type conversion results in discrepancy,
+     *    the file size is too large for type size_t.
+     */
+    if ((fileSize < 0) || (fileSize != (off_t)size)) { 
+        fprintf(stderr, "%s : filesize too large \n", filename);
+        exit(ERROR_largeFile);
+    }
+    return size;
 }
 
 /*! fopen_orDie() :
@@ -128,31 +142,43 @@ static void* malloc_orDie(size_t size)
 }
 
 /*! loadFile_orDie() :
- * Read size bytes from a file.
+ * load file into buffer (memory).
  *
  * Note: This function will send an error to stderr and exit if it
  * cannot read data from the given file path.
  *
- * @return If successful this function will return a pointer to read
- * data otherwise it will printout an error to stderr and exit.
+ * @return If successful this function will load file into buffer and
+ * return file size, otherwise it will printout an error to stderr and exit.
  */
-static void* loadFile_orDie(const char* fileName, size_t* size)
+static size_t loadFile_orDie(const char* fileName, void* buffer, int bufferSize)
 {
-    off_t const fileSize = fsize_orDie(fileName);
-    size_t const buffSize = (size_t)fileSize;
-    if ((off_t)buffSize < fileSize) {   /* narrowcast overflow */
-        fprintf(stderr, "%s : filesize too large \n", fileName);
-        exit(ERROR_largeFile);
-    }
+    size_t const fileSize = fsize_orDie(fileName);
+    assert(fileSize <= bufferSize);
+
     FILE* const inFile = fopen_orDie(fileName, "rb");
-    void* const buffer = malloc_orDie(buffSize);
-    size_t const readSize = fread(buffer, 1, buffSize, inFile);
-    if (readSize != (size_t)buffSize) {
+    size_t const readSize = fread(buffer, 1, fileSize, inFile);
+    if (readSize != (size_t)fileSize) {
         fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
         exit(ERROR_fread);
     }
     fclose(inFile);  /* can't fail, read only */
-    *size = buffSize;
+    return fileSize;
+}
+
+/*! mallocAndLoadFile_orDie() :
+ * allocate memory buffer and then load file into it.
+ *
+ * Note: This function will send an error to stderr and exit if memory allocation
+ * fails or it cannot read data from the given file path.
+ *
+ * @return If successful this function will return buffer and bufferSize(=fileSize),
+ * otherwise it will printout an error to stderr and exit.
+ */
+static void* mallocAndLoadFile_orDie(const char* fileName, size_t* bufferSize) {
+    size_t const fileSize = fsize_orDie(fileName);
+    *bufferSize = fileSize;
+    void* const buffer = malloc_orDie(*bufferSize);
+    loadFile_orDie(fileName, buffer, *bufferSize);
     return buffer;
 }