From cdf7e82222758cecb01438f09cf1f7b08bf1688d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 25 May 2017 18:05:49 -0700 Subject: [PATCH] Added ZSTD_initStaticCDict() --- doc/zstd_manual.html | 22 +++++- lib/compress/zstd_compress.c | 111 ++++++++++++++++++++++--------- lib/decompress/zstd_decompress.c | 3 +- lib/zstd.h | 23 ++++++- tests/fuzzer.c | 23 +++++++ 5 files changed, 145 insertions(+), 37 deletions(-) diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index 5040dd7d..a4b59bc4 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -408,8 +408,8 @@ size_t ZSTD_estimateDCtxSize(void); 
size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
 size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
 
  Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
-         an internal ?Dict will be created, which size is not estimated.
-         In this case, get additional size by using ZSTD_estimate?DictSize 
+         an internal ?Dict will be created, which size is not estimated here.
+         In this case, get total size by adding ZSTD_estimate?DictSize 
 

size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference);
@@ -460,6 +460,24 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, unsigned byReference);
 
  Create a ZSTD_CDict using external alloc and free, and customized compression parameters

+
ZSTD_CDict* ZSTD_initStaticCDict(
+                void* workspace, size_t workspaceSize,
+          const void* dict, size_t dictSize, unsigned byReference,
+                ZSTD_compressionParameters cParams);
+
  Generate a digested dictionary in provided memory area.
+  workspace: The memory area to emplace the dictionary into.
+             Provided pointer must 8-bytes aligned.
+             It must outlive dictionary usage.
+  workspaceSize: Use ZSTD_estimateCDictSize()
+                 to determine how large workspace must be.
+  cParams : use ZSTD_getCParams() to transform a compression level
+            into its relevants cParams.
+ @return : pointer to ZSTD_CDict*, or NULL if error (size too small)
+  Note : there is no corresponding "free" function.
+         Since workspace was allocated externally, it must be freed externally.
+ 
+

+ 
ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
 
   @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
    `estimatedSrcSize` value is optional, select 0 if not known 
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index aa9f64c5..0e93abcc 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3239,7 +3239,6 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
  *  Estimate amount of memory that will be needed to create a dictionary with following arguments */
 size_t ZSTD_estimateCDictSize(ZSTD_compressionParameters cParams, size_t dictSize, unsigned byReference)
 {
-    cParams = ZSTD_adjustCParams(cParams, 0, dictSize);
     return sizeof(ZSTD_CDict) + ZSTD_estimateCCtxSize(cParams)
            + (byReference ? 0 : dictSize);
 }
@@ -3258,6 +3257,34 @@ static ZSTD_parameters ZSTD_makeParams(ZSTD_compressionParameters cParams, ZSTD_
     return params;
 }
 
+static size_t ZSTD_initCDict_internal(
+                    ZSTD_CDict* cdict,
+              const void* dictBuffer, size_t dictSize, unsigned byReference,
+                    ZSTD_compressionParameters cParams)
+{
+    if ((byReference) || (!dictBuffer) || (!dictSize)) {
+        cdict->dictBuffer = NULL;
+        cdict->dictContent = dictBuffer;
+    } else {
+        void* const internalBuffer = ZSTD_malloc(dictSize, cdict->refContext->customMem);
+        if (!internalBuffer) return ERROR(memory_allocation);
+        memcpy(internalBuffer, dictBuffer, dictSize);
+        cdict->dictBuffer = internalBuffer;
+        cdict->dictContent = internalBuffer;
+    }
+    cdict->dictContentSize = dictSize;
+
+    {   ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */,
+                    0 /* checksumFlag */, 0 /* noDictIDFlag */ };  /* dummy */
+        ZSTD_parameters const params = ZSTD_makeParams(cParams, fParams);
+        CHECK_F( ZSTD_compressBegin_advanced(cdict->refContext,
+                                            cdict->dictContent, dictSize,
+                                            params, 0 /* srcSize */) );
+    }
+
+    return 0;
+}
+
 ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, unsigned byReference,
                                       ZSTD_compressionParameters cParams, ZSTD_customMem customMem)
 {
@@ -3269,44 +3296,19 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, u
         ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem);
 
         if (!cdict || !cctx) {
-            DEBUGLOG(5, "!cdict || !cctx");
             ZSTD_free(cdict, customMem);
             ZSTD_freeCCtx(cctx);
             return NULL;
         }
+        cdict->refContext = cctx;
 
-        if ((byReference) || (!dictBuffer) || (!dictSize)) {
-            cdict->dictBuffer = NULL;
-            cdict->dictContent = dictBuffer;
-        } else {
-            void* const internalBuffer = ZSTD_malloc(dictSize, customMem);
-            if (!internalBuffer) {
-                DEBUGLOG(5, "!internalBuffer");
-                ZSTD_free(cctx, customMem);
-                ZSTD_free(cdict, customMem);
-                return NULL;
-            }
-            memcpy(internalBuffer, dictBuffer, dictSize);
-            cdict->dictBuffer = internalBuffer;
-            cdict->dictContent = internalBuffer;
+        if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
+                                        dictBuffer, dictSize, byReference,
+                                        cParams) )) {
+            ZSTD_freeCDict(cdict);
+            return NULL;
         }
 
-        {   ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */,
-                    0 /* checksumFlag */, 0 /* noDictIDFlag */ }; /* dummy */
-            ZSTD_parameters const params = ZSTD_makeParams(cParams, fParams);
-            size_t const errorCode = ZSTD_compressBegin_advanced(cctx,
-                                cdict->dictContent, dictSize, params, 0);
-            if (ZSTD_isError(errorCode)) {
-                DEBUGLOG(5, "ZSTD_compressBegin_advanced error : %s",
-                            ZSTD_getErrorName(errorCode));
-                ZSTD_free(cdict->dictBuffer, customMem);
-                ZSTD_free(cdict, customMem);
-                ZSTD_freeCCtx(cctx);
-                return NULL;
-        }   }
-
-        cdict->refContext = cctx;
-        cdict->dictContentSize = dictSize;
         return cdict;
     }
 }
@@ -3336,6 +3338,51 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
     }
 }
 
+/*! ZSTD_initStaticCDict_advanced() :
+ *  Generate a digested dictionary in provided memory area.
+ *  workspace: The memory area to emplace the dictionary into.
+ *             Provided pointer must 8-bytes aligned.
+ *             It must outlive dictionary usage.
+ *  workspaceSize: Use ZSTD_estimateCDictSize()
+ *                 to determine how large workspace must be.
+ *  cParams : use ZSTD_getCParams() to transform a compression level
+ *            into its relevants cParams.
+ * @return : pointer to ZSTD_CDict*, or NULL if error (size too small)
+ *  Note : there is no corresponding "free" function.
+ *         Since workspace was allocated externally, it must be freed externally.
+ */
+ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize,
+                           const void* dict, size_t dictSize, unsigned byReference,
+                                 ZSTD_compressionParameters cParams)
+{
+    size_t const cctxSize = ZSTD_estimateCCtxSize(cParams);
+    size_t const neededSize = sizeof(ZSTD_CDict) + (byReference ? 0 : dictSize)
+                            + cctxSize;
+    ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace;
+    void* ptr;
+    DEBUGLOG(2, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7);
+    if ((size_t)workspace & 7) return NULL;  /* 8-aligned */
+    DEBUGLOG(2, "(workspaceSize < neededSize) : (%u < %u) => %u",
+        (U32)workspaceSize, (U32)neededSize, (U32)(workspaceSize < neededSize));
+    if (workspaceSize < neededSize) return NULL;
+
+    if (!byReference) {
+        memcpy(cdict+1, dict, dictSize);
+        dict = cdict+1;
+        ptr = (char*)workspace + sizeof(ZSTD_CDict) + dictSize;
+    } else {
+        ptr = cdict+1;
+    }
+    cdict->refContext = ZSTD_initStaticCCtx(ptr, cctxSize);
+
+    if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
+                                        dict, dictSize, 1 /* by Reference */,
+                                        cParams) ))
+        return NULL;
+
+    return cdict;
+}
+
 static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) {
     return ZSTD_getParamsFromCCtx(cdict->refContext);
 }
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index e3721929..fe7446ab 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -2053,7 +2053,8 @@ ZSTD_DDict* ZSTD_initStaticDDict(void* workspace, size_t workspaceSize,
 {
     size_t const neededSpace = sizeof(ZSTD_DDict) + (byReference ? 0 : dictSize);
     ZSTD_DDict* const ddict = (ZSTD_DDict*)workspace;
-    if (workspaceSize < neededSpace) return NULL;  /* minimum size */
+    if ((size_t)workspace & 7) return NULL;  /* 8-aligned */
+    if (workspaceSize < neededSpace) return NULL;
     if (!byReference) {
         memcpy(ddict+1, dict, dictSize);  /* local copy */
         dict = ddict+1;
diff --git a/lib/zstd.h b/lib/zstd.h
index 82501994..b1d8ecad 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -495,8 +495,8 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
 
 /*! ZSTD_estimate?StreamSize() :
  *  Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
- *         an internal ?Dict will be created, which size is not estimated.
- *         In this case, get additional size by using ZSTD_estimate?DictSize */
+ *         an internal ?Dict will be created, which size is not estimated here.
+ *         In this case, get total size by adding ZSTD_estimate?DictSize */
 ZSTDLIB_API size_t ZSTD_estimateCStreamSize(ZSTD_compressionParameters cParams);
 ZSTDLIB_API size_t ZSTD_estimateDStreamSize(ZSTD_frameHeader fHeader);
 
@@ -540,6 +540,7 @@ typedef enum {
  *  @result : 0, or an error code (which can be tested with ZSTD_isError()) */
 ZSTDLIB_API size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value);
 
+
 /*! ZSTD_createCDict_byReference() :
  *  Create a digested dictionary for compression
  *  Dictionary content is simply referenced, and therefore stays in dictBuffer.
@@ -551,6 +552,24 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, siz
 ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
                                                   ZSTD_compressionParameters cParams, ZSTD_customMem customMem);
 
+/*! ZSTD_initStaticCDict_advanced() :
+ *  Generate a digested dictionary in provided memory area.
+ *  workspace: The memory area to emplace the dictionary into.
+ *             Provided pointer must 8-bytes aligned.
+ *             It must outlive dictionary usage.
+ *  workspaceSize: Use ZSTD_estimateCDictSize()
+ *                 to determine how large workspace must be.
+ *  cParams : use ZSTD_getCParams() to transform a compression level
+ *            into its relevants cParams.
+ * @return : pointer to ZSTD_CDict*, or NULL if error (size too small)
+ *  Note : there is no corresponding "free" function.
+ *         Since workspace was allocated externally, it must be freed externally.
+ */
+ZSTDLIB_API ZSTD_CDict* ZSTD_initStaticCDict(
+                            void* workspace, size_t workspaceSize,
+                      const void* dict, size_t dictSize, unsigned byReference,
+                            ZSTD_compressionParameters cParams);
+
 /*! ZSTD_getCParams() :
 *   @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
 *   `estimatedSrcSize` value is optional, select 0 if not known */
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 90cfe648..ee0aff08 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -531,6 +531,29 @@ static int basicUnitTests(U32 seed, double compressibility)
                   if (r != CNBuffSize) goto _output_error);
         DISPLAYLEVEL(4, "OK \n");
 
+        DISPLAYLEVEL(4, "test%3i : compress with static CDict : ", testNb++);
+        {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
+            size_t const cdictSize = ZSTD_estimateCDictSize(cParams, dictSize, 0);
+            void* const cdictBuffer = malloc(cdictSize);
+            if (cdictBuffer==NULL) goto _output_error;
+            {   ZSTD_CDict* const cdict = ZSTD_initStaticCDict(cdictBuffer, cdictSize,
+                                            dictBuffer, dictSize, 0 /* by Reference */,
+                                            cParams);
+                if (cdict == NULL) {
+                    DISPLAY("ZSTD_initStaticCDict failed ");
+                    goto _output_error;
+                }
+                cSize = ZSTD_compress_usingCDict(cctx,
+                                compressedBuffer, ZSTD_compressBound(CNBuffSize),
+                                CNBuffer, CNBuffSize, cdict);
+                if (ZSTD_isError(cSize)) {
+                    DISPLAY("ZSTD_compress_usingCDict failed ");
+                    goto _output_error;
+            }   }
+            free(cdictBuffer);
+        }
+        DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
+
         DISPLAYLEVEL(4, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++);
         {   ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ };
             ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);