From 4da53219a07c07c3a3e1456c5acc74fa9faa3f73 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 7 Dec 2016 11:18:40 +0100 Subject: [PATCH] zstd Manual updated to 1.1.2 --- contrib/gen_html/gen_html.cpp | 1 + doc/zstd_manual.html | 91 +++++++++++++++++++++++++---------- lib/zstd.h | 2 +- 3 files changed, 67 insertions(+), 27 deletions(-) diff --git a/contrib/gen_html/gen_html.cpp b/contrib/gen_html/gen_html.cpp index 987b7917..22ff65b1 100644 --- a/contrib/gen_html/gen_html.cpp +++ b/contrib/gen_html/gen_html.cpp @@ -176,6 +176,7 @@ int main(int argc, char *argv[]) { } sout << "


" << endl << endl; } else if (exclam == '=') { /* comments of type /*= and /**= mean: use a

header and show also all functions until first empty line */ + trim(comments[0], " "); sout << "

" << comments[0] << "

";
             for (l=1; l
 
 
-zstd 1.1.1 Manual
+zstd 1.1.2 Manual
 
 
-
zstd 1.1.1 Manual

+
zstd 1.1.2 Manual

 

 
Contents

 
    
@@ -48,7 +48,7 @@
 
 
    Version
    
 
-
    unsigned ZSTD_versionNumber (void);  /**< returns version number of ZSTD */
+
    unsigned ZSTD_versionNumber(void);   /**< library version number; to be used when checking dll version */
 

    
 
    Simple API
    
 
@@ -88,21 +88,29 @@
     note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. 
 

    
 
-
    Helper functions
    int         ZSTD_maxCLevel(void);               /*!< maximum compression level available */
+
    Helper functions
    int         ZSTD_maxCLevel(void);               /*!< maximum compression level available */
 size_t      ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case scenario */
 unsigned    ZSTD_isError(size_t code);          /*!< tells if a `size_t` function result is an error code */
 const char* ZSTD_getErrorName(size_t code);     /*!< provides readable string from an error code */
-

    
+

    
 
    Explicit memory management
    
 
+
    Compression context
       When compressing many times,
+   it is recommended to allocate a context just once, and re-use it for each successive compression operation.
+   This will make workload friendlier for system's memory.
+   Use one context per thread for parallel execution in multi-threaded environments. 
+
    typedef struct ZSTD_CCtx_s ZSTD_CCtx;
+ZSTD_CCtx* ZSTD_createCCtx(void);
+size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
+

    
 
    size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
 
        Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). 
 

    
 
-
    Decompression context
    typedef struct ZSTD_DCtx_s ZSTD_DCtx;
+
    Decompression context
    typedef struct ZSTD_DCtx_s ZSTD_DCtx;
 ZSTD_DCtx* ZSTD_createDCtx(void);
 size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
-

    
+

    
 
    size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 
       Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()). 
 

    
@@ -220,14 +228,14 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
  
 

 
-
Streaming compression functions
typedef struct ZSTD_CStream_s ZSTD_CStream;
+
Streaming compression functions
typedef struct ZSTD_CStream_s ZSTD_CStream;
 ZSTD_CStream* ZSTD_createCStream(void);
 size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
 size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
 size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
 size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
 size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
-


+


 
size_t ZSTD_CStreamInSize(void);    /**< recommended size for input buffer */
 


 
size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
@@ -248,17 +256,17 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
   If `output.pos < output.size`, decoder has flushed everything it could.
   @return : 0 when a frame is completely decoded and fully flushed,
             an error code, which can be tested using ZSTD_isError(),
-            any other value > 0, which means there is still some work to do to complete the frame.
-            The return value is a suggested next input size (just an hint, to help latency).
+            any other value > 0, which means there is still some decoding to do to complete current frame.
+            The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame.
  
 

 
-
Streaming decompression functions
typedef struct ZSTD_DStream_s ZSTD_DStream;
+
Streaming decompression functions
typedef struct ZSTD_DStream_s ZSTD_DStream;
 ZSTD_DStream* ZSTD_createDStream(void);
 size_t ZSTD_freeDStream(ZSTD_DStream* zds);
 size_t ZSTD_initDStream(ZSTD_DStream* zds);
 size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
-


+


 
size_t ZSTD_DStreamInSize(void);    /*!< recommended size for input buffer */
 


 
size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
@@ -295,10 +303,10 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
     ZSTD_frameParameters fParams;
 } ZSTD_parameters;
 


-
Custom memory allocation functions
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
+
Custom memory allocation functions
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
 typedef void  (*ZSTD_freeFunction) (void* opaque, void* address);
 typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
-


+


 
Advanced compression functions

 
 
size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
@@ -352,6 +360,13 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 
 
Advanced decompression functions

 
+
unsigned ZSTD_isFrame(const void* buffer, size_t size);
+
  Tells if the content of `buffer` starts with a valid Frame Identifier.
+  Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0.
+  Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled.
+  Note 3 : Skippable Frame Identifiers are considered valid. 
+


+
 
size_t ZSTD_estimateDCtxSize(void);
 
  Gives the potential amount of memory allocated to create a ZSTD_DCtx 
 


@@ -368,24 +383,48 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 
  Gives the amount of memory used by a given ZSTD_DDict 
 


 
+
unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
+
  Provides the dictID stored within dictionary.
+  if @return == 0, the dictionary is not conformant with Zstandard specification.
+  It can still be loaded, but as a content-only dictionary. 
+


+
+
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
+
  Provides the dictID of the dictionary loaded into `ddict`.
+  If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+  Non-conformant dictionaries can still be loaded, but as content-only dictionaries. 
+


+
+
unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
+
  Provides the dictID required to decompressed the frame stored within `src`.
+  If @return == 0, the dictID could not be decoded.
+  This could for one of the following reasons :
+  - The frame does not require a dictionary to be decoded (most common case).
+  - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
+    Note : this use case also happens when using a non-conformant dictionary.
+  - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
+  - This is not a Zstandard frame.
+  When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. 
+


+
 
Advanced streaming functions

 
-
Advanced Streaming compression functions
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+
Advanced Streaming compression functions
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
                                              ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be zero == unknown */
 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  /**< note : cdict will just be referenced, and must outlive compression session */
 size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);  /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */
 size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
-


-
Advanced Streaming decompression functions
typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
+


+
Advanced Streaming decompression functions
typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
 ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
 size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
 size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
 size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);  /**< note : ddict will just be referenced, and must outlive decompression session */
 size_t ZSTD_resetDStream(ZSTD_DStream* zds);  /**< re-use decompression parameters from previous init; saves dictionary loading */
 size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
-


+


 
Buffer-less and synchronous inner streaming functions
   This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
   But it's also a complex one, with many restrictions (documented below).
@@ -422,13 +461,13 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
   You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame.
 

 
-
Buffer-less streaming compression functions
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
+
Buffer-less streaming compression functions
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
 size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
 size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
 size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
 size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-


+


 
Buffer-less streaming decompression (synchronous mode)
   A ZSTD_DCtx object is required to track streaming operations.
   Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
@@ -472,7 +511,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const vo
   Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
   This information is not required to properly decode a frame.
 
-  == Special case : skippable frames ==
+  == Special case : skippable frames == 
 
   Skippable frames allow integration of user-defined data into a flow of concatenated frames.
   Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows :
@@ -491,7 +530,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const vo
     unsigned checksumFlag;
 } ZSTD_frameParams;
 


-
Buffer-less streaming decompression functions
size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize);   /**< doesn't consume input, see details below */
+
Buffer-less streaming decompression functions
size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize);   /**< doesn't consume input, see details below */
 size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
 size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
 void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
@@ -499,7 +538,7 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
 size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
 ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
-


+


 
Block functions
     Block functions produce and decode raw zstd blocks, without frame metadata.
     Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
@@ -524,10 +563,10 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
         Use ZSTD_insertBlock() in such a case.
 

 
-
Raw zstd block functions
size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
+
Raw zstd block functions
size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
 size_t ZSTD_compressBlock  (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize);  /**< insert block into `dctx` history. Useful for uncompressed blocks */
-


+


 
 
diff --git a/lib/zstd.h b/lib/zstd.h
index 3989cc30..66ce1360 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -604,7 +604,7 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
   Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
   This information is not required to properly decode a frame.
 
-  == Special case : skippable frames ==
+  == Special case : skippable frames == 
 
   Skippable frames allow integration of user-defined data into a flow of concatenated frames.
   Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows :