AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

implement API-level changes

Browse Source
This commit is contained in:
shakeelrao 2019-02-28 22:55:18 -08:00
parent dce9a09772
commit 8930c3c79b
4 changed files with 103 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
doc/zstd_manual.html
View File

@ -127,11 +127,10 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
</p></pre><BR>
<h3>Helper functions</h3><pre></pre><b><pre>#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) </b>/* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */<b>
size_t ZSTD_compressBound(size_t srcSize); </b>/*!< maximum compressed size in worst case single-pass scenario */<b>
unsigned long long ZSTD_decompressBound(const void* src, size_t srcSice); </b>/*!< maximum decompressed size of the compressed source */<b>
unsigned ZSTD_isError(size_t code); </b>/*!< tells if a `size_t` function result is an error code */<b>
const char* ZSTD_getErrorName(size_t code); </b>/*!< provides readable string from an error code */<b>
int ZSTD_maxCLevel(void); </b>/*!< maximum compression level available */<b>
size_t ZSTD_compressBound(size_t srcSize); </b>/*!< maximum compressed size in worst case single-pass scenario */<b>
unsigned ZSTD_isError(size_t code); </b>/*!< tells if a `size_t` function result is an error code */<b>
const char* ZSTD_getErrorName(size_t code); </b>/*!< provides readable string from an error code */<b>
int ZSTD_maxCLevel(void); </b>/*!< maximum compression level available */<b>
</pre></b><BR>
<a name="Chapter5"></a><h2>Explicit context</h2><pre></pre>

144
lib/decompress/zstd_decompress.c
View File

@ -433,40 +433,73 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t he
return 0;
}
/**
* Contains the compressed frame size and an upper-bound for the decompressed frame size.
* Note: before using `compressedSize` or `decompressedBound`,
* you must check the field for errors using ZSTD_isError().
*/
typedef struct {
size_t compressedSize;
size_t decompressedBound;
} ZSTD_frameSizeInfo;
static size_t ZSTD_findFrameCompressedSize_internal(const void *src, size_t srcSize, size_t* bound)
static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize)
{
ZSTD_frameSizeInfo frameSizeInfo;
memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo));
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
if (ZSTD_isLegacy(src, srcSize))
return ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
if (ZSTD_isLegacy(src, srcSize)) {
frameSizeInfo.compressedSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
frameSizeInfo.decompressedBound = ERROR(version_unsupported);
return frameSizeInfo;
}
#endif
if ( (srcSize >= ZSTD_SKIPPABLEHEADERSIZE)
&& (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START ) {
return readSkippableFrameSize(src, srcSize);
if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE)
&& (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize);
return frameSizeInfo;
} else {
const BYTE* ip = (const BYTE*)src;
const BYTE* const ipstart = ip;
size_t remainingSize = srcSize;
ZSTD_frameHeader zfh;
unsigned nbBlocks = 0;
ZSTD_frameHeader zfh;
/* Extract Frame Header */
{ size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize);
if (ZSTD_isError(ret)) return ret;
RETURN_ERROR_IF(ret > 0, srcSize_wrong);
{
size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize);
if (ZSTD_isError(ret)) {
frameSizeInfo.compressedSize = ret;
frameSizeInfo.decompressedBound = ret;
return frameSizeInfo;
}
if (ret > 0) {
frameSizeInfo.compressedSize = ERROR(srcSize_wrong);
frameSizeInfo.decompressedBound = ERROR(srcSize_wrong);
return frameSizeInfo;
}
}
ip += zfh.headerSize;
remainingSize -= zfh.headerSize;
/* Loop on each block */
/* Iterate over each block */
while (1) {
blockProperties_t blockProperties;
size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
if (ZSTD_isError(cBlockSize)) return cBlockSize;
if (ZSTD_isError(cBlockSize)) {
frameSizeInfo.compressedSize = cBlockSize;
frameSizeInfo.decompressedBound = cBlockSize;
return frameSizeInfo;
}
RETURN_ERROR_IF(ZSTD_blockHeaderSize + cBlockSize > remainingSize,
srcSize_wrong);
if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) {
frameSizeInfo.compressedSize = ERROR(srcSize_wrong);
frameSizeInfo.decompressedBound = ERROR(srcSize_wrong);
return frameSizeInfo;
}
ip += ZSTD_blockHeaderSize + cBlockSize;
remainingSize -= ZSTD_blockHeaderSize + cBlockSize;
@ -475,14 +508,21 @@ static size_t ZSTD_findFrameCompressedSize_internal(const void *src, size_t srcS
if (blockProperties.lastBlock) break;
}
if (zfh.checksumFlag) { /* Final frame content checksum */
RETURN_ERROR_IF(remainingSize < 4, srcSize_wrong);
/* Final frame content checksum */
if (zfh.checksumFlag) {
if (remainingSize < 4) {
frameSizeInfo.compressedSize = ERROR(srcSize_wrong);
frameSizeInfo.decompressedBound = ERROR(srcSize_wrong);
return frameSizeInfo;
}
ip += 4;
}
if (bound != NULL) *bound = (nbBlocks * zfh.blockSizeMax); /* set to block-based bound */
return ip - ipstart;
frameSizeInfo.compressedSize = ip - ipstart;
frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN)
? zfh.frameContentSize
: nbBlocks * zfh.blockSizeMax;
return frameSizeInfo;
}
}
@ -493,7 +533,8 @@ static size_t ZSTD_findFrameCompressedSize_internal(const void *src, size_t srcS
* @return : the compressed size of the frame starting at `src` */
size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
{
return ZSTD_findFrameCompressedSize_internal(src, srcSize, NULL);
ZSTD_frameSizeInfo frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
return frameSizeInfo.compressedSize;
}
@ -502,56 +543,23 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
* `src` must point to the start of a ZSTD frame or a skippeable frame
* `srcSize` must be at least as large as the frame contained
* @return : maximum decompressed size of the compressed source
* or an error code which can be tested with ZSTD_isError()
*/
unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
size_t ZSTD_decompressBound(const void* src, size_t srcSize)
{
unsigned long long totalDstSize = 0;
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
if (ZSTD_isLegacy(src, srcSize))
return ERROR(version_unsupported);
#endif
/* Loop over each frame */
while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) {
U32 const magicNumber = MEM_readLE32(src);
if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
size_t const skippableSize = readSkippableFrameSize(src, srcSize);
if (ZSTD_isError(skippableSize))
return skippableSize;
if (srcSize < skippableSize) {
return ZSTD_CONTENTSIZE_ERROR;
}
src = (const BYTE *)src + skippableSize;
srcSize -= skippableSize;
continue;
}
{ size_t bound = 0;
unsigned long long frameBound = 0;
size_t frameSrcSize = 0;
unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
if (ret == ZSTD_CONTENTSIZE_ERROR) return ret;
frameSrcSize = ZSTD_findFrameCompressedSize_internal(src, srcSize, &bound);
if (ZSTD_isError(frameSrcSize)) {
return ZSTD_CONTENTSIZE_ERROR;
}
frameBound = (ret == ZSTD_CONTENTSIZE_UNKNOWN) ? bound : ret;
/* check for overflow */
if (totalDstSize + frameBound < totalDstSize) return ZSTD_CONTENTSIZE_ERROR;
totalDstSize += frameBound;
src = (const BYTE *)src + frameSrcSize;
srcSize -= frameSrcSize;
}
} /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */
if (srcSize) return ZSTD_CONTENTSIZE_ERROR;
return totalDstSize;
size_t bound = 0;
/* Iterate over each frame */
while (srcSize > 0) {
ZSTD_frameSizeInfo frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
size_t compressedSize = frameSizeInfo.compressedSize;
size_t decompressedBound = frameSizeInfo.decompressedBound;
FORWARD_IF_ERROR(compressedSize);
FORWARD_IF_ERROR(decompressedBound);
src = (const BYTE*)src + compressedSize;
srcSize -= compressedSize;
bound += decompressedBound;
}
return bound;
}

24
lib/zstd.h
View File

@ -148,11 +148,10 @@ ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t
/*====== Helper functions ======*/
#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSice); /*!< maximum decompressed size of the compressed source */
ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */
ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */
ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */
ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */
/***************************************
@ -1104,6 +1103,21 @@ typedef enum {
* however it does mean that all frame data must be present and valid. */
ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
/** ZSTD_decompressBound() :
* currently incompatible with legacy mode
* `src` must point to the start of a ZSTD frame or a skippeable frame
* `srcSize` must be at least as large as the frame contained
* @return : maximum decompressed size of the compressed source
* or an error code which can be tested with ZSTD_isError()
*
* note 1 : the bound is exact when Frame_Content_Size field is available in EVERY frame of `src`.
* note 2 : when Frame_Content_Size isn't provided, the upper-bound for that frame is calculated by:
* upper-bound = min(128 KB, Window_Size)
* note 3 : we always use Frame_Content_Size to bound the decompressed frame size if it's present.
** the above formula is only used when Frame_Content_Size is missing.
*/
ZSTDLIB_API size_t ZSTD_decompressBound(const void* src, size_t srcSice);
/*! ZSTD_frameHeaderSize() :
* srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX.
* @return : size of the Frame Header,

8
tests/fuzzer.c
View File

@ -378,8 +378,8 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : tight ZSTD_decompressBound test : ", testNb++);
{
unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize);
if (bound != CNBuffSize) goto _output_error;
size_t rSize = ZSTD_decompressBound(compressedBuffer, cSize);
if (rSize != CNBuffSize) goto _output_error;
}
DISPLAYLEVEL(3, "OK \n");
@ -909,8 +909,8 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : get tight decompressed bound of multiple frames : ", testNb++);
{ unsigned long long const r = ZSTD_decompressBound(compressedBuffer, cSize);
if (r != CNBuffSize / 2) goto _output_error; }
{ size_t const rSize = ZSTD_decompressBound(compressedBuffer, cSize);
if (rSize != CNBuffSize / 2) goto _output_error; }
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : decompress multiple frames : ", testNb++);