AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Decompressed size functions now handle multiframes and distinguish cases

Browse Source 
- Add ZSTD_findDecompressedSize
    - Traverses multiple frames to find total output size
- Add ZSTD_getFrameContentSize
    - Gets the decompressed size of a single frame by reading header
- Deprecate ZSTD_getDecompressedSize
This commit is contained in:
Sean Purcell 2017-02-07 13:50:09 -08:00
parent 60259eb9a0
commit 4e709712e1
24 changed files with 496 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/dictionary_decompression.c
View File

@ -76,7 +76,7 @@ static void decompress(const char* fname, const ZSTD_DDict* ddict)
{
size_t cSize;
void* const cBuff = loadFile_orDie(fname, &cSize);
unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize);
unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
if (rSize==0) {
fprintf(stderr, "%s : original size unknown \n", fname);
exit(6);

2
examples/simple_decompression.c
View File

@ -63,7 +63,7 @@ static void decompress(const char* fname)
{
size_t cSize;
void* const cBuff = loadFile_X(fname, &cSize);
unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize);
unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
if (rSize==0) {
printf("%s : original size unknown. Use streaming decompression instead. \n", fname);
exit(5);

146
lib/decompress/zstd_decompress.c
View File

@ -306,6 +306,100 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
return 0;
}
static size_t ZSTD_frameSrcSize(const void* src, size_t srcSize);
/** ZSTD_getFrameContentSize() :
* compatible with legacy mode
* @return : decompressed size of the single frame pointed to be `src` if known, otherwise
* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
* - ZSTD_CONTENTSIZE_ERROR if an error occured (e.g. invalid magic number, srcSize too small) */
unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
{
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
if (ZSTD_isLegacy(src, srcSize)) {
unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize);
return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret;
}
#endif
{
ZSTD_frameParams fParams;
if (ZSTD_getFrameParams(&fParams, src, srcSize) != 0) return ZSTD_CONTENTSIZE_ERROR;
if (fParams.windowSize == 0) {
/* Either skippable or empty frame, size == 0 either way */
return 0;
} else if (fParams.frameContentSize != 0) {
return fParams.frameContentSize;
} else {
return ZSTD_CONTENTSIZE_UNKNOWN;
}
}
}
/** ZSTD_findDecompressedSize() :
* compatible with legacy mode
* `srcSize` must be the exact length of some number of ZSTD compressed and/or
* skippable frames
* @return : decompressed size of the frames contained */
unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
{
{
unsigned long long totalDstSize = 0;
while (srcSize >= ZSTD_frameHeaderSize_prefix) {
const U32 magicNumber = MEM_readLE32(src);
if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
size_t skippableSize;
if (srcSize < ZSTD_skippableHeaderSize)
return ERROR(srcSize_wrong);
skippableSize = MEM_readLE32((const BYTE *)src + 4) +
ZSTD_skippableHeaderSize;
if (srcSize < skippableSize) {
/* srcSize_wrong */
return 0;
}
src = (const BYTE *)src + skippableSize;
srcSize -= skippableSize;
continue;
}
{
unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret;
/* check for overflow */
if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR;
totalDstSize += ret;
}
{
size_t frameSrcSize;
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
if (ZSTD_isLegacy(src, srcSize))
{
frameSrcSize = ZSTD_frameSrcSizeLegacy(src, srcSize);
}
else
#endif
{
frameSrcSize = ZSTD_frameSrcSize(src, srcSize);
}
if (ZSTD_isError(frameSrcSize)) {
return 0;
}
src = (const BYTE *)src + frameSrcSize;
srcSize -= frameSrcSize;
}
}
if (srcSize) {
/* srcSize_wrong */
return 0;
}
return totalDstSize;
}
}
/** ZSTD_getDecompressedSize() :
* compatible with legacy mode
@ -316,14 +410,8 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
- frame header not complete (`srcSize` too small) */
unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)
{
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
if (ZSTD_isLegacy(src, srcSize)) return ZSTD_getDecompressedSize_legacy(src, srcSize);
#endif
{ ZSTD_frameParams fparams;
size_t const frResult = ZSTD_getFrameParams(&fparams, src, srcSize);
if (frResult!=0) return 0;
return fparams.frameContentSize;
}
unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
return ret >= ZSTD_CONTENTSIZE_ERROR ? 0 : ret;
}
@ -1363,6 +1451,48 @@ size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t len
return length;
}
static size_t ZSTD_frameSrcSize(const void *src, size_t srcSize)
{
const BYTE* ip = (const BYTE*)src;
const BYTE* const ipstart = ip;
size_t remainingSize = srcSize;
ZSTD_frameParams fParams;
size_t const headerSize = ZSTD_frameHeaderSize(ip, remainingSize);
if (ZSTD_isError(headerSize)) return headerSize;
/* Frame Header */
{
size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize);
if (ZSTD_isError(ret)) return ret;
if (ret > 0) return ERROR(srcSize_wrong);
}
ip += headerSize;
remainingSize -= headerSize;
/* Loop on each block */
while (1) {
blockProperties_t blockProperties;
size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
if (ZSTD_isError(cBlockSize)) return cBlockSize;
if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) return ERROR(srcSize_wrong);
ip += ZSTD_blockHeaderSize + cBlockSize;
remainingSize -= ZSTD_blockHeaderSize + cBlockSize;
if (blockProperties.lastBlock) break;
}
if (fParams.checksumFlag) { /* Frame content checksum verification */
if (remainingSize < 4) return ERROR(srcSize_wrong);
ip += 4;
remainingSize -= 4;
}
return ip - ipstart;
}
/*! ZSTD_decompressFrame() :
* `dctx` must be properly initialized */

2
lib/dll/libzstd.def
View File

@ -58,6 +58,8 @@ EXPORTS
ZSTD_getBlockSizeMax
ZSTD_getCParams
ZSTD_getDecompressedSize
ZSTD_findDecompressedSize
ZSTD_getFrameContentSize
ZSTD_getErrorName
ZSTD_getFrameParams
ZSTD_getParams

24
lib/legacy/zstd_legacy.h
View File

@ -123,6 +123,30 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
}
}
MEM_STATIC size_t ZSTD_frameSrcSizeLegacy(const void *src,
size_t compressedSize)
{
U32 const version = ZSTD_isLegacy(src, compressedSize);
switch(version)
{
case 1 :
return ZSTDv01_frameSrcSize(src, compressedSize);
case 2 :
return ZSTDv02_frameSrcSize(src, compressedSize);
case 3 :
return ZSTDv03_frameSrcSize(src, compressedSize);
case 4 :
return ZSTDv04_frameSrcSize(src, compressedSize);
case 5 :
return ZSTDv05_frameSrcSize(src, compressedSize);
case 6 :
return ZSTDv06_frameSrcSize(src, compressedSize);
case 7 :
return ZSTDv07_frameSrcSize(src, compressedSize);
default :
return ERROR(prefix_unknown);
}
}
MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version)
{

31
lib/legacy/zstd_v01.c
View File

@ -1992,6 +1992,37 @@ size_t ZSTDv01_decompress(void* dst, size_t maxDstSize, const void* src, size_t
return ZSTDv01_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
}
size_t ZSTDv01_frameSrcSize(const void* src, size_t srcSize)
{
const BYTE* ip = (const BYTE*)src;
size_t remainingSize = srcSize;
U32 magicNumber;
blockProperties_t blockProperties;
/* Frame Header */
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
magicNumber = ZSTD_readBE32(src);
if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
/* Loop on each block */
while (1)
{
size_t blockSize = ZSTDv01_getcBlockSize(ip, remainingSize, &blockProperties);
if (ZSTDv01_isError(blockSize)) return blockSize;
ip += ZSTD_blockHeaderSize;
remainingSize -= ZSTD_blockHeaderSize;
if (blockSize > remainingSize) return ERROR(srcSize_wrong);
if (blockSize == 0) break; /* bt_end */
ip += blockSize;
remainingSize -= blockSize;
}
return ip - (const BYTE*)src;
}
/*******************************
* Streaming Decompression API

8
lib/legacy/zstd_v01.h
View File

@ -34,6 +34,14 @@ ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format
size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize,
const void* src, size_t compressedSize);
/**
ZSTDv01_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.1.x format
compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
return : the number of bytes that would be read to decompress this frame
or an errorCode if it fails (which can be tested using ZSTDv01_isError())
*/
size_t ZSTDv01_frameSrcSize(const void* src, size_t compressedSize);
/**
ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error
*/

37
lib/legacy/zstd_v02.c
View File

@ -3378,6 +3378,38 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
}
static size_t ZSTD_frameSrcSize(const void *src, size_t srcSize)
{
const BYTE* ip = (const BYTE*)src;
size_t remainingSize = srcSize;
U32 magicNumber;
blockProperties_t blockProperties;
/* Frame Header */
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
magicNumber = MEM_readLE32(src);
if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
/* Loop on each block */
while (1)
{
size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
if (ZSTD_isError(cBlockSize)) return cBlockSize;
ip += ZSTD_blockHeaderSize;
remainingSize -= ZSTD_blockHeaderSize;
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
if (cBlockSize == 0) break; /* bt_end */
ip += cBlockSize;
remainingSize -= cBlockSize;
}
return ip - (const BYTE*)src;
}
/*******************************
* Streaming Decompression API
@ -3492,6 +3524,11 @@ size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
}
size_t ZSTDv02_frameSrcSize(const void *src, size_t compressedSize)
{
return ZSTD_frameSrcSize(src, compressedSize);
}
ZSTDv02_Dctx* ZSTDv02_createDCtx(void)
{
return (ZSTDv02_Dctx*)ZSTD_createDCtx();

8
lib/legacy/zstd_v02.h
View File

@ -34,6 +34,14 @@ ZSTDv02_decompress() : decompress ZSTD frames compliant with v0.2.x format
size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
const void* src, size_t compressedSize);
/**
ZSTDv02_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.2.x format
compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
return : the number of bytes that would be read to decompress this frame
or an errorCode if it fails (which can be tested using ZSTDv02_isError())
*/
size_t ZSTDv02_frameSrcSize(const void* src, size_t compressedSize);
/**
ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error
*/

37
lib/legacy/zstd_v03.c
View File

@ -3019,6 +3019,38 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
}
static size_t ZSTD_frameSrcSize(const void* src, size_t srcSize)
{
const BYTE* ip = (const BYTE*)src;
size_t remainingSize = srcSize;
U32 magicNumber;
blockProperties_t blockProperties;
/* Frame Header */
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
magicNumber = MEM_readLE32(src);
if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
/* Loop on each block */
while (1)
{
size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
if (ZSTD_isError(cBlockSize)) return cBlockSize;
ip += ZSTD_blockHeaderSize;
remainingSize -= ZSTD_blockHeaderSize;
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
if (cBlockSize == 0) break; /* bt_end */
ip += cBlockSize;
remainingSize -= cBlockSize;
}
return ip - (const BYTE*)src;
}
/*******************************
* Streaming Decompression API
@ -3133,6 +3165,11 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
}
size_t ZSTDv03_frameSrcSize(const void* src, size_t srcSize)
{
return ZSTD_frameSrcSize(src, srcSize);
}
ZSTDv03_Dctx* ZSTDv03_createDCtx(void)
{
return (ZSTDv03_Dctx*)ZSTD_createDCtx();

8
lib/legacy/zstd_v03.h
View File

@ -35,6 +35,14 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
const void* src, size_t compressedSize);
/**
ZSTDv03_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.3.x format
compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
return : the number of bytes that would be read to decompress this frame
or an errorCode if it fails (which can be tested using ZSTDv03_isError())
*/
size_t ZSTDv03_frameSrcSize(const void* src, size_t compressedSize);
/**
ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error
*/
unsigned ZSTDv03_isError(size_t code);

33
lib/legacy/zstd_v04.c
View File

@ -3326,6 +3326,35 @@ static size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
return op-ostart;
}
static size_t ZSTD_frameSrcSize(const void* src, size_t srcSize)
{
const BYTE* ip = (const BYTE*)src;
size_t remainingSize = srcSize;
blockProperties_t blockProperties;
/* Frame Header */
if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown);
ip += ZSTD_frameHeaderSize_min; remainingSize -= ZSTD_frameHeaderSize_min;
/* Loop on each block */
while (1)
{
size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties);
if (ZSTD_isError(cBlockSize)) return cBlockSize;
ip += ZSTD_blockHeaderSize;
remainingSize -= ZSTD_blockHeaderSize;
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
if (cBlockSize == 0) break; /* bt_end */
ip += cBlockSize;
remainingSize -= cBlockSize;
}
return ip - (const BYTE*)src;
}
/* ******************************
* Streaming Decompression API
@ -3753,6 +3782,10 @@ size_t ZSTDv04_decompress(void* dst, size_t maxDstSize, const void* src, size_t
#endif
}
size_t ZSTDv04_frameSrcSize(const void* src, size_t srcSize)
{
return ZSTD_frameSrcSize(src, srcSize);
}
size_t ZSTDv04_resetDCtx(ZSTDv04_Dctx* dctx) { return ZSTD_resetDCtx(dctx); }

8
lib/legacy/zstd_v04.h
View File

@ -34,6 +34,14 @@ ZSTDv04_decompress() : decompress ZSTD frames compliant with v0.4.x format
size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize,
const void* src, size_t compressedSize);
/**
ZSTDv04_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.4.x format
compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
return : the number of bytes that would be read to decompress this frame
or an errorCode if it fails (which can be tested using ZSTDv04_isError())
*/
size_t ZSTDv04_frameSrcSize(const void* src, size_t compressedSize);
/**
ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error
*/

29
lib/legacy/zstd_v05.c
View File

@ -3583,6 +3583,35 @@ size_t ZSTDv05_decompress(void* dst, size_t maxDstSize, const void* src, size_t
#endif
}
size_t ZSTDv05_frameSrcSize(const void *src, size_t srcSize)
{
const BYTE* ip = (const BYTE*)src;
size_t remainingSize = srcSize;
blockProperties_t blockProperties;
/* Frame Header */
if (srcSize < ZSTDv05_frameHeaderSize_min) return ERROR(srcSize_wrong);
if (MEM_readLE32(src) != ZSTDv05_MAGICNUMBER) return ERROR(prefix_unknown);
ip += ZSTDv05_frameHeaderSize_min; remainingSize -= ZSTDv05_frameHeaderSize_min;
/* Loop on each block */
while (1)
{
size_t cBlockSize = ZSTDv05_getcBlockSize(ip, remainingSize, &blockProperties);
if (ZSTDv05_isError(cBlockSize)) return cBlockSize;
ip += ZSTDv05_blockHeaderSize;
remainingSize -= ZSTDv05_blockHeaderSize;
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
if (cBlockSize == 0) break; /* bt_end */
ip += cBlockSize;
remainingSize -= cBlockSize;
}
return ip - (const BYTE*)src;
}
/* ******************************
* Streaming Decompression API

7
lib/legacy/zstd_v05.h
View File

@ -32,6 +32,13 @@ extern "C" {
size_t ZSTDv05_decompress( void* dst, size_t dstCapacity,
const void* src, size_t compressedSize);
/**
ZSTDv05_getFrameSrcSize() : get the source length of a ZSTD frame
compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
return : the number of bytes that would be read to decompress this frame
or an errorCode if it fails (which can be tested using ZSTDv05_isError())
*/
size_t ZSTDv05_frameSrcSize(const void* src, size_t compressedSize);
/* *************************************
* Helper functions

31
lib/legacy/zstd_v06.c
View File

@ -3729,6 +3729,37 @@ size_t ZSTDv06_decompress(void* dst, size_t dstCapacity, const void* src, size_t
#endif
}
size_t ZSTDv06_frameSrcSize(const void* src, size_t srcSize)
{
const BYTE* ip = (const BYTE*)src;
size_t remainingSize = srcSize;
blockProperties_t blockProperties = { bt_compressed, 0 };
/* Frame Header */
{ size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, ZSTDv06_frameHeaderSize_min);
if (ZSTDv06_isError(frameHeaderSize)) return frameHeaderSize;
if (MEM_readLE32(src) != ZSTDv06_MAGICNUMBER) return ERROR(prefix_unknown);
if (srcSize < frameHeaderSize+ZSTDv06_blockHeaderSize) return ERROR(srcSize_wrong);
ip += frameHeaderSize; remainingSize -= frameHeaderSize;
}
/* Loop on each block */
while (1) {
size_t const cBlockSize = ZSTDv06_getcBlockSize(ip, remainingSize, &blockProperties);
if (ZSTDv06_isError(cBlockSize)) return cBlockSize;
ip += ZSTDv06_blockHeaderSize;
remainingSize -= ZSTDv06_blockHeaderSize;
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
if (cBlockSize == 0) break; /* bt_end */
ip += cBlockSize;
remainingSize -= cBlockSize;
}
return ip - (const BYTE*)src;
}
/*_******************************
* Streaming Decompression API

7
lib/legacy/zstd_v06.h
View File

@ -41,6 +41,13 @@ extern "C" {
ZSTDLIBv06_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity,
const void* src, size_t compressedSize);
/**
ZSTDv06_getFrameSrcSize() : get the source length of a ZSTD frame
compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
return : the number of bytes that would be read to decompress this frame
or an errorCode if it fails (which can be tested using ZSTDv06_isError())
*/
size_t ZSTDv06_frameSrcSize(const void* src, size_t compressedSize);
/* *************************************
* Helper functions

32
lib/legacy/zstd_v07.c
View File

@ -3968,6 +3968,38 @@ size_t ZSTDv07_decompress(void* dst, size_t dstCapacity, const void* src, size_t
#endif
}
size_t ZSTDv07_frameSrcSize(const void* src, size_t srcSize)
{
const BYTE* ip = (const BYTE*)src;
size_t remainingSize = srcSize;
/* check */
if (srcSize < ZSTDv07_frameHeaderSize_min+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong);
/* Frame Header */
{ size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, ZSTDv07_frameHeaderSize_min);
if (ZSTDv07_isError(frameHeaderSize)) return frameHeaderSize;
if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) return ERROR(prefix_unknown);
if (srcSize < frameHeaderSize+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong);
ip += frameHeaderSize; remainingSize -= frameHeaderSize;
}
/* Loop on each block */
while (1) {
blockProperties_t blockProperties;
size_t const cBlockSize = ZSTDv07_getcBlockSize(ip, remainingSize, &blockProperties);
if (ZSTDv07_isError(cBlockSize)) return cBlockSize;
ip += ZSTDv07_blockHeaderSize;
remainingSize -= ZSTDv07_blockHeaderSize;
if (cBlockSize > remainingSize) return ERROR(srcSize_wrong);
ip += cBlockSize;
remainingSize -= cBlockSize;
}
return ip - (const BYTE*)src;
}
/*_******************************
* Streaming Decompression API

8
lib/legacy/zstd_v07.h
View File

@ -48,6 +48,14 @@ unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize);
ZSTDLIBv07_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity,
const void* src, size_t compressedSize);
/**
ZSTDv07_getFrameSrcSize() : get the source length of a ZSTD frame
compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src'
return : the number of bytes that would be read to decompress this frame
or an errorCode if it fails (which can be tested using ZSTDv07_isError())
*/
size_t ZSTDv07_frameSrcSize(const void* src, size_t compressedSize);
/*====== Helper functions ======*/
ZSTDLIBv07_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
ZSTDLIBv07_API const char* ZSTDv07_getErrorName(size_t code); /*!< provides readable string from an error code */

40
lib/zstd.h
View File

@ -80,7 +80,7 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
int compressionLevel);
/*! ZSTD_decompress() :
`compressedSize` : must be the _exact_ size of a single compressed frame.
`compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
`dstCapacity` is an upper bound of originalSize.
If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
@ -88,7 +88,45 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
const void* src, size_t compressedSize);
#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
/*! ZSTD_getFrameContentSize() :
* `src` should point to the start of a ZSTD encoded frame
* @return : decompressed size of the frame pointed to be `src` if known, otherwise
* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
* - ZSTD_CONTENTSIZE_ERROR if an error occured (e.g. invalid magic number, srcSize too small) */
ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
/*! ZSTD_findDecompressedSize() :
* `src` should point the start of a series of ZSTD encoded and/or skippable frames
* `srcSize` must be the _exact_ size of this series
* (i.e. there should be a frame boundary exactly `srcSize` bytes after `src`)
* @return : the decompressed size of all data in the contained frames, as a 64-bit value _if known_
* - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
* - if an error occurred: ZSTD_CONTENTSIZE_ERROR
*
* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
* When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
* In which case, it's necessary to use streaming mode to decompress data.
* Optionally, application can still use ZSTD_decompress() while relying on implied limits.
* (For example, data may be necessarily cut into blocks <= 16 KB).
* note 2 : decompressed size is always present when compression is done with ZSTD_compress()
* note 3 : decompressed size can be very large (64-bits value),
* potentially larger than what local system can handle as a single memory segment.
* In which case, it's necessary to use streaming mode to decompress data.
* note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
* Always ensure result fits within application's authorized limits.
* Each application can set its own limits.
* note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
* read each contained frame header. This is efficient as most of the data is skipped,
* however it does mean that all frame data must be present and valid. */
ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
/*! ZSTD_getDecompressedSize() :
* WARNING: This function is now obsolete. ZSTD_findDecompressedSize should be used,
* or if only exactly one ZSTD frame is needed, ZSTD_getFrameContentSize can be used.
*
* 'src' is the start of a zstd compressed frame.
* @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.

4
programs/bench.c
View File

@ -184,7 +184,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
U64 dSize64 = 0;
U32 fileNb;
for (fileNb=0; fileNb<nbFiles; fileNb++) {
U64 const fSize64 = ZSTD_getDecompressedSize(srcPtr, fileSizes[fileNb]);
U64 const fSize64 = ZSTD_findDecompressedSize(srcPtr, fileSizes[fileNb]);
if (fSize64==0) EXM_THROW(32, "Impossible to determine original size ");
dSize64 += fSize64;
srcPtr += fileSizes[fileNb];
@ -217,7 +217,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
blockTable[nbBlocks].cRoom = g_decodeOnly ? thisBlockSize : ZSTD_compressBound(thisBlockSize);
blockTable[nbBlocks].cSize = blockTable[nbBlocks].cRoom;
blockTable[nbBlocks].resPtr = (void*)resPtr;
blockTable[nbBlocks].resSize = g_decodeOnly ? (size_t) ZSTD_getDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize;
blockTable[nbBlocks].resSize = g_decodeOnly ? (size_t) ZSTD_findDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize;
srcPtr += thisBlockSize;
cPtr += blockTable[nbBlocks].cRoom;
resPtr += thisBlockSize;

4
tests/fuzzer.c
View File

@ -138,7 +138,7 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : decompressed size test : ", testNb++);
{ unsigned long long const rSize = ZSTD_getDecompressedSize(compressedBuffer, cSize);
{ unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize);
if (rSize != CNBuffSize) goto _output_error;
}
DISPLAYLEVEL(4, "OK \n");
@ -648,7 +648,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
} }
/* Decompressed size test */
{ unsigned long long const rSize = ZSTD_getDecompressedSize(cBuffer, cSize);
{ unsigned long long const rSize = ZSTD_findDecompressedSize(cBuffer, cSize);
CHECK(rSize != sampleSize, "decompressed size incorrect");
}

2
tests/symbols.c
View File

@ -15,6 +15,8 @@ static const void *symbols[] = {
&ZSTD_compress,
&ZSTD_decompress,
&ZSTD_getDecompressedSize,
&ZSTD_findDecompressedSize,
&ZSTD_getFrameContentSize,
&ZSTD_maxCLevel,
&ZSTD_compressBound,
&ZSTD_isError,

2
tests/zstreamtest.c
View File

@ -307,7 +307,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
{ size_t const r = ZSTD_endStream(zc, &outBuff);
if (r != 0) goto _output_error; } /* error, or some data not flushed */
{ unsigned long long origSize = ZSTD_getDecompressedSize(outBuff.dst, outBuff.pos);
{ unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos);
if ((size_t)origSize != CNBufferSize) goto _output_error; } /* exact original size must be present */
DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);