From f323bf7d327cdda51802f7c3bad5f5b8bc2401d2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 7 Jul 2016 13:14:21 +0200 Subject: [PATCH] added : ZSTD_getDecompressedSize() --- NEWS | 1 + lib/common/zstd.h | 9 ++++++++ lib/decompress/zstd_decompress.c | 22 ++++++++++++++++++++ lib/legacy/zstd_legacy.h | 24 ++++++++++++++++++++++ lib/legacy/zstd_v06.c | 4 +--- lib/legacy/zstd_v06.h | 2 +- programs/fuzzer.c | 35 ++++++++++++++++++++++---------- 7 files changed, 82 insertions(+), 15 deletions(-) diff --git a/NEWS b/NEWS index a62160ed..cc9a59d2 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ v0.7.3 added : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner. +added : `ZSTD_getDecompressedSize()` added : OpenBSD target, by Juan Francisco Cantero Hurtado fixed : dictBuilder using HC levels, reported by Bartosz Taudul fixed : legacy support from ZSTD_decompress_usingDDict(), reported by Felix Handte diff --git a/lib/common/zstd.h b/lib/common/zstd.h index 4de1b3dc..0ba4ca81 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -296,6 +296,15 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, /*--- Advanced Decompression functions ---*/ +/** ZSTD_getDecompressedSize() : +* compatible with legacy mode +* @return : decompressed size if known, 0 otherwise + note : 0 can mean any of the following : + - decompressed size is not provided within frame header + - frame header unknown / not supported + - frame header not completely provided (`srcSize` too small) */ +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + /*! ZSTD_createDCtx_advanced() : * Create a ZSTD decompression context using external alloc and free functions */ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 3cc38cd0..a43e8035 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -391,6 +391,28 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t } +/** ZSTD_getDecompressedSize() : +* compatible with legacy mode +* @return : decompressed size if known, 0 otherwise + note : 0 can mean any of the following : + - decompressed size is not provided within frame header + - frame header unknown / not supported + - frame header not completely provided (`srcSize` too small) */ +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) { +#if ZSTD_LEGACY_SUPPORT + if (srcSize < 4) return 0; + { U32 const magic = MEM_readLE32(src); + if (ZSTD_isLegacy(magic)) return ZSTD_getDecompressedSize_legacy(src, srcSize); + } +#endif + { ZSTD_frameParams fparams; + size_t const frResult = ZSTD_getFrameParams(&fparams, src, srcSize); + if (frResult!=0) return 0; + return fparams.frameContentSize; + } +} + + /** ZSTD_decodeFrameHeader() : * `srcSize` must be the size provided by ZSTD_frameHeaderSize(). * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h index 22921bec..56b3e821 100644 --- a/lib/legacy/zstd_legacy.h +++ b/lib/legacy/zstd_legacy.h @@ -69,6 +69,30 @@ MEM_STATIC unsigned ZSTD_isLegacy (U32 magicNumberLE) } +MEM_STATIC unsigned long long ZSTD_getDecompressedSize_legacy(const void* src, size_t srcSize) +{ + if (srcSize < 4) return 0; + + { U32 const magic = MEM_readLE32(src); + U32 const version = ZSTD_isLegacy(magic); + if (!version) return 0; /* not a supported legacy format */ + if (version < 5) return 0; /* no decompressed size in frame header */ + if (version==5) { + ZSTDv05_parameters fParams; + size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize); + if (frResult != 0) return 0; + return fParams.srcSize; + } + if (version==6) { + ZSTDv06_frameParams fParams; + size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize); + if (frResult != 0) return 0; + return fParams.frameContentSize; + } + return 0; /* should not be possible */ + } +} + MEM_STATIC size_t ZSTD_decompressLegacy( void* dst, size_t dstCapacity, const void* src, size_t compressedSize, diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index 2640c86b..ce6967eb 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -36,7 +36,7 @@ #include "zstd_v06.h" #include /* size_t, ptrdiff_t */ #include /* memcpy */ -#include /* malloc, free, qsort */ +#include /* malloc, free, qsort */ @@ -535,8 +535,6 @@ ZSTDLIB_API size_t ZSTDv06_decompress_usingPreparedDCtx( -struct ZSTDv06_frameParams_s { U64 frameContentSize; U32 windowLog; }; - #define ZSTDv06_FRAMEHEADERSIZE_MAX 13 /* for static allocation */ static const size_t ZSTDv06_frameHeaderSize_min = 5; static const size_t ZSTDv06_frameHeaderSize_max = ZSTDv06_FRAMEHEADERSIZE_MAX; diff --git a/lib/legacy/zstd_v06.h b/lib/legacy/zstd_v06.h index 55619bef..177f1483 100644 --- a/lib/legacy/zstd_v06.h +++ b/lib/legacy/zstd_v06.h @@ -107,7 +107,7 @@ ZSTDLIB_API size_t ZSTDv06_decompress_usingDict(ZSTDv06_DCtx* dctx, /*-************************ * Advanced Streaming API ***************************/ - +struct ZSTDv06_frameParams_s { unsigned long long frameContentSize; unsigned windowLog; }; typedef struct ZSTDv06_frameParams_s ZSTDv06_frameParams; ZSTDLIB_API size_t ZSTDv06_getFrameParams(ZSTDv06_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ diff --git a/programs/fuzzer.c b/programs/fuzzer.c index cd87775e..2126e124 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -137,6 +137,12 @@ static int basicUnitTests(U32 seed, double compressibility) cSize=r ); DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); + DISPLAYLEVEL(4, "test%3i : decompressed size test : ", testNb++); + { unsigned long long const rSize = ZSTD_getDecompressedSize(compressedBuffer, cSize); + if (rSize != CNBuffSize) goto _output_error; + } + DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize); CHECKPLUS( r , ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize), if (r != CNBuffSize) goto _output_error); @@ -390,19 +396,21 @@ static int basicUnitTests(U32 seed, double compressibility) U32 rSeed = 1; /* create batch of 3-bytes sequences */ - { int i; for (i=0; i < NB3BYTESSEQ; i++) { - _3BytesSeqs[i][0] = (BYTE)(FUZ_rand(&rSeed) & 255); - _3BytesSeqs[i][1] = (BYTE)(FUZ_rand(&rSeed) & 255); - _3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&rSeed) & 255); - }} + { int i; + for (i=0; i < NB3BYTESSEQ; i++) { + _3BytesSeqs[i][0] = (BYTE)(FUZ_rand(&rSeed) & 255); + _3BytesSeqs[i][1] = (BYTE)(FUZ_rand(&rSeed) & 255); + _3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&rSeed) & 255); + } } /* randomly fills CNBuffer with prepared 3-bytes sequences */ - { int i; for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */ - U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK; - ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0]; - ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1]; - ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2]; - } }} + { int i; + for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */ + U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK; + ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0]; + ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1]; + ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2]; + } } } DISPLAYLEVEL(4, "test%3i : compress lots 3-bytes sequences : ", testNb++); { CHECK_V(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH), CNBuffer, _3BYTESTESTLENGTH, 19) ); @@ -556,6 +564,11 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); } } } + /* Decompressed size test */ + { unsigned long long const rSize = ZSTD_getDecompressedSize(cBuffer, cSize); + CHECK(rSize != sampleSize, "decompressed size incorrect"); + } + /* frame header decompression test */ { ZSTD_frameParams dParams; size_t const check = ZSTD_getFrameParams(&dParams, cBuffer, cSize);