Create a ZSTD decompression context using external alloc and free functions
ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); +workspace: The memory area to emplace the context into. + Provided pointer must 8-bytes aligned. + It must outlive context usage. + workspaceSize: Use ZSTD_estimateDCtxSize() or ZSTD_estimateDStreamSize() + to determine how large workspace must be to support scenario. + @return : pointer to ZSTD_DCtx*, or NULL if error (size too small) + Note : zstd will never resize nor malloc() when using a static dctx. + If it needs more memory than available, it will simply error out. + Note 2 : there is no corresponding "free" function. + Since workspace was allocated externally, it must be freed externally. + Limitation : currently not compatible with internal DDict creation, + such as ZSTD_initDStream_usingDict(). + +
ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);Create a digested dictionary, ready to start decompression operation without startup delay. Dictionary content is simply referenced, and therefore stays in dictBuffer. diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index deee3cfc..58ed98e9 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -190,7 +190,7 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize) /* note : this code should be shared with resetCCtx, instead of copied */ { void* ptr = cctx->workSpace; cctx->hufCTable = (HUF_CElt*)ptr; - ptr = (char*)cctx->hufCTable + hufCTable_size; /* note : HUF_CElt* is incomplete type, size is estimated via macro */ + ptr = (char*)cctx->hufCTable + hufCTable_size; cctx->offcodeCTable = (FSE_CTable*) ptr; ptr = (char*)ptr + offcodeCTable_size; cctx->matchlengthCTable = (FSE_CTable*) ptr; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 1552fec3..2db493e5 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -133,8 +133,7 @@ struct ZSTD_DCtx_s ZSTD_customMem customMem; size_t litSize; size_t rleSize; - BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH]; - BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + size_t staticSize; /* streaming */ ZSTD_DDict* ddictLocal; @@ -154,6 +153,10 @@ struct ZSTD_DCtx_s U32 previousLegacyVersion; U32 legacyVersion; U32 hostageByte; + + /* workspace */ + BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH]; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; }; /* typedef'd to ZSTD_DCtx within "zstd.h" */ size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) @@ -186,24 +189,45 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) return 0; } -ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) +static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) { - ZSTD_DCtx* dctx; - - if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; - if (!customMem.customAlloc || !customMem.customFree) return NULL; - - dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem); - if (!dctx) return NULL; - memcpy(&dctx->customMem, &customMem, sizeof(customMem)); ZSTD_decompressBegin(dctx); /* cannot fail */ - dctx->streamStage = zdss_init; + dctx->staticSize = 0; dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; dctx->ddict = NULL; dctx->ddictLocal = NULL; dctx->inBuff = NULL; dctx->inBuffSize = 0; dctx->outBuffSize= 0; + dctx->streamStage = zdss_init; +} + +ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) +{ + ZSTD_DCtx* dctx; + + if (!customMem.customAlloc && !customMem.customFree) + customMem = defaultCustomMem; + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem); + if (!dctx) return NULL; + memcpy(&dctx->customMem, &customMem, sizeof(customMem)); + ZSTD_initDCtx_internal(dctx); + return dctx; +} + +ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize) +{ + ZSTD_DCtx* dctx = (ZSTD_DCtx*) workspace; + + if ((size_t)workspace & 7) return NULL; /* 8-aligned */ + if (workspaceSize < sizeof(ZSTD_DCtx)) return NULL; /* minimum size */ + + ZSTD_initDCtx_internal(dctx); + dctx->staticSize = workspaceSize; + dctx->inBuff = (char*)(dctx+1); return dctx; } @@ -229,10 +253,11 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) } } +/* no longer appropriate */ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) { - size_t const workSpaceSize = (ZSTD_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH) + ZSTD_frameHeaderSize_max; - memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */ + size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx); + memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */ } @@ -2283,15 +2308,22 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->blockSize = blockSize; if ((zds->inBuffSize < blockSize) || (zds->outBuffSize < neededOutSize)) { size_t const bufferSize = blockSize + neededOutSize; - DEBUGLOG(4, "inBuff : from %u to %u", + DEBUGLOG(5, "inBuff : from %u to %u", (U32)zds->inBuffSize, (U32)blockSize); - DEBUGLOG(4, "outBuff : from %u to %u", + DEBUGLOG(5, "outBuff : from %u to %u", (U32)zds->outBuffSize, (U32)neededOutSize); - ZSTD_free(zds->inBuff, zds->customMem); - zds->inBuffSize = 0; - zds->outBuffSize = 0; - zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem); - if (zds->inBuff == NULL) return ERROR(memory_allocation); + if (zds->staticSize) { /* static DCtx */ + DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize); + assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* already checked at init */ + if (bufferSize > zds->staticSize - sizeof(ZSTD_DCtx)) + return ERROR(memory_allocation); + } else { + ZSTD_free(zds->inBuff, zds->customMem); + zds->inBuffSize = 0; + zds->outBuffSize = 0; + zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem); + if (zds->inBuff == NULL) return ERROR(memory_allocation); + } zds->inBuffSize = blockSize; zds->outBuff = zds->inBuff + zds->inBuffSize; zds->outBuffSize = neededOutSize; @@ -2317,11 +2349,10 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->outEnd = zds->outStart + decodedSize; zds->streamStage = zdss_flush; break; - } - if (ip==iend) { someMoreWork = 0; break; } /* no more input */ - zds->streamStage = zdss_load; - /* pass-through */ - } + } } + if (ip==iend) { someMoreWork = 0; break; } /* no more input */ + zds->streamStage = zdss_load; + /* pass-through */ case zdss_load: { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds); @@ -2342,9 +2373,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->inPos = 0; /* input is consumed */ if (!decodedSize && !isSkipFrame) { zds->streamStage = zdss_read; break; } /* this was just a header */ zds->outEnd = zds->outStart + decodedSize; - zds->streamStage = zdss_flush; - /* pass-through */ } } + zds->streamStage = zdss_flush; + /* pass-through */ case zdss_flush: { size_t const toFlushSize = zds->outEnd - zds->outStart; @@ -2356,11 +2387,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if (zds->outStart + zds->blockSize > zds->outBuffSize) zds->outStart = zds->outEnd = 0; break; - } - /* cannot complete flush */ - someMoreWork = 0; - break; - } + } } + /* cannot complete flush */ + someMoreWork = 0; + break; + default: return ERROR(GENERIC); /* impossible */ } } @@ -2377,15 +2408,15 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB return 1; } input->pos++; /* release hostage */ - } + } /* zds->hostageByte */ return 0; - } + } /* zds->outEnd == zds->outStart */ if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */ input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */ zds->hostageByte=1; } return 1; - } + } /* nextSrcSizeHint==0 */ nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block); /* preload header of next block */ if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */ nextSrcSizeHint -= zds->inPos; /* already loaded*/ diff --git a/lib/zstd.h b/lib/zstd.h index 81f9eb73..bb043e48 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -529,7 +529,7 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); -/* !!! Soon to be deprecated !!! */ +/* !!! To be deprecated !!! */ typedef enum { ZSTD_p_forceWindow, /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */ ZSTD_p_forceRawDict /* Force loading dictionary in "content-only" mode (no header analysis) */ @@ -815,6 +815,22 @@ ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size); * Create a ZSTD decompression context using external alloc and free functions */ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); +/*! ZSTD_initStaticDCtx() : initialize a fixed-size zstd decompression context + * workspace: The memory area to emplace the context into. + * Provided pointer must 8-bytes aligned. + * It must outlive context usage. + * workspaceSize: Use ZSTD_estimateDCtxSize() or ZSTD_estimateDStreamSize() + * to determine how large workspace must be to support scenario. + * @return : pointer to ZSTD_DCtx*, or NULL if error (size too small) + * Note : zstd will never resize nor malloc() when using a static dctx. + * If it needs more memory than available, it will simply error out. + * Note 2 : there is no corresponding "free" function. + * Since workspace was allocated externally, it must be freed externally. + * Limitation : currently not compatible with internal DDict creation, + * such as ZSTD_initDStream_usingDict(). + */ +ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); + /*! ZSTD_createDDict_byReference() : * Create a digested dictionary, ready to start decompression operation without startup delay. * Dictionary content is simply referenced, and therefore stays in dictBuffer. diff --git a/tests/fuzzer.c b/tests/fuzzer.c index a1dc6ba7..b2de6ec6 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -74,7 +74,6 @@ static clock_t FUZ_clockSpan(clock_t cStart) return clock() - cStart; /* works even when overflow; max span ~ 30mn */ } - #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) static unsigned FUZ_rand(unsigned* src) { @@ -104,6 +103,7 @@ static unsigned FUZ_highbit32(U32 v32) #define CHECK_V(var, fn) size_t const var = fn; if (ZSTD_isError(var)) goto _output_error #define CHECK(fn) { CHECK_V(err, fn); } #define CHECKPLUS(var, fn, more) { CHECK_V(var, fn); more; } + static int basicUnitTests(U32 seed, double compressibility) { size_t const CNBuffSize = 5 MB; @@ -195,14 +195,19 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "test%3i : create static CCtx for level %u :", testNb++, STATIC_CCTX_LEVEL); { ZSTD_compressionParameters const cParams = ZSTD_getCParams(STATIC_CCTX_LEVEL, 0, 0); size_t const staticCCtxSize = ZSTD_estimateCStreamSize(cParams); - void* staticCCtxBuffer = malloc(staticCCtxSize); - if (staticCCtxBuffer==NULL) { + void* const staticCCtxBuffer = malloc(staticCCtxSize); + size_t const staticDCtxSize = ZSTD_estimateDCtxSize(); + void* const staticDCtxBuffer = malloc(staticDCtxSize); + if (staticCCtxBuffer==NULL || staticDCtxBuffer==NULL) { + free(staticCCtxBuffer); + free(staticDCtxBuffer); DISPLAY("Not enough memory, aborting\n"); testResult = 1; goto _end; } { ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize); - if (staticCCtx==NULL) goto _output_error; + ZSTD_DCtx* staticDCtx = ZSTD_initStaticDCtx(staticDCtxBuffer, staticDCtxSize); + if ((staticCCtx==NULL) || (staticDCtx==NULL)) goto _output_error; DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL); @@ -215,13 +220,24 @@ static int basicUnitTests(U32 seed, double compressibility) compressedBuffer, ZSTD_compressBound(CNBuffSize), CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL), cSize=r ); - DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); + DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", + (U32)cSize, (double)cSize/CNBuffSize*100); - DISPLAYLEVEL(4, "test%3i : decompress verification test : ", testNb++); - { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); + DISPLAYLEVEL(4, "test%3i : simple decompression test with static DCtx : ", testNb++); + { size_t const r = ZSTD_decompressDCtx(staticDCtx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize); if (r != CNBuffSize) goto _output_error; } DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); + { size_t u; + for (u=0; u