AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

added ZSTD_initStaticDCtx()

Browse Source
This commit is contained in:
Yann Collet 2017-05-24 17:41:41 -07:00
parent ba183005d3
commit 0fdc71c3dc
5 changed files with 135 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
doc/zstd_manual.html
View File

@ -691,6 +691,22 @@ size_t ZSTD_CDict_loadDictionary(ZSTD_CDict* cdict, const void* dict, size_t dic
</b><p> Create a ZSTD decompression context using external alloc and free functions
</p></pre><BR>
<pre><b>ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize);
</b><p> workspace: The memory area to emplace the context into.
Provided pointer must 8-bytes aligned.
It must outlive context usage.
workspaceSize: Use ZSTD_estimateDCtxSize() or ZSTD_estimateDStreamSize()
to determine how large workspace must be to support scenario.
@return : pointer to ZSTD_DCtx*, or NULL if error (size too small)
Note : zstd will never resize nor malloc() when using a static dctx.
If it needs more memory than available, it will simply error out.
Note 2 : there is no corresponding "free" function.
Since workspace was allocated externally, it must be freed externally.
Limitation : currently not compatible with internal DDict creation,
such as ZSTD_initDStream_usingDict().
</p></pre><BR>
<pre><b>ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
</b><p> Create a digested dictionary, ready to start decompression operation without startup delay.
Dictionary content is simply referenced, and therefore stays in dictBuffer.

2
lib/compress/zstd_compress.c
View File

@ -190,7 +190,7 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize)
/* note : this code should be shared with resetCCtx, instead of copied */
{ void* ptr = cctx->workSpace;
cctx->hufCTable = (HUF_CElt*)ptr;
ptr = (char*)cctx->hufCTable + hufCTable_size; /* note : HUF_CElt* is incomplete type, size is estimated via macro */
ptr = (char*)cctx->hufCTable + hufCTable_size;
cctx->offcodeCTable = (FSE_CTable*) ptr;
ptr = (char*)ptr + offcodeCTable_size;
cctx->matchlengthCTable = (FSE_CTable*) ptr;

103
lib/decompress/zstd_decompress.c
View File

@ -133,8 +133,7 @@ struct ZSTD_DCtx_s
ZSTD_customMem customMem;
size_t litSize;
size_t rleSize;
BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
size_t staticSize;
/* streaming */
ZSTD_DDict* ddictLocal;
@ -154,6 +153,10 @@ struct ZSTD_DCtx_s
U32 previousLegacyVersion;
U32 legacyVersion;
U32 hostageByte;
/* workspace */
BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx)
@ -186,24 +189,45 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
return 0;
}
ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
{
ZSTD_DCtx* dctx;
if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree) return NULL;
dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem);
if (!dctx) return NULL;
memcpy(&dctx->customMem, &customMem, sizeof(customMem));
ZSTD_decompressBegin(dctx); /* cannot fail */
dctx->streamStage = zdss_init;
dctx->staticSize = 0;
dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
dctx->ddict = NULL;
dctx->ddictLocal = NULL;
dctx->inBuff = NULL;
dctx->inBuffSize = 0;
dctx->outBuffSize= 0;
dctx->streamStage = zdss_init;
}
ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
{
ZSTD_DCtx* dctx;
if (!customMem.customAlloc && !customMem.customFree)
customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree)
return NULL;
dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem);
if (!dctx) return NULL;
memcpy(&dctx->customMem, &customMem, sizeof(customMem));
ZSTD_initDCtx_internal(dctx);
return dctx;
}
ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize)
{
ZSTD_DCtx* dctx = (ZSTD_DCtx*) workspace;
if ((size_t)workspace & 7) return NULL; /* 8-aligned */
if (workspaceSize < sizeof(ZSTD_DCtx)) return NULL; /* minimum size */
ZSTD_initDCtx_internal(dctx);
dctx->staticSize = workspaceSize;
dctx->inBuff = (char*)(dctx+1);
return dctx;
}
@ -229,10 +253,11 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
}
}
/* no longer appropriate */
void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
{
size_t const workSpaceSize = (ZSTD_BLOCKSIZE_MAX+WILDCOPY_OVERLENGTH) + ZSTD_frameHeaderSize_max;
memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */
size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx);
memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */
}
@ -2283,15 +2308,22 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
zds->blockSize = blockSize;
if ((zds->inBuffSize < blockSize) || (zds->outBuffSize < neededOutSize)) {
size_t const bufferSize = blockSize + neededOutSize;
DEBUGLOG(4, "inBuff : from %u to %u",
DEBUGLOG(5, "inBuff : from %u to %u",
(U32)zds->inBuffSize, (U32)blockSize);
DEBUGLOG(4, "outBuff : from %u to %u",
DEBUGLOG(5, "outBuff : from %u to %u",
(U32)zds->outBuffSize, (U32)neededOutSize);
ZSTD_free(zds->inBuff, zds->customMem);
zds->inBuffSize = 0;
zds->outBuffSize = 0;
zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem);
if (zds->inBuff == NULL) return ERROR(memory_allocation);
if (zds->staticSize) { /* static DCtx */
DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize);
assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* already checked at init */
if (bufferSize > zds->staticSize - sizeof(ZSTD_DCtx))
return ERROR(memory_allocation);
} else {
ZSTD_free(zds->inBuff, zds->customMem);
zds->inBuffSize = 0;
zds->outBuffSize = 0;
zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem);
if (zds->inBuff == NULL) return ERROR(memory_allocation);
}
zds->inBuffSize = blockSize;
zds->outBuff = zds->inBuff + zds->inBuffSize;
zds->outBuffSize = neededOutSize;
@ -2317,11 +2349,10 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
zds->outEnd = zds->outStart + decodedSize;
zds->streamStage = zdss_flush;
break;
}
if (ip==iend) { someMoreWork = 0; break; } /* no more input */
zds->streamStage = zdss_load;
/* pass-through */
}
} }
if (ip==iend) { someMoreWork = 0; break; } /* no more input */
zds->streamStage = zdss_load;
/* pass-through */
case zdss_load:
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds);
@ -2342,9 +2373,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
zds->inPos = 0; /* input is consumed */
if (!decodedSize && !isSkipFrame) { zds->streamStage = zdss_read; break; } /* this was just a header */
zds->outEnd = zds->outStart + decodedSize;
zds->streamStage = zdss_flush;
/* pass-through */
} }
zds->streamStage = zdss_flush;
/* pass-through */
case zdss_flush:
{ size_t const toFlushSize = zds->outEnd - zds->outStart;
@ -2356,11 +2387,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
if (zds->outStart + zds->blockSize > zds->outBuffSize)
zds->outStart = zds->outEnd = 0;
break;
}
/* cannot complete flush */
someMoreWork = 0;
break;
}
} }
/* cannot complete flush */
someMoreWork = 0;
break;
default: return ERROR(GENERIC); /* impossible */
} }
@ -2377,15 +2408,15 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
return 1;
}
input->pos++; /* release hostage */
}
} /* zds->hostageByte */
return 0;
}
} /* zds->outEnd == zds->outStart */
if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */
input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */
zds->hostageByte=1;
}
return 1;
}
} /* nextSrcSizeHint==0 */
nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block); /* preload header of next block */
if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */
nextSrcSizeHint -= zds->inPos; /* already loaded*/

18
lib/zstd.h
View File

@ -529,7 +529,7 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
ZSTDLIB_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize);
/* !!! Soon to be deprecated !!! */
/* !!! To be deprecated !!! */
typedef enum {
ZSTD_p_forceWindow, /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */
ZSTD_p_forceRawDict /* Force loading dictionary in "content-only" mode (no header analysis) */
@ -815,6 +815,22 @@ ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size);
* Create a ZSTD decompression context using external alloc and free functions */
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
/*! ZSTD_initStaticDCtx() : initialize a fixed-size zstd decompression context
* workspace: The memory area to emplace the context into.
* Provided pointer must 8-bytes aligned.
* It must outlive context usage.
* workspaceSize: Use ZSTD_estimateDCtxSize() or ZSTD_estimateDStreamSize()
* to determine how large workspace must be to support scenario.
* @return : pointer to ZSTD_DCtx*, or NULL if error (size too small)
* Note : zstd will never resize nor malloc() when using a static dctx.
* If it needs more memory than available, it will simply error out.
* Note 2 : there is no corresponding "free" function.
* Since workspace was allocated externally, it must be freed externally.
* Limitation : currently not compatible with internal DDict creation,
* such as ZSTD_initDStream_usingDict().
*/
ZSTDLIB_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize);
/*! ZSTD_createDDict_byReference() :
* Create a digested dictionary, ready to start decompression operation without startup delay.
* Dictionary content is simply referenced, and therefore stays in dictBuffer.

41
tests/fuzzer.c
View File

@ -74,7 +74,6 @@ static clock_t FUZ_clockSpan(clock_t cStart)
return clock() - cStart; /* works even when overflow; max span ~ 30mn */
}
#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
static unsigned FUZ_rand(unsigned* src)
{
@ -104,6 +103,7 @@ static unsigned FUZ_highbit32(U32 v32)
#define CHECK_V(var, fn) size_t const var = fn; if (ZSTD_isError(var)) goto _output_error
#define CHECK(fn) { CHECK_V(err, fn); }
#define CHECKPLUS(var, fn, more) { CHECK_V(var, fn); more; }
static int basicUnitTests(U32 seed, double compressibility)
{
size_t const CNBuffSize = 5 MB;
@ -195,14 +195,19 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "test%3i : create static CCtx for level %u :", testNb++, STATIC_CCTX_LEVEL);
{ ZSTD_compressionParameters const cParams = ZSTD_getCParams(STATIC_CCTX_LEVEL, 0, 0);
size_t const staticCCtxSize = ZSTD_estimateCStreamSize(cParams);
void* staticCCtxBuffer = malloc(staticCCtxSize);
if (staticCCtxBuffer==NULL) {
void* const staticCCtxBuffer = malloc(staticCCtxSize);
size_t const staticDCtxSize = ZSTD_estimateDCtxSize();
void* const staticDCtxBuffer = malloc(staticDCtxSize);
if (staticCCtxBuffer==NULL || staticDCtxBuffer==NULL) {
free(staticCCtxBuffer);
free(staticDCtxBuffer);
DISPLAY("Not enough memory, aborting\n");
testResult = 1;
goto _end;
}
{ ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize);
if (staticCCtx==NULL) goto _output_error;
ZSTD_DCtx* staticDCtx = ZSTD_initStaticDCtx(staticDCtxBuffer, staticDCtxSize);
if ((staticCCtx==NULL) || (staticDCtx==NULL)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
@ -215,13 +220,24 @@ static int basicUnitTests(U32 seed, double compressibility)
compressedBuffer, ZSTD_compressBound(CNBuffSize),
CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL),
cSize=r );
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n",
(U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : decompress verification test : ", testNb++);
{ size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
DISPLAYLEVEL(4, "test%3i : simple decompression test with static DCtx : ", testNb++);
{ size_t const r = ZSTD_decompressDCtx(staticDCtx,
decodedBuffer, CNBuffSize,
compressedBuffer, cSize);
if (r != CNBuffSize) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
{ size_t u;
for (u=0; u<CNBuffSize; u++) {
if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u])
goto _output_error;;
} }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : init CCtx for too large level (must fail) : ", testNb++);
{ size_t const r = ZSTD_compressBegin(staticCCtx, ZSTD_maxCLevel());
if (!ZSTD_isError(r)) goto _output_error; }
@ -241,8 +257,19 @@ static int basicUnitTests(U32 seed, double compressibility)
{ size_t const r = ZSTD_initCStream_usingDict(staticCCtx, CNBuffer, 64 KB, 1);
if (!ZSTD_isError(r)) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : init DStream (should fail) : ", testNb++);
{ size_t const r = ZSTD_initDStream(staticDCtx);
if (ZSTD_isError(r)) goto _output_error; }
{ ZSTD_outBuffer output = { decodedBuffer, CNBuffSize, 0 };
ZSTD_inBuffer input = { compressedBuffer, ZSTD_FRAMEHEADERSIZE_MAX+1, 0 };
size_t const r = ZSTD_decompressStream(staticDCtx, &output, &input);
if (!ZSTD_isError(r)) goto _output_error;
}
DISPLAYLEVEL(4, "OK \n");
}
free(staticCCtxBuffer);
free(staticDCtxBuffer);
}