AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #2354 from terrelln/stable-buffer

Browse Source 
Add ZSTD_c_stable{In,Out}Buffer and optimize when set
This commit is contained in:
Nick Terrell 2020-10-30 15:06:56 -07:00 committed by GitHub
commit 7205e609a9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 414 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
lib/common/error_private.c
View File

@ -48,6 +48,7 @@ const char* ERR_getErrorString(ERR_enum code)
case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong";
case PREFIX(srcBuffer_wrong): return "Source buffer is wrong";
case PREFIX(maxCode):
default: return notErrorCode;
}

1
lib/common/zstd_errors.h
View File

@ -77,6 +77,7 @@ typedef enum {
ZSTD_error_frameIndex_tooLarge = 100,
ZSTD_error_seekableIO = 102,
ZSTD_error_dstBuffer_wrong = 104,
ZSTD_error_srcBuffer_wrong = 105,
ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */
} ZSTD_ErrorCode;

6
lib/common/zstd_internal.h
View File

@ -336,6 +336,12 @@ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src,
* In which case, resize it down to free some memory */
#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128
/* Controls whether the input/output buffer is buffered or stable. */
typedef enum {
ZSTD_bm_buffered = 0, /* Buffer the input/output */
ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */
} ZSTD_bufferMode_e;
/*-*******************************************
* Private declarations

197
lib/compress/zstd_compress.c
View File

@ -452,6 +452,12 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
bounds.upperBound = ZSTD_SRCSIZEHINT_MAX;
return bounds;
case ZSTD_c_stableInBuffer:
case ZSTD_c_stableOutBuffer:
bounds.lowerBound = (int)ZSTD_bm_buffered;
bounds.upperBound = (int)ZSTD_bm_stable;
return bounds;
default:
bounds.error = ERROR(parameter_unsupported);
return bounds;
@ -509,6 +515,8 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
case ZSTD_c_literalCompressionMode:
case ZSTD_c_targetCBlockSize:
case ZSTD_c_srcSizeHint:
case ZSTD_c_stableInBuffer:
case ZSTD_c_stableOutBuffer:
default:
return 0;
}
@ -557,6 +565,8 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
case ZSTD_c_ldmBucketSizeLog:
case ZSTD_c_targetCBlockSize:
case ZSTD_c_srcSizeHint:
case ZSTD_c_stableInBuffer:
case ZSTD_c_stableOutBuffer:
break;
default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
@ -748,6 +758,16 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
CCtxParams->srcSizeHint = value;
return CCtxParams->srcSizeHint;
case ZSTD_c_stableInBuffer:
BOUNDCHECK(ZSTD_c_stableInBuffer, value);
CCtxParams->inBufferMode = (ZSTD_bufferMode_e)value;
return CCtxParams->inBufferMode;
case ZSTD_c_stableOutBuffer:
BOUNDCHECK(ZSTD_c_stableOutBuffer, value);
CCtxParams->outBufferMode = (ZSTD_bufferMode_e)value;
return CCtxParams->outBufferMode;
default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
}
}
@ -859,6 +879,12 @@ size_t ZSTD_CCtxParams_getParameter(
case ZSTD_c_srcSizeHint :
*value = (int)CCtxParams->srcSizeHint;
break;
case ZSTD_c_stableInBuffer :
*value = (int)CCtxParams->inBufferMode;
break;
case ZSTD_c_stableOutBuffer :
*value = (int)CCtxParams->outBufferMode;
break;
default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
}
return 0;
@ -1325,8 +1351,12 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
{ ZSTD_compressionParameters const cParams =
ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize;
size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1;
size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered)
? ((size_t)1 << cParams.windowLog) + blockSize
: 0;
size_t const outBuffSize = (params->outBufferMode == ZSTD_bm_buffered)
? ZSTD_compressBound(blockSize) + 1
: 0;
return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
&cParams, &params->ldmParams, 1, inBuffSize, outBuffSize,
@ -1435,16 +1465,6 @@ static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms)
ms->dictMatchState = NULL;
}
/**
* Indicates whether this compression proceeds directly from user-provided
* source buffer to user-provided destination buffer (ZSTDb_not_buffered), or
* whether the context needs to buffer the input/output (ZSTDb_buffered).
*/
typedef enum {
ZSTDb_not_buffered,
ZSTDb_buffered
} ZSTD_buffered_policy_e;
/**
* Controls, for this matchState reset, whether the tables need to be cleared /
* prepared for the coming compression (ZSTDcrp_makeClean), or whether the
@ -1572,8 +1592,12 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
U32 const divider = (params.cParams.minMatch==3) ? 3 : 4;
size_t const maxNbSeq = blockSize / divider;
size_t const buffOutSize = (zbuff==ZSTDb_buffered) ? ZSTD_compressBound(blockSize)+1 : 0;
size_t const buffInSize = (zbuff==ZSTDb_buffered) ? windowSize + blockSize : 0;
size_t const buffOutSize = (zbuff == ZSTDb_buffered && params.outBufferMode == ZSTD_bm_buffered)
? ZSTD_compressBound(blockSize) + 1
: 0;
size_t const buffInSize = (zbuff == ZSTDb_buffered && params.inBufferMode == ZSTD_bm_buffered)
? windowSize + blockSize
: 0;
size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize);
int const indexTooClose = ZSTD_indexTooCloseToMax(zc->blockState.matchState.window);
@ -1648,6 +1672,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
zc->seqStore.maxNbLit = blockSize;
/* buffers */
zc->bufferedPolicy = zbuff;
zc->inBuffSize = buffInSize;
zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize);
zc->outBuffSize = buffOutSize;
@ -1965,7 +1990,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize)
{
ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
ZSTD_buffered_policy_e const zbuff = (ZSTD_buffered_policy_e)(srcCCtx->inBuffSize>0);
ZSTD_buffered_policy_e const zbuff = srcCCtx->bufferedPolicy;
ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered==1);
if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
fParams.contentSizeFlag = (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN);
@ -4035,10 +4060,14 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
/* check expectations */
DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%u", (unsigned)flushMode);
assert(zcs->inBuff != NULL);
assert(zcs->inBuffSize > 0);
assert(zcs->outBuff != NULL);
assert(zcs->outBuffSize > 0);
if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) {
assert(zcs->inBuff != NULL);
assert(zcs->inBuffSize > 0);
}
if (zcs->appliedParams.outBufferMode == ZSTD_bm_buffered) {
assert(zcs->outBuff != NULL);
assert(zcs->outBuffSize > 0);
}
assert(output->pos <= output->size);
assert(input->pos <= input->size);
assert((U32)flushMode <= (U32)ZSTD_e_end);
@ -4051,7 +4080,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
case zcss_load:
if ( (flushMode == ZSTD_e_end)
&& ((size_t)(oend-op) >= ZSTD_compressBound(iend-ip)) /* enough dstCapacity */
&& ( (size_t)(oend-op) >= ZSTD_compressBound(iend-ip) /* Enough output space */
|| zcs->appliedParams.outBufferMode == ZSTD_bm_stable) /* OR we are allowed to return dstSizeTooSmall */
&& (zcs->inBuffPos == 0) ) {
/* shortcut to compression pass directly into output buffer */
size_t const cSize = ZSTD_compressEnd(zcs,
@ -4064,8 +4094,9 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
someMoreWork = 0; break;
}
/* complete loading into inBuffer */
{ size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos;
/* complete loading into inBuffer in buffered mode */
if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) {
size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos;
size_t const loaded = ZSTD_limitCopy(
zcs->inBuff + zcs->inBuffPos, toLoad,
ip, iend-ip);
@ -4085,31 +4116,49 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
}
/* compress current block (note : this stage cannot be stopped in the middle) */
DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode);
{ void* cDst;
{ int const inputBuffered = (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered);
void* cDst;
size_t cSize;
size_t const iSize = zcs->inBuffPos - zcs->inToCompress;
size_t oSize = oend-op;
unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend);
if (oSize >= ZSTD_compressBound(iSize))
size_t const iSize = inputBuffered
? zcs->inBuffPos - zcs->inToCompress
: MIN((size_t)(iend - ip), zcs->blockSize);
if (oSize >= ZSTD_compressBound(iSize) || zcs->appliedParams.outBufferMode == ZSTD_bm_stable)
cDst = op; /* compress into output buffer, to skip flush stage */
else
cDst = zcs->outBuff, oSize = zcs->outBuffSize;
cSize = lastBlock ?
ZSTD_compressEnd(zcs, cDst, oSize,
zcs->inBuff + zcs->inToCompress, iSize) :
ZSTD_compressContinue(zcs, cDst, oSize,
zcs->inBuff + zcs->inToCompress, iSize);
FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed");
zcs->frameEnded = lastBlock;
/* prepare next block */
zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize;
if (zcs->inBuffTarget > zcs->inBuffSize)
zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize;
DEBUGLOG(5, "inBuffTarget:%u / inBuffSize:%u",
(unsigned)zcs->inBuffTarget, (unsigned)zcs->inBuffSize);
if (!lastBlock)
assert(zcs->inBuffTarget <= zcs->inBuffSize);
zcs->inToCompress = zcs->inBuffPos;
if (inputBuffered) {
unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend);
cSize = lastBlock ?
ZSTD_compressEnd(zcs, cDst, oSize,
zcs->inBuff + zcs->inToCompress, iSize) :
ZSTD_compressContinue(zcs, cDst, oSize,
zcs->inBuff + zcs->inToCompress, iSize);
FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed");
zcs->frameEnded = lastBlock;
/* prepare next block */
zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize;
if (zcs->inBuffTarget > zcs->inBuffSize)
zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize;
DEBUGLOG(5, "inBuffTarget:%u / inBuffSize:%u",
(unsigned)zcs->inBuffTarget, (unsigned)zcs->inBuffSize);
if (!lastBlock)
assert(zcs->inBuffTarget <= zcs->inBuffSize);
zcs->inToCompress = zcs->inBuffPos;
} else {
unsigned const lastBlock = (ip + iSize == iend);
assert(flushMode == ZSTD_e_end /* Already validated */);
cSize = lastBlock ?
ZSTD_compressEnd(zcs, cDst, oSize, ip, iSize) :
ZSTD_compressContinue(zcs, cDst, oSize, ip, iSize);
/* Consume the input prior to error checking to mirror buffered mode. */
if (iSize > 0)
ip += iSize;
FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed");
zcs->frameEnded = lastBlock;
if (lastBlock)
assert(ip == iend);
}
if (cDst == op) { /* no need to flush */
op += cSize;
if (zcs->frameEnded) {
@ -4126,6 +4175,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
/* fall-through */
case zcss_flush:
DEBUGLOG(5, "flush stage");
assert(zcs->appliedParams.outBufferMode == ZSTD_bm_buffered);
{ size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize;
size_t const flushed = ZSTD_limitCopy(op, (size_t)(oend-op),
zcs->outBuff + zcs->outBuffFlushedSize, toFlush);
@ -4180,6 +4230,41 @@ size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuf
return ZSTD_nextInputSizeHint_MTorST(zcs);
}
/* After a compression call set the expected input/output buffer.
* This is validated at the start of the next compression call.
*/
static void ZSTD_setBufferExpectations(ZSTD_CCtx* cctx, ZSTD_outBuffer const* output, ZSTD_inBuffer const* input)
{
if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) {
cctx->expectedInBuffer = *input;
}
if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) {
cctx->expectedOutBufferSize = output->size - output->pos;
}
}
/* Validate that the input/output buffers match the expectations set by
* ZSTD_setBufferExpectations.
*/
static size_t ZSTD_checkBufferStability(ZSTD_CCtx const* cctx,
ZSTD_outBuffer const* output,
ZSTD_inBuffer const* input,
ZSTD_EndDirective endOp)
{
if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) {
ZSTD_inBuffer const expect = cctx->expectedInBuffer;
if (expect.src != input->src || expect.pos != input->pos || expect.size != input->size)
RETURN_ERROR(srcBuffer_wrong, "ZSTD_c_stableInBuffer enabled but input differs!");
if (endOp != ZSTD_e_end)
RETURN_ERROR(srcBuffer_wrong, "ZSTD_c_stableInBuffer can only be used with ZSTD_e_end!");
}
if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) {
size_t const outBufferSize = output->size - output->pos;
if (cctx->expectedOutBufferSize != outBufferSize)
RETURN_ERROR(dstBuffer_wrong, "ZSTD_c_stableOutBuffer enabled but output size differs!");
}
return 0;
}
size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
ZSTD_outBuffer* output,
@ -4239,7 +4324,7 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType,
cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) , "");
cctx->streamStage = zcss_load;
cctx->appliedParams.nbWorkers = params.nbWorkers;
cctx->appliedParams = params;
} else
#endif
{ U64 const pledgedSrcSize = cctx->pledgedSrcSizePlusOne - 1;
@ -4252,14 +4337,24 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
assert(cctx->appliedParams.nbWorkers == 0);
cctx->inToCompress = 0;
cctx->inBuffPos = 0;
/* for small input: avoid automatic flush on reaching end of block, since it would require to add a 3-bytes null block to end frame */
cctx->inBuffTarget = cctx->blockSize + (cctx->blockSize == pledgedSrcSize);
if (cctx->appliedParams.inBufferMode == ZSTD_bm_buffered) {
/* for small input: avoid automatic flush on reaching end of block, since
* it would require to add a 3-bytes null block to end frame
*/
cctx->inBuffTarget = cctx->blockSize + (cctx->blockSize == pledgedSrcSize);
} else {
cctx->inBuffTarget = 0;
}
cctx->outBuffContentSize = cctx->outBuffFlushedSize = 0;
cctx->streamStage = zcss_load;
cctx->frameEnded = 0;
} }
}
/* Set initial buffer expectations now that we've initialized */
ZSTD_setBufferExpectations(cctx, output, input);
}
/* end of transparent initialization stage */
FORWARD_IF_ERROR(ZSTD_checkBufferStability(cctx, output, input, endOp), "invalid buffers");
/* compression stage */
#ifdef ZSTD_MULTITHREAD
if (cctx->appliedParams.nbWorkers > 0) {
@ -4283,11 +4378,13 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
* flush, or we are out of output space.
*/
assert(!forceMaxProgress || flushMin == 0 || output->pos == output->size);
ZSTD_setBufferExpectations(cctx, output, input);
return flushMin;
}
#endif
FORWARD_IF_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) , "");
DEBUGLOG(5, "completed ZSTD_compressStream2");
ZSTD_setBufferExpectations(cctx, output, input);
return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */
}
@ -4310,14 +4407,22 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
ZSTD_bufferMode_e const originalInBufferMode = cctx->requestedParams.inBufferMode;
ZSTD_bufferMode_e const originalOutBufferMode = cctx->requestedParams.outBufferMode;
DEBUGLOG(4, "ZSTD_compress2 (srcSize=%u)", (unsigned)srcSize);
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
/* Enable stable input/output buffers. */
cctx->requestedParams.inBufferMode = ZSTD_bm_stable;
cctx->requestedParams.outBufferMode = ZSTD_bm_stable;
{ size_t oPos = 0;
size_t iPos = 0;
size_t const result = ZSTD_compressStream2_simpleArgs(cctx,
dst, dstCapacity, &oPos,
src, srcSize, &iPos,
ZSTD_e_end);
/* Reset to the original values. */
cctx->requestedParams.inBufferMode = originalInBufferMode;
cctx->requestedParams.outBufferMode = originalOutBufferMode;
FORWARD_IF_ERROR(result, "ZSTD_compressStream2_simpleArgs failed");
if (result != 0) { /* compression not completed, due to lack of output space */
assert(oPos == dstCapacity);

21
lib/compress/zstd_compress_internal.h
View File

@ -238,6 +238,10 @@ struct ZSTD_CCtx_params_s {
/* Dedicated dict search algorithm trigger */
int enableDedicatedDictSearch;
/* Input/output buffer modes */
ZSTD_bufferMode_e inBufferMode;
ZSTD_bufferMode_e outBufferMode;
/* Internal use, for createCCtxParams() and freeCCtxParams() only */
ZSTD_customMem customMem;
}; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
@ -245,6 +249,16 @@ struct ZSTD_CCtx_params_s {
#define COMPRESS_SEQUENCES_WORKSPACE_SIZE (sizeof(unsigned) * (MaxSeq + 2))
#define ENTROPY_WORKSPACE_SIZE (HUF_WORKSPACE_SIZE + COMPRESS_SEQUENCES_WORKSPACE_SIZE)
/**
* Indicates whether this compression proceeds directly from user-provided
* source buffer to user-provided destination buffer (ZSTDb_not_buffered), or
* whether the context needs to buffer the input/output (ZSTDb_buffered).
*/
typedef enum {
ZSTDb_not_buffered,
ZSTDb_buffered
} ZSTD_buffered_policy_e;
struct ZSTD_CCtx_s {
ZSTD_compressionStage_e stage;
int cParamsChanged; /* == 1 if cParams(except wlog) or compression level are changed in requestedParams. Triggers transmission of new params to ZSTDMT (if available) then reset to 0. */
@ -274,6 +288,9 @@ struct ZSTD_CCtx_s {
ZSTD_blockState_t blockState;
U32* entropyWorkspace; /* entropy workspace of ENTROPY_WORKSPACE_SIZE bytes */
/* Wether we are streaming or not */
ZSTD_buffered_policy_e bufferedPolicy;
/* streaming */
char* inBuff;
size_t inBuffSize;
@ -287,6 +304,10 @@ struct ZSTD_CCtx_s {
ZSTD_cStreamStage streamStage;
U32 frameEnded;
/* Stable in/out buffer verification */
ZSTD_inBuffer expectedInBuffer;
size_t expectedOutBufferSize;
/* Dictionary */
ZSTD_localDict localDict;
const ZSTD_CDict* cdict;

5
lib/compress/zstd_cwksp.h
View File

@ -178,6 +178,8 @@ MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) {
* else is though.
*/
MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) {
if (size == 0)
return 0;
#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
return size + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
#else
@ -228,6 +230,9 @@ MEM_STATIC void* ZSTD_cwksp_reserve_internal(
ZSTD_cwksp_internal_advance_phase(ws, phase);
alloc = (BYTE *)ws->allocStart - bytes;
if (bytes == 0)
return NULL;
#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
/* over-reserve space */
alloc = (BYTE *)alloc - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;

20
lib/decompress/zstd_decompress.c
View File

@ -99,7 +99,7 @@ static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx)
assert(dctx->streamStage == zdss_init);
dctx->format = ZSTD_f_zstd1;
dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
dctx->outBufferMode = ZSTD_obm_buffered;
dctx->outBufferMode = ZSTD_bm_buffered;
dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum;
}
@ -1431,8 +1431,8 @@ ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam)
ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless);
return bounds;
case ZSTD_d_stableOutBuffer:
bounds.lowerBound = (int)ZSTD_obm_buffered;
bounds.upperBound = (int)ZSTD_obm_stable;
bounds.lowerBound = (int)ZSTD_bm_buffered;
bounds.upperBound = (int)ZSTD_bm_stable;
return bounds;
case ZSTD_d_forceIgnoreChecksum:
bounds.lowerBound = (int)ZSTD_d_validateChecksum;
@ -1495,7 +1495,7 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value
return 0;
case ZSTD_d_stableOutBuffer:
CHECK_DBOUNDS(ZSTD_d_stableOutBuffer, value);
dctx->outBufferMode = (ZSTD_outBufferMode_e)value;
dctx->outBufferMode = (ZSTD_bufferMode_e)value;
return 0;
case ZSTD_d_forceIgnoreChecksum:
CHECK_DBOUNDS(ZSTD_d_forceIgnoreChecksum, value);
@ -1585,7 +1585,7 @@ static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const*
{
ZSTD_outBuffer const expect = zds->expectedOutBuffer;
/* No requirement when ZSTD_obm_stable is not enabled. */
if (zds->outBufferMode != ZSTD_obm_stable)
if (zds->outBufferMode != ZSTD_bm_stable)
return 0;
/* Any buffer is allowed in zdss_init, this must be the same for every other call until
* the context is reset.
@ -1595,7 +1595,7 @@ static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const*
/* The buffer must match our expectation exactly. */
if (expect.dst == output->dst && expect.pos == output->pos && expect.size == output->size)
return 0;
RETURN_ERROR(dstBuffer_wrong, "ZSTD_obm_stable enabled but output differs!");
RETURN_ERROR(dstBuffer_wrong, "ZSTD_d_stableOutBuffer enabled but output differs!");
}
/* Calls ZSTD_decompressContinue() with the right parameters for ZSTD_decompressStream()
@ -1607,7 +1607,7 @@ static size_t ZSTD_decompressContinueStream(
ZSTD_DStream* zds, char** op, char* oend,
void const* src, size_t srcSize) {
int const isSkipFrame = ZSTD_isSkipFrame(zds);
if (zds->outBufferMode == ZSTD_obm_buffered) {
if (zds->outBufferMode == ZSTD_bm_buffered) {
size_t const dstSize = isSkipFrame ? 0 : zds->outBuffSize - zds->outStart;
size_t const decodedSize = ZSTD_decompressContinue(zds,
zds->outBuff + zds->outStart, dstSize, src, srcSize);
@ -1627,7 +1627,7 @@ static size_t ZSTD_decompressContinueStream(
/* Flushing is not needed. */
zds->streamStage = zdss_read;
assert(*op <= oend);
assert(zds->outBufferMode == ZSTD_obm_stable);
assert(zds->outBufferMode == ZSTD_bm_stable);
}
return 0;
}
@ -1740,7 +1740,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
} }
/* Check output buffer is large enough for ZSTD_odm_stable. */
if (zds->outBufferMode == ZSTD_obm_stable
if (zds->outBufferMode == ZSTD_bm_stable
&& zds->fParams.frameType != ZSTD_skippableFrame
&& zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN
&& (U64)(size_t)(oend-op) < zds->fParams.frameContentSize) {
@ -1770,7 +1770,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
/* Adapt buffer sizes to frame header instructions */
{ size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */);
size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_obm_buffered
size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_bm_buffered
? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize)
: 0;

7
lib/decompress/zstd_decompress_internal.h
View File

@ -99,11 +99,6 @@ typedef enum {
ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */
} ZSTD_dictUses_e;
typedef enum {
ZSTD_obm_buffered = 0, /* Buffer the output */
ZSTD_obm_stable = 1 /* ZSTD_outBuffer is stable */
} ZSTD_outBufferMode_e;
struct ZSTD_DCtx_s
{
const ZSTD_seqSymbol* LLTptr;
@ -158,7 +153,7 @@ struct ZSTD_DCtx_s
U32 legacyVersion;
U32 hostageByte;
int noForwardProgress;
ZSTD_outBufferMode_e outBufferMode;
ZSTD_bufferMode_e outBufferMode;
ZSTD_outBuffer expectedOutBuffer;
/* workspace */

59
lib/zstd.h
View File

@ -415,6 +415,8 @@ typedef enum {
* ZSTD_c_targetCBlockSize
* ZSTD_c_srcSizeHint
* ZSTD_c_enableDedicatedDictSearch
* ZSTD_c_stableInBuffer
* ZSTD_c_stableOutBuffer
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
* note : never ever use experimentalParam? names directly;
* also, the enums values themselves are unstable and can still change.
@ -426,7 +428,9 @@ typedef enum {
ZSTD_c_experimentalParam5=1002,
ZSTD_c_experimentalParam6=1003,
ZSTD_c_experimentalParam7=1004,
ZSTD_c_experimentalParam8=1005
ZSTD_c_experimentalParam8=1005,
ZSTD_c_experimentalParam9=1006,
ZSTD_c_experimentalParam10=1007
} ZSTD_cParameter;
typedef struct {
@ -1643,6 +1647,59 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
*/
#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8
/* ZSTD_c_stableInBuffer
* Experimental parameter.
* Default is 0 == disabled. Set to 1 to enable.
*
* Tells the compressor that the ZSTD_inBuffer will ALWAYS be the same
* between calls, except for the modifications that zstd makes to pos (the
* caller must not modify pos). This is checked by the compressor, and
* compression will fail if it ever changes. This means the only flush
* mode that makes sense is ZSTD_e_end, so zstd will error if ZSTD_e_end
* is not used. The data in the ZSTD_inBuffer in the range [src, src + pos)
* MUST not be modified during compression or you will get data corruption.
*
* When this flag is enabled zstd won't allocate an input window buffer,
* because the user guarantees it can reference the ZSTD_inBuffer until
* the frame is complete. But, it will still allocate an output buffer
* large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also
* avoid the memcpy() from the input buffer to the input window buffer.
*
* NOTE: ZSTD_compressStream2() will error if ZSTD_e_end is not used.
* That means this flag cannot be used with ZSTD_compressStream().
*
* NOTE: So long as the ZSTD_inBuffer always points to valid memory, using
* this flag is ALWAYS memory safe, and will never access out-of-bounds
* memory. However, compression WILL fail if you violate the preconditions.
*
* WARNING: The data in the ZSTD_inBuffer in the range [dst, dst + pos) MUST
* not be modified during compression or you will get data corruption. This
* is because zstd needs to reference data in the ZSTD_inBuffer to find
* matches. Normally zstd maintains its own window buffer for this purpose,
* but passing this flag tells zstd to use the user provided buffer.
*/
#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9
/* ZSTD_c_stableOutBuffer
* Experimental parameter.
* Default is 0 == disabled. Set to 1 to enable.
*
* Tells he compressor that the ZSTD_outBuffer will not be resized between
* calls. Specifically: (out.size - out.pos) will never grow. This gives the
* compressor the freedom to say: If the compressed data doesn't fit in the
* output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to
* always decompress directly into the output buffer, instead of decompressing
* into an internal buffer and copying to the output buffer.
*
* When this flag is enabled zstd won't allocate an output buffer, because
* it can write directly to the ZSTD_outBuffer. It will still allocate the
* input window buffer (see ZSTD_c_stableInBuffer).
*
* Zstd will check that (out.size - out.pos) never grows and return an error
* if it does. While not strictly necessary, this should prevent surprises.
*/
#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10
/*! ZSTD_CCtx_getParameter() :
* Get the requested compression parameter value, selected by enum ZSTD_cParameter,
* and store it into int* value.

160
tests/zstreamtest.c
View File

@ -754,6 +754,166 @@ static int basicUnitTests(U32 seed, double compressibility)
ZSTD_freeDCtx(dctx);
}
/* Compression with ZSTD_c_stable{In,Out}Buffer */
{ ZSTD_CCtx* cctx = ZSTD_createCCtx();
ZSTD_inBuffer in;
ZSTD_outBuffer out;
size_t cctxSize1;
size_t cctxSize2;
in.src = CNBuffer;
in.size = CNBufferSize;
out.dst = compressedBuffer;
out.size = compressedBufferSize;
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
DISPLAYLEVEL(3, "test%3i : ZSTD_compress2() uses stable input and output : ", testNb++);
CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize));
CHECK(!(cSize < ZSTD_compressBound(CNBufferSize)), "cSize too large for test");
CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, cSize + 4, CNBuffer, CNBufferSize));
CHECK_Z(cctxSize1 = ZSTD_sizeof_CCtx(cctx));
{ ZSTD_CCtx* cctx2 = ZSTD_createCCtx();
in.pos = out.pos = 0;
CHECK_Z(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_continue));
CHECK(!(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_end) == 0), "Not finished");
CHECK_Z(cctxSize2 = ZSTD_sizeof_CCtx(cctx2));
ZSTD_freeCCtx(cctx2);
}
{ ZSTD_CCtx* cctx3 = ZSTD_createCCtx();
ZSTD_parameters params = ZSTD_getParams(0, CNBufferSize, 0);
size_t cSize3;
params.fParams.checksumFlag = 1;
cSize3 = ZSTD_compress_advanced(cctx3, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize, NULL, 0, params);
CHECK_Z(cSize3);
CHECK(!(cSize == cSize3), "Must be same compressed size");
CHECK(!(cctxSize1 == ZSTD_sizeof_CCtx(cctx3)), "Must be same CCtx size");
ZSTD_freeCCtx(cctx3);
}
CHECK(!(cctxSize1 < cctxSize2), "Stable buffers means less allocated size");
CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_compress2() doesn't modify user parameters : ", testNb++);
{
int stableInBuffer;
int stableOutBuffer;
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableInBuffer, &stableInBuffer));
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableOutBuffer, &stableOutBuffer));
CHECK(!(stableInBuffer == 0), "Modified");
CHECK(!(stableOutBuffer == 0), "Modified");
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize));
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableInBuffer, &stableInBuffer));
CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableOutBuffer, &stableOutBuffer));
CHECK(!(stableInBuffer == 1), "Modified");
CHECK(!(stableOutBuffer == 1), "Modified");
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer and ZSTD_c_stableOutBuffer : ", testNb++);
CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
in.pos = out.pos = 0;
CHECK(!(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) == 0), "Not finished");
CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer and ZSTD_c_stableOutBuffer allocated size : ", testNb++);
{ size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
CHECK(!(cctxSize1 == cctxSize), "Must be the same size as single pass");
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer only : ", testNb++);
CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
in.pos = out.pos = 0;
out.size = cSize / 4;
for (;;) {
size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
CHECK_Z(ret);
if (ret == 0)
break;
out.size = MIN(out.size + cSize / 4, compressedBufferSize);
}
CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer modify buffer : ", testNb++);
in.pos = out.pos = 0;
out.size = cSize / 4;
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
in.src = (char const*)in.src + in.pos;
in.size -= in.pos;
in.pos = 0;
{ size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
CHECK(!ZSTD_isError(ret), "Must error");
CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_srcBuffer_wrong), "Must be this error");
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer with continue and flush : ", testNb++);
in.src = CNBuffer;
in.size = CNBufferSize;
in.pos = 0;
out.pos = 0;
out.size = compressedBufferSize;
CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only));
{ size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
CHECK(!ZSTD_isError(ret), "Must error");
CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_srcBuffer_wrong), "Must be this error");
}
CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only));
{ size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush);
CHECK(!ZSTD_isError(ret), "Must error");
CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_srcBuffer_wrong), "Must be this error");
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer allocated size : ", testNb++);
{ size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass");
CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming");
cctxSize1 = cctxSize;
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableOutBuffer only : ", testNb++);
CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
in.pos = out.pos = 0;
in.size = MIN(CNBufferSize, 10);
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
in.pos = 0;
in.size = CNBufferSize - in.size;
CHECK(!(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) == 0), "Not finished");
CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableOutBuffer modify buffer : ", testNb++);
in.pos = out.pos = 0;
in.size = CNBufferSize;
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
in.pos = out.pos = 0;
{ size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
CHECK(!ZSTD_isError(ret), "Must have errored");
CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_dstBuffer_wrong), "Must be this error");
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableOutBuffer allocated size : ", testNb++);
{ size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass and stableInBuffer");
CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming");
}
DISPLAYLEVEL(3, "OK \n");
ZSTD_freeCCtx(cctx);
}
/* CDict scenario */
DISPLAYLEVEL(3, "test%3i : digested dictionary : ", testNb++);
{ ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1 /*byRef*/ );