AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[libzstd] Add ZSTD_c_literalCompressionMode flag

Browse Source 
It controls the literals compression. It is either
`auto`, `huffman`, or `uncompressed`. It defaults to
`auto`, which is the current behavior.
This commit is contained in:
Nick Terrell 2019-02-13 14:59:22 -08:00
parent 32a98bafdc
commit f9513115e4
3 changed files with 56 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
lib/compress/zstd_compress.c
View File

@ -360,6 +360,12 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
bounds.upperBound = ZSTD_dictForceCopy; /* note : how to ensure at compile time that this is the highest value enum ? */
return bounds;
case ZSTD_c_literalCompressionMode:
ZSTD_STATIC_ASSERT(ZSTD_lcm_auto < ZSTD_lcm_huffman && ZSTD_lcm_huffman < ZSTD_lcm_uncompressed);
bounds.lowerBound = ZSTD_lcm_auto;
bounds.upperBound = ZSTD_lcm_uncompressed;
return bounds;
default:
{ ZSTD_bounds const boundError = { ERROR(parameter_unsupported), 0, 0 };
return boundError;
@ -396,6 +402,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
case ZSTD_c_minMatch:
case ZSTD_c_targetLength:
case ZSTD_c_strategy:
case ZSTD_c_literalCompressionMode:
return 1;
case ZSTD_c_format:
@ -463,6 +470,9 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
case ZSTD_c_forceAttachDict:
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_c_literalCompressionMode:
return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
case ZSTD_c_nbWorkers:
RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported,
"MT not compatible with static alloc");
@ -575,6 +585,13 @@ size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams,
return CCtxParams->attachDictPref;
}
case ZSTD_c_literalCompressionMode : {
const ZSTD_literalCompressionMode_e lcm = (ZSTD_literalCompressionMode_e)value;
BOUNDCHECK(ZSTD_c_literalCompressionMode, lcm);
CCtxParams->literalCompressionMode = lcm;
return CCtxParams->literalCompressionMode;
}
case ZSTD_c_nbWorkers :
#ifndef ZSTD_MULTITHREAD
RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading");
@ -688,6 +705,9 @@ size_t ZSTD_CCtxParam_getParameter(
case ZSTD_c_forceAttachDict :
*value = CCtxParams->attachDictPref;
break;
case ZSTD_c_literalCompressionMode :
*value = CCtxParams->literalCompressionMode;
break;
case ZSTD_c_nbWorkers :
#ifndef ZSTD_MULTITHREAD
assert(CCtxParams->nbWorkers == 0);
@ -2369,6 +2389,21 @@ static size_t ZSTD_encodeSequences(
sequences, nbSeq, longOffsets);
}
static int ZSTD_disableLiteralsCompression(const ZSTD_CCtx_params* cctxParams)
{
switch (cctxParams->literalCompressionMode) {
case ZSTD_lcm_huffman:
return 0;
case ZSTD_lcm_uncompressed:
return 1;
default:
assert(0 /* impossible: pre-validated */);
/* fall-through */
case ZSTD_lcm_auto:
return (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0);
}
}
/* ZSTD_compressSequences_internal():
* actually compresses both literals and sequences */
MEM_STATIC size_t
@ -2404,10 +2439,10 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
/* Compress literals */
{ const BYTE* const literals = seqStorePtr->litStart;
size_t const litSize = seqStorePtr->lit - literals;
int const disableLiteralCompression = (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0);
size_t const cSize = ZSTD_compressLiterals(
&prevEntropy->huf, &nextEntropy->huf,
cctxParams->cParams.strategy, disableLiteralCompression,
cctxParams->cParams.strategy,
ZSTD_disableLiteralsCompression(cctxParams),
op, dstCapacity,
literals, litSize,
workspace, wkspSize,

1
lib/compress/zstd_compress_internal.h
View File

@ -188,6 +188,7 @@ struct ZSTD_CCtx_params_s {
* 1<<wLog, even for dictionary */
ZSTD_dictAttachPref_e attachDictPref;
ZSTD_literalCompressionMode_e literalCompressionMode;
/* Multithreading: used to pass parameters to mtctx */
int nbWorkers;

19
lib/zstd.h
View File

@ -625,6 +625,7 @@ typedef enum {
* ZSTD_c_format
* ZSTD_c_forceMaxWindow
* ZSTD_c_forceAttachDict
* ZSTD_c_literalCompressionMode
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
* note : never ever use experimentalParam? names directly;
* also, the enums values themselves are unstable and can still change.
@ -632,7 +633,8 @@ typedef enum {
ZSTD_c_experimentalParam1=500,
ZSTD_c_experimentalParam2=10,
ZSTD_c_experimentalParam3=1000,
ZSTD_c_experimentalParam4=1001
ZSTD_c_experimentalParam4=1001,
ZSTD_c_experimentalParam5=1002,
} ZSTD_cParameter;
@ -1064,6 +1066,15 @@ typedef enum {
ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */
} ZSTD_dictAttachPref_e;
typedef enum {
ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level.
* Negative compression levels will be uncompressed, and positive compression
* levels will be compressed. */
ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be
* emitted if Huffman compression is not profitable. */
ZSTD_lcm_uncompressed = 2, /**< Always emit uncompressed literals. */
} ZSTD_literalCompressionMode_e;
/***************************************
* Frame size functions
@ -1318,6 +1329,12 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
* See the comments on that enum for an explanation of the feature. */
#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4
/* Controls how the literals are compressed (default is auto).
* The value must be of type ZSTD_literalCompressionMode_e.
* See ZSTD_literalCompressionMode_t enum definition for details.
*/
#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5
/*! ZSTD_CCtx_getParameter() :
* Get the requested compression parameter value, selected by enum ZSTD_cParameter,
* and store it into int* value.