From bf1856c26ff9c25dd0f7d92407674593ad926e0d Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Fri, 3 Apr 2020 12:07:17 -0700 Subject: [PATCH 01/12] removing max(1, ..) --- programs/fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index b8194ad6..6dd99f0f 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1389,7 +1389,7 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs, DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", srcFileName, - (double)compressedfilesize / (readsize+(!readsize)/*avoid div by zero*/) * 100, + (double)compressedfilesize / readsize * 100, (unsigned long long)readsize, (unsigned long long) compressedfilesize, dstFileName); From d0412f3abae899b368f74e98b36cd26d26d9c540 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Fri, 3 Apr 2020 12:10:02 -0700 Subject: [PATCH 02/12] no percentage on readsize == 0 --- programs/fileio.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 6dd99f0f..f2ae1bd4 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1387,11 +1387,18 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs, /* Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", - srcFileName, - (double)compressedfilesize / readsize * 100, - (unsigned long long)readsize, (unsigned long long) compressedfilesize, - dstFileName); + if (readsize == 0) { + DISPLAYLEVEL(2,"%-20s : (%6llu => %6llu bytes, %s) \n", + srcFileName, + (unsigned long long)readsize, (unsigned long long) compressedfilesize, + dstFileName); + } else { + DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", + srcFileName, + (double)compressedfilesize / readsize * 100, + (unsigned long long)readsize, (unsigned long long) compressedfilesize, + dstFileName); + } /* Elapsed Time and CPU Load */ { clock_t const cpuEnd = clock(); From 05574ec1410205e5634e5ad587ed9fcb5a4479e2 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Fri, 3 Apr 2020 12:40:31 -0700 Subject: [PATCH 03/12] adding oversizeDuration to dctx and macros --- lib/decompress/zstd_decompress.c | 1 + lib/decompress/zstd_decompress_internal.h | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 1770476a..3c51db43 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -111,6 +111,7 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) dctx->legacyContext = NULL; dctx->previousLegacyVersion = 0; dctx->noForwardProgress = 0; + dctx->oversizedDuration = 0; dctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); } diff --git a/lib/decompress/zstd_decompress_internal.h b/lib/decompress/zstd_decompress_internal.h index 059c8194..68c31381 100644 --- a/lib/decompress/zstd_decompress_internal.h +++ b/lib/decompress/zstd_decompress_internal.h @@ -95,6 +95,9 @@ typedef enum { ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */ } ZSTD_dictUses_e; +#define ZSTD_OVERSIZED_MAXDURATION 128 +#define ZSTD_OVERSIZED_FACTOR 3 + struct ZSTD_DCtx_s { const ZSTD_seqSymbol* LLTptr; @@ -151,6 +154,8 @@ struct ZSTD_DCtx_s /* workspace */ BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH]; BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + + size_t oversizedDuration; }; /* typedef'd to ZSTD_DCtx within "zstd.h" */ From 936aa63ff133221f7d9fb172fd198fb747a8597e Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Fri, 3 Apr 2020 13:20:22 -0700 Subject: [PATCH 04/12] adding oversized check on decompression --- lib/decompress/zstd_decompress.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 3c51db43..b3742343 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1508,6 +1508,24 @@ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, return length; } +static int ZSTD_isOversized(ZSTD_DStream* zds) +{ + size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); + size_t const neededOutBuffSize = ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize); + int const inOversized = zds->inBuffSize >= neededInBuffSize * ZSTD_OVERSIZED_FACTOR; + int const outOversized = zds->outBuffSize >= neededOutBuffSize * ZSTD_OVERSIZED_FACTOR; + return inOversized || outOversized; +} + +static void ZSTD_updateOversizedDuration(ZSTD_DStream* zds) +{ + zds->oversizedDuration += ZSTD_isOversized(zds) != 0; +} + +static int ZSTD_isOversizedTooLong(ZSTD_DStream* zds) +{ + return zds->oversizedDuration >= ZSTD_OVERSIZED_MAXDURATION; +} size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input) { @@ -1634,10 +1652,15 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize, frameParameter_windowTooLarge); + ZSTD_updateOversizedDuration(zds); /* Adapt buffer sizes to frame header instructions */ { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); size_t const neededOutBuffSize = ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize); - if ((zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize)) { + + int const tooSmall = (zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize); + int const tooLarge = ZSTD_isOversizedTooLong(zds); + + if (tooSmall || tooLarge) { size_t const bufferSize = neededInBuffSize + neededOutBuffSize; DEBUGLOG(4, "inBuff : from %u to %u", (U32)zds->inBuffSize, (U32)neededInBuffSize); From d598c88fb3ff590c14ce7dd62d9bbb90e672213c Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Fri, 3 Apr 2020 13:31:47 -0700 Subject: [PATCH 05/12] adding fclose before return --- contrib/diagnose_corruption/check_flipped_bits.c | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/diagnose_corruption/check_flipped_bits.c b/contrib/diagnose_corruption/check_flipped_bits.c index 78473ea5..ed3db57d 100644 --- a/contrib/diagnose_corruption/check_flipped_bits.c +++ b/contrib/diagnose_corruption/check_flipped_bits.c @@ -105,6 +105,7 @@ static char* readFile(const char* filename, size_t* size) { bytes_read = fread(buf, 1, *size, f); if (bytes_read != *size) { fprintf(stderr, "failed to read whole file\n"); + fclose(f); free(buf); return NULL; } From 1d267dc5d661479e760b6e60bada16aa576ff944 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Fri, 3 Apr 2020 13:39:02 -0700 Subject: [PATCH 06/12] returning on null check --- contrib/diagnose_corruption/check_flipped_bits.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/diagnose_corruption/check_flipped_bits.c b/contrib/diagnose_corruption/check_flipped_bits.c index ed3db57d..cc40ab84 100644 --- a/contrib/diagnose_corruption/check_flipped_bits.c +++ b/contrib/diagnose_corruption/check_flipped_bits.c @@ -100,6 +100,8 @@ static char* readFile(const char* filename, size_t* size) { buf = malloc(*size); if (buf == NULL) { fprintf(stderr, "malloc failed\n"); + fclose(f); + return NULL; } bytes_read = fread(buf, 1, *size, f); From a4cbe79ccb84092777b0f5d7c69f30d47b832d54 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Fri, 3 Apr 2020 14:09:21 -0700 Subject: [PATCH 07/12] Using in and out size together --- lib/decompress/zstd_decompress.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index b3742343..2c87a41d 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1512,9 +1512,7 @@ static int ZSTD_isOversized(ZSTD_DStream* zds) { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); size_t const neededOutBuffSize = ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize); - int const inOversized = zds->inBuffSize >= neededInBuffSize * ZSTD_OVERSIZED_FACTOR; - int const outOversized = zds->outBuffSize >= neededOutBuffSize * ZSTD_OVERSIZED_FACTOR; - return inOversized || outOversized; + return (zds->inBuffSize + zds->outBuffSize) >= (neededInBuffSize + neededOutBuffSize) * ZSTD_OVERSIZED_FACTOR; } static void ZSTD_updateOversizedDuration(ZSTD_DStream* zds) From ae47d503559659ef6e0878d30fdc7a6ed584ed49 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Fri, 3 Apr 2020 14:12:23 -0700 Subject: [PATCH 08/12] only computing sizes once --- lib/decompress/zstd_decompress.c | 63 ++++++++++++++++---------------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 2c87a41d..ba64689f 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1508,16 +1508,14 @@ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, return length; } -static int ZSTD_isOversized(ZSTD_DStream* zds) +static int ZSTD_isOversized(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) { - size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); - size_t const neededOutBuffSize = ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize); return (zds->inBuffSize + zds->outBuffSize) >= (neededInBuffSize + neededOutBuffSize) * ZSTD_OVERSIZED_FACTOR; } -static void ZSTD_updateOversizedDuration(ZSTD_DStream* zds) +static void ZSTD_updateOversizedDuration(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) { - zds->oversizedDuration += ZSTD_isOversized(zds) != 0; + zds->oversizedDuration += ZSTD_isOversized(zds, neededInBuffSize, neededOutBuffSize) != 0; } static int ZSTD_isOversizedTooLong(ZSTD_DStream* zds) @@ -1650,37 +1648,38 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize, frameParameter_windowTooLarge); - ZSTD_updateOversizedDuration(zds); /* Adapt buffer sizes to frame header instructions */ { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); size_t const neededOutBuffSize = ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize); - int const tooSmall = (zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize); - int const tooLarge = ZSTD_isOversizedTooLong(zds); - - if (tooSmall || tooLarge) { - size_t const bufferSize = neededInBuffSize + neededOutBuffSize; - DEBUGLOG(4, "inBuff : from %u to %u", - (U32)zds->inBuffSize, (U32)neededInBuffSize); - DEBUGLOG(4, "outBuff : from %u to %u", - (U32)zds->outBuffSize, (U32)neededOutBuffSize); - if (zds->staticSize) { /* static DCtx */ - DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize); - assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */ - RETURN_ERROR_IF( - bufferSize > zds->staticSize - sizeof(ZSTD_DCtx), - memory_allocation); - } else { - ZSTD_free(zds->inBuff, zds->customMem); - zds->inBuffSize = 0; - zds->outBuffSize = 0; - zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem); - RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation); - } - zds->inBuffSize = neededInBuffSize; - zds->outBuff = zds->inBuff + zds->inBuffSize; - zds->outBuffSize = neededOutBuffSize; - } } + ZSTD_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize); + + { int const tooSmall = (zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize); + int const tooLarge = ZSTD_isOversizedTooLong(zds); + + if (tooSmall || tooLarge) { + size_t const bufferSize = neededInBuffSize + neededOutBuffSize; + DEBUGLOG(4, "inBuff : from %u to %u", + (U32)zds->inBuffSize, (U32)neededInBuffSize); + DEBUGLOG(4, "outBuff : from %u to %u", + (U32)zds->outBuffSize, (U32)neededOutBuffSize); + if (zds->staticSize) { /* static DCtx */ + DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize); + assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */ + RETURN_ERROR_IF( + bufferSize > zds->staticSize - sizeof(ZSTD_DCtx), + memory_allocation); + } else { + ZSTD_free(zds->inBuff, zds->customMem); + zds->inBuffSize = 0; + zds->outBuffSize = 0; + zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem); + RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation); + } + zds->inBuffSize = neededInBuffSize; + zds->outBuff = zds->inBuff + zds->inBuffSize; + zds->outBuffSize = neededOutBuffSize; + } } } zds->streamStage = zdss_read; /* fall-through */ From 3a4c8cc9b37964b8f1e7975a85ac94d1f2843bb0 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Fri, 3 Apr 2020 14:14:46 -0700 Subject: [PATCH 09/12] adding dctx to function name --- lib/decompress/zstd_decompress.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index ba64689f..7ab2baf3 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1508,17 +1508,17 @@ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, return length; } -static int ZSTD_isOversized(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) +static int ZSTD_DCtx_isOverflow(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) { return (zds->inBuffSize + zds->outBuffSize) >= (neededInBuffSize + neededOutBuffSize) * ZSTD_OVERSIZED_FACTOR; } -static void ZSTD_updateOversizedDuration(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) +static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) { - zds->oversizedDuration += ZSTD_isOversized(zds, neededInBuffSize, neededOutBuffSize) != 0; + zds->oversizedDuration += ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize) != 0; } -static int ZSTD_isOversizedTooLong(ZSTD_DStream* zds) +static int ZSTD_DCtx_isOversizedTooLong(ZSTD_DStream* zds) { return zds->oversizedDuration >= ZSTD_OVERSIZED_MAXDURATION; } @@ -1652,10 +1652,10 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); size_t const neededOutBuffSize = ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize); - ZSTD_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize); + ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize); { int const tooSmall = (zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize); - int const tooLarge = ZSTD_isOversizedTooLong(zds); + int const tooLarge = ZSTD_DCtx_isOversizedTooLong(zds); if (tooSmall || tooLarge) { size_t const bufferSize = neededInBuffSize + neededOutBuffSize; From 0a172c5e437237a197c13646839ac9ef6e101548 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Fri, 3 Apr 2020 14:21:24 -0700 Subject: [PATCH 10/12] converting to if --- lib/decompress/zstd_decompress.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 7ab2baf3..c3690f02 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1515,7 +1515,10 @@ static int ZSTD_DCtx_isOverflow(ZSTD_DStream* zds, size_t const neededInBuffSize static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) { - zds->oversizedDuration += ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize) != 0; + if (ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize)) + zds->oversizedDuration++; + else + zds->oversizedDuration = 0; } static int ZSTD_DCtx_isOversizedTooLong(ZSTD_DStream* zds) From 01548667491ff72d264b265f5ab8d3673487b9cf Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Fri, 3 Apr 2020 14:26:15 -0700 Subject: [PATCH 11/12] moving consts to zstd_internal and reusing them --- lib/common/zstd_internal.h | 10 ++++++++++ lib/compress/zstd_cwksp.h | 10 ---------- lib/decompress/zstd_decompress.c | 4 ++-- lib/decompress/zstd_decompress_internal.h | 3 --- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index ce80f8c5..099a98e7 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -248,6 +248,16 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e } } +/* define "workspace is too large" as this number of times larger than needed */ +#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 + +/* when workspace is continuously too large + * during at least this number of times, + * context's memory usage is considered wasteful, + * because it's sized to handle a worst case scenario which rarely happens. + * In which case, resize it down to free some memory */ +#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 + /*-******************************************* * Private declarations diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 2dfab3a4..5886893d 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -24,16 +24,6 @@ extern "C" { * Constants ***************************************/ -/* define "workspace is too large" as this number of times larger than needed */ -#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 - -/* when workspace is continuously too large - * during at least this number of times, - * context's memory usage is considered wasteful, - * because it's sized to handle a worst case scenario which rarely happens. - * In which case, resize it down to free some memory */ -#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 - /* Since the workspace is effectively its own little malloc implementation / * arena, when we run under ASAN, we should similarly insert redzones between * each internal element of the workspace, so ASAN will catch overruns that diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index c3690f02..75472304 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1510,7 +1510,7 @@ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, static int ZSTD_DCtx_isOverflow(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) { - return (zds->inBuffSize + zds->outBuffSize) >= (neededInBuffSize + neededOutBuffSize) * ZSTD_OVERSIZED_FACTOR; + return (zds->inBuffSize + zds->outBuffSize) >= (neededInBuffSize + neededOutBuffSize) * ZSTD_WORKSPACETOOLARGE_FACTOR; } static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) @@ -1523,7 +1523,7 @@ static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const ne static int ZSTD_DCtx_isOversizedTooLong(ZSTD_DStream* zds) { - return zds->oversizedDuration >= ZSTD_OVERSIZED_MAXDURATION; + return zds->oversizedDuration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION; } size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input) diff --git a/lib/decompress/zstd_decompress_internal.h b/lib/decompress/zstd_decompress_internal.h index 68c31381..29b4d0ac 100644 --- a/lib/decompress/zstd_decompress_internal.h +++ b/lib/decompress/zstd_decompress_internal.h @@ -95,9 +95,6 @@ typedef enum { ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */ } ZSTD_dictUses_e; -#define ZSTD_OVERSIZED_MAXDURATION 128 -#define ZSTD_OVERSIZED_FACTOR 3 - struct ZSTD_DCtx_s { const ZSTD_seqSymbol* LLTptr; From 31e76f1ed4f1fd34b2e1b0cfd78ea2f4e2930306 Mon Sep 17 00:00:00 2001 From: Bimba Shrestha Date: Sat, 4 Apr 2020 08:49:24 -0700 Subject: [PATCH 12/12] adding test for dctx size reduction --- tests/fuzzer.c | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 1333ec9b..c102046c 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -525,7 +525,48 @@ static int basicUnitTests(U32 const seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(3, "test%3d: superblock uncompressible data, too many nocompress superblocks : ", testNb++) + DISPLAYLEVEL(3, "test%3d: check DCtx size is reduced after many oversized calls : ", testNb++); + { + size_t const largeFrameSrcSize = 200; + size_t const smallFrameSrcSize = 10; + size_t const nbFrames = 256; + + size_t i = 0, consumed = 0, produced = 0, prevDCtxSize = 0; + int sizeReduced = 0; + + BYTE* const dst = (BYTE*)compressedBuffer; + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + + /* create a large frame and then a bunch of small frames */ + size_t srcSize = ZSTD_compress((void*)dst, + compressedBufferSize, CNBuffer, largeFrameSrcSize, 3); + for (i = 0; i < nbFrames; i++) + srcSize += ZSTD_compress((void*)(dst + srcSize), + compressedBufferSize - srcSize, CNBuffer, + smallFrameSrcSize, 3); + + /* decompressStream and make sure that dctx size was reduced at least once */ + while (consumed < srcSize) { + ZSTD_inBuffer in = {(void*)(dst + consumed), MIN(1, srcSize - consumed), 0}; + ZSTD_outBuffer out = {(BYTE*)CNBuffer + produced, CNBuffSize - produced, 0}; + ZSTD_decompressStream(dctx, &out, &in); + consumed += in.pos; + produced += out.pos; + + /* success! size was reduced from the previous frame */ + if (prevDCtxSize > ZSTD_sizeof_DCtx(dctx)) + sizeReduced = 1; + + prevDCtxSize = ZSTD_sizeof_DCtx(dctx); + } + + assert(sizeReduced); + + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3d: superblock uncompressible data, too many nocompress superblocks : ", testNb++); { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); const BYTE* src = (BYTE*)CNBuffer; BYTE* dst = (BYTE*)compressedBuffer;