From d14cc881b0f039d641cd3a7df33bad897320e654 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 17 Jan 2018 12:39:58 -0800 Subject: [PATCH] zstdmt : fixed very large window sizes would create too large buffers, since default job size == window size * 4. This would crash on 32-bit systems. Also : jobSize being a 32-bit unsigned, it cannot be >= 4 GB, so the formula was failing for large window sizes >= 1 GB. Fixed now : max job Size is 2 GB, whatever the window size. --- lib/compress/zstdmt_compress.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index fe6ca52c..b3ccfe39 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -11,6 +11,7 @@ /* ====== Tuning parameters ====== */ #define ZSTDMT_NBTHREADS_MAX 200 +#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (2 GB)) /* note : limited by `jobSize` type, which is `unsigned` */ #define ZSTDMT_OVERLAPLOG_DEFAULT 6 @@ -843,7 +844,14 @@ size_t ZSTDMT_initCStream_internal( assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); assert(!((dict) && (cdict))); /* either dict or cdict, not both */ assert(zcs->cctxPool->totalCCtx == params.nbThreads); - zcs->singleBlockingThread = pledgedSrcSize <= ZSTDMT_JOBSIZE_MIN; /* do not trigger multi-threading when srcSize is too small */ + zcs->singleBlockingThread = (pledgedSrcSize <= ZSTDMT_JOBSIZE_MIN); /* do not trigger multi-threading when srcSize is too small */ + if (params.jobSize == 0) { + if (params.cParams.windowLog >= 29) + params.jobSize = ZSTDMT_JOBSIZE_MAX; + else + params.jobSize = 1 << (params.cParams.windowLog + 2); + } + if (params.jobSize > ZSTDMT_JOBSIZE_MAX) params.jobSize = ZSTDMT_JOBSIZE_MAX; if (zcs->singleBlockingThread) { ZSTD_CCtx_params const singleThreadParams = ZSTDMT_initJobCCtxParams(params); @@ -879,7 +887,7 @@ size_t ZSTDMT_initCStream_internal( assert(params.overlapSizeLog <= 9); zcs->targetPrefixSize = (params.overlapSizeLog==0) ? 0 : (size_t)1 << (params.cParams.windowLog - (9 - params.overlapSizeLog)); DEBUGLOG(4, "overlapLog=%u => %u KB", params.overlapSizeLog, (U32)(zcs->targetPrefixSize>>10)); - zcs->targetSectionSize = params.jobSize ? params.jobSize : (size_t)1 << (params.cParams.windowLog + 2); + zcs->targetSectionSize = params.jobSize; if (zcs->targetSectionSize < ZSTDMT_JOBSIZE_MIN) zcs->targetSectionSize = ZSTDMT_JOBSIZE_MIN; if (zcs->targetSectionSize < zcs->targetPrefixSize) zcs->targetSectionSize = zcs->targetPrefixSize; /* job size must be >= overlap size */ DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(zcs->targetSectionSize>>10), params.jobSize);