Merge branch 'dev' into newFormats
Fixed conflicts in zstdmt_compress.c
This commit is contained in:
Yann Collet
commit
54a827fff0
8
Makefile
8
Makefile
@ -112,7 +112,7 @@ CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_B
|
||||
list:
|
||||
@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
|
||||
|
||||
.PHONY: install clangtest gpptest armtest usan asan uasan
|
||||
.PHONY: install clangtest armtest usan asan uasan
|
||||
install:
|
||||
@$(MAKE) -C $(ZSTDDIR) $@
|
||||
@$(MAKE) -C $(PRGDIR) $@
|
||||
@ -179,8 +179,10 @@ ppcfuzz: clean
|
||||
ppc64fuzz: clean
|
||||
CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest
|
||||
|
||||
gpptest: clean
|
||||
CC=$(CXX) $(MAKE) -C $(PRGDIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
|
||||
.PHONY: cxxtest
|
||||
cxxtest: CXXFLAGS += -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror
|
||||
cxxtest: clean
|
||||
$(MAKE) -C $(PRGDIR) all CC="$(CXX) -Wno-deprecated" CFLAGS="$(CXXFLAGS)" # adding -Wno-deprecated to avoid clang++ warning on dealing with C files directly
|
||||
|
||||
gcc5test: clean
|
||||
gcc-5 -v
|
||||
|
||||
@ -33,7 +33,7 @@ typedef struct POOL_job_s {
|
||||
struct POOL_ctx_s {
|
||||
ZSTD_customMem customMem;
|
||||
/* Keep track of the threads */
|
||||
pthread_t *threads;
|
||||
ZSTD_pthread_t *threads;
|
||||
size_t numThreads;
|
||||
|
||||
/* The queue is a circular buffer */
|
||||
@ -48,11 +48,11 @@ struct POOL_ctx_s {
|
||||
int queueEmpty;
|
||||
|
||||
/* The mutex protects the queue */
|
||||
pthread_mutex_t queueMutex;
|
||||
ZSTD_pthread_mutex_t queueMutex;
|
||||
/* Condition variable for pushers to wait on when the queue is full */
|
||||
pthread_cond_t queuePushCond;
|
||||
ZSTD_pthread_cond_t queuePushCond;
|
||||
/* Condition variables for poppers to wait on when the queue is empty */
|
||||
pthread_cond_t queuePopCond;
|
||||
ZSTD_pthread_cond_t queuePopCond;
|
||||
/* Indicates if the queue is shutting down */
|
||||
int shutdown;
|
||||
};
|
||||
@ -67,14 +67,14 @@ static void* POOL_thread(void* opaque) {
|
||||
if (!ctx) { return NULL; }
|
||||
for (;;) {
|
||||
/* Lock the mutex and wait for a non-empty queue or until shutdown */
|
||||
pthread_mutex_lock(&ctx->queueMutex);
|
||||
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
||||
|
||||
while (ctx->queueEmpty && !ctx->shutdown) {
|
||||
pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
|
||||
ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
|
||||
}
|
||||
/* empty => shutting down: so stop */
|
||||
if (ctx->queueEmpty) {
|
||||
pthread_mutex_unlock(&ctx->queueMutex);
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
return opaque;
|
||||
}
|
||||
/* Pop a job off the queue */
|
||||
@ -83,17 +83,17 @@ static void* POOL_thread(void* opaque) {
|
||||
ctx->numThreadsBusy++;
|
||||
ctx->queueEmpty = ctx->queueHead == ctx->queueTail;
|
||||
/* Unlock the mutex, signal a pusher, and run the job */
|
||||
pthread_mutex_unlock(&ctx->queueMutex);
|
||||
pthread_cond_signal(&ctx->queuePushCond);
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
ZSTD_pthread_cond_signal(&ctx->queuePushCond);
|
||||
|
||||
job.function(job.opaque);
|
||||
|
||||
/* If the intended queue size was 0, signal after finishing job */
|
||||
if (ctx->queueSize == 1) {
|
||||
pthread_mutex_lock(&ctx->queueMutex);
|
||||
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
||||
ctx->numThreadsBusy--;
|
||||
pthread_mutex_unlock(&ctx->queueMutex);
|
||||
pthread_cond_signal(&ctx->queuePushCond);
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
ZSTD_pthread_cond_signal(&ctx->queuePushCond);
|
||||
} }
|
||||
} /* for (;;) */
|
||||
/* Unreachable */
|
||||
@ -120,12 +120,12 @@ POOL_ctx *POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customM
|
||||
ctx->queueTail = 0;
|
||||
ctx->numThreadsBusy = 0;
|
||||
ctx->queueEmpty = 1;
|
||||
(void)pthread_mutex_init(&ctx->queueMutex, NULL);
|
||||
(void)pthread_cond_init(&ctx->queuePushCond, NULL);
|
||||
(void)pthread_cond_init(&ctx->queuePopCond, NULL);
|
||||
(void)ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL);
|
||||
(void)ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL);
|
||||
(void)ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL);
|
||||
ctx->shutdown = 0;
|
||||
/* Allocate space for the thread handles */
|
||||
ctx->threads = (pthread_t*)ZSTD_malloc(numThreads * sizeof(pthread_t), customMem);
|
||||
ctx->threads = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), customMem);
|
||||
ctx->numThreads = 0;
|
||||
ctx->customMem = customMem;
|
||||
/* Check for errors */
|
||||
@ -133,7 +133,7 @@ POOL_ctx *POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customM
|
||||
/* Initialize the threads */
|
||||
{ size_t i;
|
||||
for (i = 0; i < numThreads; ++i) {
|
||||
if (pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) {
|
||||
if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) {
|
||||
ctx->numThreads = i;
|
||||
POOL_free(ctx);
|
||||
return NULL;
|
||||
@ -148,25 +148,25 @@ POOL_ctx *POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customM
|
||||
*/
|
||||
static void POOL_join(POOL_ctx *ctx) {
|
||||
/* Shut down the queue */
|
||||
pthread_mutex_lock(&ctx->queueMutex);
|
||||
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
||||
ctx->shutdown = 1;
|
||||
pthread_mutex_unlock(&ctx->queueMutex);
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
/* Wake up sleeping threads */
|
||||
pthread_cond_broadcast(&ctx->queuePushCond);
|
||||
pthread_cond_broadcast(&ctx->queuePopCond);
|
||||
ZSTD_pthread_cond_broadcast(&ctx->queuePushCond);
|
||||
ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
|
||||
/* Join all of the threads */
|
||||
{ size_t i;
|
||||
for (i = 0; i < ctx->numThreads; ++i) {
|
||||
pthread_join(ctx->threads[i], NULL);
|
||||
ZSTD_pthread_join(ctx->threads[i], NULL);
|
||||
} }
|
||||
}
|
||||
|
||||
void POOL_free(POOL_ctx *ctx) {
|
||||
if (!ctx) { return; }
|
||||
POOL_join(ctx);
|
||||
pthread_mutex_destroy(&ctx->queueMutex);
|
||||
pthread_cond_destroy(&ctx->queuePushCond);
|
||||
pthread_cond_destroy(&ctx->queuePopCond);
|
||||
ZSTD_pthread_mutex_destroy(&ctx->queueMutex);
|
||||
ZSTD_pthread_cond_destroy(&ctx->queuePushCond);
|
||||
ZSTD_pthread_cond_destroy(&ctx->queuePopCond);
|
||||
ZSTD_free(ctx->queue, ctx->customMem);
|
||||
ZSTD_free(ctx->threads, ctx->customMem);
|
||||
ZSTD_free(ctx, ctx->customMem);
|
||||
@ -176,7 +176,7 @@ size_t POOL_sizeof(POOL_ctx *ctx) {
|
||||
if (ctx==NULL) return 0; /* supports sizeof NULL */
|
||||
return sizeof(*ctx)
|
||||
+ ctx->queueSize * sizeof(POOL_job)
|
||||
+ ctx->numThreads * sizeof(pthread_t);
|
||||
+ ctx->numThreads * sizeof(ZSTD_pthread_t);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -198,12 +198,12 @@ void POOL_add(void* ctxVoid, POOL_function function, void *opaque) {
|
||||
POOL_ctx* const ctx = (POOL_ctx*)ctxVoid;
|
||||
if (!ctx) { return; }
|
||||
|
||||
pthread_mutex_lock(&ctx->queueMutex);
|
||||
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
||||
{ POOL_job const job = {function, opaque};
|
||||
|
||||
/* Wait until there is space in the queue for the new job */
|
||||
while (isQueueFull(ctx) && !ctx->shutdown) {
|
||||
pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
|
||||
ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
|
||||
}
|
||||
/* The queue is still going => there is space */
|
||||
if (!ctx->shutdown) {
|
||||
@ -212,8 +212,8 @@ void POOL_add(void* ctxVoid, POOL_function function, void *opaque) {
|
||||
ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize;
|
||||
}
|
||||
}
|
||||
pthread_mutex_unlock(&ctx->queueMutex);
|
||||
pthread_cond_signal(&ctx->queuePopCond);
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
ZSTD_pthread_cond_signal(&ctx->queuePopCond);
|
||||
}
|
||||
|
||||
#else /* ZSTD_MULTITHREAD not defined */
|
||||
|
||||
@ -35,12 +35,12 @@ int g_ZSTD_threading_useles_symbol;
|
||||
|
||||
static unsigned __stdcall worker(void *arg)
|
||||
{
|
||||
pthread_t* const thread = (pthread_t*) arg;
|
||||
ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg;
|
||||
thread->arg = thread->start_routine(thread->arg);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int pthread_create(pthread_t* thread, const void* unused,
|
||||
int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
|
||||
void* (*start_routine) (void*), void* arg)
|
||||
{
|
||||
(void)unused;
|
||||
@ -54,16 +54,16 @@ int pthread_create(pthread_t* thread, const void* unused,
|
||||
return 0;
|
||||
}
|
||||
|
||||
int _pthread_join(pthread_t * thread, void **value_ptr)
|
||||
int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
|
||||
{
|
||||
DWORD result;
|
||||
|
||||
if (!thread->handle) return 0;
|
||||
if (!thread.handle) return 0;
|
||||
|
||||
result = WaitForSingleObject(thread->handle, INFINITE);
|
||||
result = WaitForSingleObject(thread.handle, INFINITE);
|
||||
switch (result) {
|
||||
case WAIT_OBJECT_0:
|
||||
if (value_ptr) *value_ptr = thread->arg;
|
||||
if (value_ptr) *value_ptr = thread.arg;
|
||||
return 0;
|
||||
case WAIT_ABANDONED:
|
||||
return EINVAL;
|
||||
|
||||
@ -44,32 +44,31 @@ extern "C" {
|
||||
|
||||
|
||||
/* mutex */
|
||||
#define pthread_mutex_t CRITICAL_SECTION
|
||||
#define pthread_mutex_init(a,b) (InitializeCriticalSection((a)), 0)
|
||||
#define pthread_mutex_destroy(a) DeleteCriticalSection((a))
|
||||
#define pthread_mutex_lock(a) EnterCriticalSection((a))
|
||||
#define pthread_mutex_unlock(a) LeaveCriticalSection((a))
|
||||
#define ZSTD_pthread_mutex_t CRITICAL_SECTION
|
||||
#define ZSTD_pthread_mutex_init(a, b) (InitializeCriticalSection((a)), 0)
|
||||
#define ZSTD_pthread_mutex_destroy(a) DeleteCriticalSection((a))
|
||||
#define ZSTD_pthread_mutex_lock(a) EnterCriticalSection((a))
|
||||
#define ZSTD_pthread_mutex_unlock(a) LeaveCriticalSection((a))
|
||||
|
||||
/* condition variable */
|
||||
#define pthread_cond_t CONDITION_VARIABLE
|
||||
#define pthread_cond_init(a, b) (InitializeConditionVariable((a)), 0)
|
||||
#define pthread_cond_destroy(a) /* No delete */
|
||||
#define pthread_cond_wait(a, b) SleepConditionVariableCS((a), (b), INFINITE)
|
||||
#define pthread_cond_signal(a) WakeConditionVariable((a))
|
||||
#define pthread_cond_broadcast(a) WakeAllConditionVariable((a))
|
||||
#define ZSTD_pthread_cond_t CONDITION_VARIABLE
|
||||
#define ZSTD_pthread_cond_init(a, b) (InitializeConditionVariable((a)), 0)
|
||||
#define ZSTD_pthread_cond_destroy(a) /* No delete */
|
||||
#define ZSTD_pthread_cond_wait(a, b) SleepConditionVariableCS((a), (b), INFINITE)
|
||||
#define ZSTD_pthread_cond_signal(a) WakeConditionVariable((a))
|
||||
#define ZSTD_pthread_cond_broadcast(a) WakeAllConditionVariable((a))
|
||||
|
||||
/* pthread_create() and pthread_join() */
|
||||
/* ZSTD_pthread_create() and ZSTD_pthread_join() */
|
||||
typedef struct {
|
||||
HANDLE handle;
|
||||
void* (*start_routine)(void*);
|
||||
void* arg;
|
||||
} pthread_t;
|
||||
} ZSTD_pthread_t;
|
||||
|
||||
int pthread_create(pthread_t* thread, const void* unused,
|
||||
int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
|
||||
void* (*start_routine) (void*), void* arg);
|
||||
|
||||
#define pthread_join(a, b) _pthread_join(&(a), (b))
|
||||
int _pthread_join(pthread_t* thread, void** value_ptr);
|
||||
int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
|
||||
|
||||
/**
|
||||
* add here more wrappers as required
|
||||
@ -80,23 +79,40 @@ int _pthread_join(pthread_t* thread, void** value_ptr);
|
||||
/* === POSIX Systems === */
|
||||
# include <pthread.h>
|
||||
|
||||
#define ZSTD_pthread_mutex_t pthread_mutex_t
|
||||
#define ZSTD_pthread_mutex_init(a, b) pthread_mutex_init((a), (b))
|
||||
#define ZSTD_pthread_mutex_destroy(a) pthread_mutex_destroy((a))
|
||||
#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock((a))
|
||||
#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock((a))
|
||||
|
||||
#define ZSTD_pthread_cond_t pthread_cond_t
|
||||
#define ZSTD_pthread_cond_init(a, b) pthread_cond_init((a), (b))
|
||||
#define ZSTD_pthread_cond_destroy(a) pthread_cond_destroy((a))
|
||||
#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait((a), (b))
|
||||
#define ZSTD_pthread_cond_signal(a) pthread_cond_signal((a))
|
||||
#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast((a))
|
||||
|
||||
#define ZSTD_pthread_t pthread_t
|
||||
#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
|
||||
#define ZSTD_pthread_join(a, b) pthread_join((a),(b))
|
||||
|
||||
#else /* ZSTD_MULTITHREAD not defined */
|
||||
/* No multithreading support */
|
||||
|
||||
#define pthread_mutex_t int /* #define rather than typedef, because sometimes pthread support is implicit, resulting in duplicated symbols */
|
||||
#define pthread_mutex_init(a,b) ((void)a, 0)
|
||||
#define pthread_mutex_destroy(a)
|
||||
#define pthread_mutex_lock(a)
|
||||
#define pthread_mutex_unlock(a)
|
||||
typedef int ZSTD_pthread_mutex_t;
|
||||
#define ZSTD_pthread_mutex_init(a, b) ((void)a, 0)
|
||||
#define ZSTD_pthread_mutex_destroy(a)
|
||||
#define ZSTD_pthread_mutex_lock(a)
|
||||
#define ZSTD_pthread_mutex_unlock(a)
|
||||
|
||||
#define pthread_cond_t int
|
||||
#define pthread_cond_init(a,b) ((void)a, 0)
|
||||
#define pthread_cond_destroy(a)
|
||||
#define pthread_cond_wait(a,b)
|
||||
#define pthread_cond_signal(a)
|
||||
#define pthread_cond_broadcast(a)
|
||||
typedef int ZSTD_pthread_cond_t;
|
||||
#define ZSTD_pthread_cond_init(a, b) ((void)a, 0)
|
||||
#define ZSTD_pthread_cond_destroy(a)
|
||||
#define ZSTD_pthread_cond_wait(a, b)
|
||||
#define ZSTD_pthread_cond_signal(a)
|
||||
#define ZSTD_pthread_cond_broadcast(a)
|
||||
|
||||
/* do not use pthread_t */
|
||||
/* do not use ZSTD_pthread_t */
|
||||
|
||||
#endif /* ZSTD_MULTITHREAD */
|
||||
|
||||
|
||||
@ -102,6 +102,7 @@ static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
|
||||
#define BIT0 1
|
||||
|
||||
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
|
||||
#define ZSTD_WINDOWLOG_DEFAULTMAX 27 /* Default maximum allowed window log */
|
||||
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
|
||||
static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
|
||||
|
||||
|
||||
@ -438,7 +438,7 @@ size_t ZSTD_CCtxParam_setParameter(
|
||||
case ZSTD_p_enableLongDistanceMatching :
|
||||
if (value != 0) {
|
||||
ZSTD_cLevelToCCtxParams(params);
|
||||
params->cParams.windowLog = ZSTD_LDM_WINDOW_LOG;
|
||||
params->cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG;
|
||||
}
|
||||
return ZSTD_ldm_initializeParameters(¶ms->ldmParams, value);
|
||||
|
||||
@ -1608,7 +1608,7 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* op = ostart;
|
||||
U32 const maxDist = 1 << cctx->appliedParams.cParams.windowLog;
|
||||
U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog;
|
||||
|
||||
if (cctx->appliedParams.fParams.checksumFlag && srcSize)
|
||||
XXH64_update(&cctx->xxhState, src, srcSize);
|
||||
@ -1639,9 +1639,9 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
|
||||
* windowLog <= 31 ==> 3<<29 + 1<<windowLog < 7<<29 < 1<<32.
|
||||
*/
|
||||
if (cctx->lowLimit > (3U<<29)) {
|
||||
U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy)) - 1;
|
||||
U32 const cycleMask = ((U32)1 << ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy)) - 1;
|
||||
U32 const current = (U32)(ip - cctx->base);
|
||||
U32 const newCurrent = (current & cycleMask) + (1 << cctx->appliedParams.cParams.windowLog);
|
||||
U32 const newCurrent = (current & cycleMask) + ((U32)1 << cctx->appliedParams.cParams.windowLog);
|
||||
U32 const correction = current - newCurrent;
|
||||
ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30);
|
||||
ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30);
|
||||
@ -1697,7 +1697,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
|
||||
U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */
|
||||
U32 const dictIDSizeCode = params.fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */
|
||||
U32 const checksumFlag = params.fParams.checksumFlag>0;
|
||||
U32 const windowSize = 1U << params.cParams.windowLog;
|
||||
U32 const windowSize = (U32)1 << params.cParams.windowLog;
|
||||
U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize);
|
||||
BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3);
|
||||
U32 const fcsCode = params.fParams.contentSizeFlag ?
|
||||
@ -1801,7 +1801,7 @@ size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
|
||||
{
|
||||
ZSTD_compressionParameters const cParams =
|
||||
ZSTD_getCParamsFromCCtxParams(cctx->appliedParams, 0, 0);
|
||||
return MIN (ZSTD_BLOCKSIZE_MAX, 1 << cParams.windowLog);
|
||||
return MIN (ZSTD_BLOCKSIZE_MAX, (U32)1 << cParams.windowLog);
|
||||
}
|
||||
|
||||
size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
|
||||
@ -1850,7 +1850,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
|
||||
case ZSTD_btopt:
|
||||
case ZSTD_btultra:
|
||||
if (srcSize >= HASH_READ_SIZE)
|
||||
ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, 1 << zc->appliedParams.cParams.searchLog, zc->appliedParams.cParams.searchLength);
|
||||
ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, (U32)1 << zc->appliedParams.cParams.searchLog, zc->appliedParams.cParams.searchLength);
|
||||
break;
|
||||
|
||||
default:
|
||||
|
||||
@ -14,14 +14,14 @@
|
||||
|
||||
#define LDM_BUCKET_SIZE_LOG 3
|
||||
#define LDM_MIN_MATCH_LENGTH 64
|
||||
#define LDM_HASH_LOG 20
|
||||
#define LDM_HASH_RLOG 7
|
||||
#define LDM_HASH_CHAR_OFFSET 10
|
||||
|
||||
size_t ZSTD_ldm_initializeParameters(ldmParams_t* params, U32 enableLdm)
|
||||
{
|
||||
ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX);
|
||||
params->enableLdm = enableLdm>0;
|
||||
params->hashLog = LDM_HASH_LOG;
|
||||
params->hashLog = 0;
|
||||
params->bucketSizeLog = LDM_BUCKET_SIZE_LOG;
|
||||
params->minMatchLength = LDM_MIN_MATCH_LENGTH;
|
||||
params->hashEveryLog = ZSTD_LDM_HASHEVERYLOG_NOTSET;
|
||||
@ -30,6 +30,10 @@ size_t ZSTD_ldm_initializeParameters(ldmParams_t* params, U32 enableLdm)
|
||||
|
||||
void ZSTD_ldm_adjustParameters(ldmParams_t* params, U32 windowLog)
|
||||
{
|
||||
if (params->hashLog == 0) {
|
||||
params->hashLog = MAX(ZSTD_HASHLOG_MIN, windowLog - LDM_HASH_RLOG);
|
||||
assert(params->hashLog <= ZSTD_HASHLOG_MAX);
|
||||
}
|
||||
if (params->hashEveryLog == ZSTD_LDM_HASHEVERYLOG_NOTSET) {
|
||||
params->hashEveryLog =
|
||||
windowLog < params->hashLog ? 0 : windowLog - params->hashLog;
|
||||
|
||||
@ -20,7 +20,7 @@ extern "C" {
|
||||
* Long distance matching
|
||||
***************************************/
|
||||
|
||||
#define ZSTD_LDM_WINDOW_LOG 27
|
||||
#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_DEFAULTMAX
|
||||
#define ZSTD_LDM_HASHEVERYLOG_NOTSET 9999
|
||||
|
||||
/** ZSTD_compressBlock_ldm_generic() :
|
||||
|
||||
@ -529,7 +529,14 @@ size_t ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
|
||||
} else {
|
||||
opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2];
|
||||
opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1];
|
||||
opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
|
||||
/* If opt[cur].off == ZSTD_REP_MOVE_OPT, then mlen != 1.
|
||||
* offset ZSTD_REP_MOVE_OPT is used for the special case
|
||||
* litLength == 0, where offset 0 means something special.
|
||||
* mlen == 1 means the previous byte was stored as a literal,
|
||||
* so they are mutually exclusive.
|
||||
*/
|
||||
assert(!(opt[cur].off == ZSTD_REP_MOVE_OPT && mlen == 1));
|
||||
opt[cur].rep[0] = (opt[cur].off == ZSTD_REP_MOVE_OPT) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
|
||||
}
|
||||
|
||||
best_mlen = minMatch;
|
||||
@ -804,7 +811,8 @@ size_t ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
|
||||
} else {
|
||||
opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2];
|
||||
opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1];
|
||||
opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
|
||||
assert(!(opt[cur].off == ZSTD_REP_MOVE_OPT && mlen == 1));
|
||||
opt[cur].rep[0] = (opt[cur].off == ZSTD_REP_MOVE_OPT) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
|
||||
}
|
||||
|
||||
best_mlen = minMatch;
|
||||
|
||||
@ -53,10 +53,10 @@ static unsigned long long GetCurrentClockTimeMicroseconds(void)
|
||||
}
|
||||
|
||||
#define MUTEX_WAIT_TIME_DLEVEL 6
|
||||
#define PTHREAD_MUTEX_LOCK(mutex) { \
|
||||
#define ZSTD_PTHREAD_MUTEX_LOCK(mutex) { \
|
||||
if (ZSTD_DEBUG >= MUTEX_WAIT_TIME_DLEVEL) { \
|
||||
unsigned long long const beforeTime = GetCurrentClockTimeMicroseconds(); \
|
||||
pthread_mutex_lock(mutex); \
|
||||
ZSTD_pthread_mutex_lock(mutex); \
|
||||
{ unsigned long long const afterTime = GetCurrentClockTimeMicroseconds(); \
|
||||
unsigned long long const elapsedTime = (afterTime-beforeTime); \
|
||||
if (elapsedTime > 1000) { /* or whatever threshold you like; I'm using 1 millisecond here */ \
|
||||
@ -64,13 +64,13 @@ static unsigned long long GetCurrentClockTimeMicroseconds(void)
|
||||
elapsedTime, #mutex); \
|
||||
} } \
|
||||
} else { \
|
||||
pthread_mutex_lock(mutex); \
|
||||
ZSTD_pthread_mutex_lock(mutex); \
|
||||
} \
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
# define PTHREAD_MUTEX_LOCK(m) pthread_mutex_lock(m)
|
||||
# define ZSTD_PTHREAD_MUTEX_LOCK(m) ZSTD_pthread_mutex_lock(m)
|
||||
# define DEBUG_PRINTHEX(l,p,n) {}
|
||||
|
||||
#endif
|
||||
@ -87,7 +87,7 @@ typedef struct buffer_s {
|
||||
static const buffer_t g_nullBuffer = { NULL, 0 };
|
||||
|
||||
typedef struct ZSTDMT_bufferPool_s {
|
||||
pthread_mutex_t poolMutex;
|
||||
ZSTD_pthread_mutex_t poolMutex;
|
||||
size_t bufferSize;
|
||||
unsigned totalBuffers;
|
||||
unsigned nbBuffers;
|
||||
@ -101,7 +101,7 @@ static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbThreads, ZSTD_custo
|
||||
ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)ZSTD_calloc(
|
||||
sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t), cMem);
|
||||
if (bufPool==NULL) return NULL;
|
||||
if (pthread_mutex_init(&bufPool->poolMutex, NULL)) {
|
||||
if (ZSTD_pthread_mutex_init(&bufPool->poolMutex, NULL)) {
|
||||
ZSTD_free(bufPool, cMem);
|
||||
return NULL;
|
||||
}
|
||||
@ -118,7 +118,7 @@ static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool)
|
||||
if (!bufPool) return; /* compatibility with free on NULL */
|
||||
for (u=0; u<bufPool->totalBuffers; u++)
|
||||
ZSTD_free(bufPool->bTable[u].start, bufPool->cMem);
|
||||
pthread_mutex_destroy(&bufPool->poolMutex);
|
||||
ZSTD_pthread_mutex_destroy(&bufPool->poolMutex);
|
||||
ZSTD_free(bufPool, bufPool->cMem);
|
||||
}
|
||||
|
||||
@ -129,10 +129,10 @@ static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool)
|
||||
+ (bufPool->totalBuffers - 1) * sizeof(buffer_t);
|
||||
unsigned u;
|
||||
size_t totalBufferSize = 0;
|
||||
pthread_mutex_lock(&bufPool->poolMutex);
|
||||
ZSTD_pthread_mutex_lock(&bufPool->poolMutex);
|
||||
for (u=0; u<bufPool->totalBuffers; u++)
|
||||
totalBufferSize += bufPool->bTable[u].size;
|
||||
pthread_mutex_unlock(&bufPool->poolMutex);
|
||||
ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
|
||||
|
||||
return poolSize + totalBufferSize;
|
||||
}
|
||||
@ -148,20 +148,20 @@ static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool)
|
||||
{
|
||||
size_t const bSize = bufPool->bufferSize;
|
||||
DEBUGLOG(5, "ZSTDMT_getBuffer");
|
||||
pthread_mutex_lock(&bufPool->poolMutex);
|
||||
ZSTD_pthread_mutex_lock(&bufPool->poolMutex);
|
||||
if (bufPool->nbBuffers) { /* try to use an existing buffer */
|
||||
buffer_t const buf = bufPool->bTable[--(bufPool->nbBuffers)];
|
||||
size_t const availBufferSize = buf.size;
|
||||
if ((availBufferSize >= bSize) & (availBufferSize <= 10*bSize)) {
|
||||
/* large enough, but not too much */
|
||||
pthread_mutex_unlock(&bufPool->poolMutex);
|
||||
ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
|
||||
return buf;
|
||||
}
|
||||
/* size conditions not respected : scratch this buffer, create new one */
|
||||
DEBUGLOG(5, "existing buffer does not meet size conditions => freeing");
|
||||
ZSTD_free(buf.start, bufPool->cMem);
|
||||
}
|
||||
pthread_mutex_unlock(&bufPool->poolMutex);
|
||||
ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
|
||||
/* create new buffer */
|
||||
DEBUGLOG(5, "create a new buffer");
|
||||
{ buffer_t buffer;
|
||||
@ -177,13 +177,13 @@ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf)
|
||||
{
|
||||
if (buf.start == NULL) return; /* compatible with release on NULL */
|
||||
DEBUGLOG(5, "ZSTDMT_releaseBuffer");
|
||||
pthread_mutex_lock(&bufPool->poolMutex);
|
||||
ZSTD_pthread_mutex_lock(&bufPool->poolMutex);
|
||||
if (bufPool->nbBuffers < bufPool->totalBuffers) {
|
||||
bufPool->bTable[bufPool->nbBuffers++] = buf; /* stored for later use */
|
||||
pthread_mutex_unlock(&bufPool->poolMutex);
|
||||
ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
|
||||
return;
|
||||
}
|
||||
pthread_mutex_unlock(&bufPool->poolMutex);
|
||||
ZSTD_pthread_mutex_unlock(&bufPool->poolMutex);
|
||||
/* Reached bufferPool capacity (should not happen) */
|
||||
DEBUGLOG(5, "buffer pool capacity reached => freeing ");
|
||||
ZSTD_free(buf.start, bufPool->cMem);
|
||||
@ -208,7 +208,7 @@ static ZSTD_CCtx_params ZSTDMT_makeJobCCtxParams(ZSTD_CCtx_params const params)
|
||||
/* a single CCtx Pool can be invoked from multiple threads in parallel */
|
||||
|
||||
typedef struct {
|
||||
pthread_mutex_t poolMutex;
|
||||
ZSTD_pthread_mutex_t poolMutex;
|
||||
unsigned totalCCtx;
|
||||
unsigned availCCtx;
|
||||
ZSTD_customMem cMem;
|
||||
@ -221,7 +221,7 @@ static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool)
|
||||
unsigned u;
|
||||
for (u=0; u<pool->totalCCtx; u++)
|
||||
ZSTD_freeCCtx(pool->cctx[u]); /* note : compatible with free on NULL */
|
||||
pthread_mutex_destroy(&pool->poolMutex);
|
||||
ZSTD_pthread_mutex_destroy(&pool->poolMutex);
|
||||
ZSTD_free(pool, pool->cMem);
|
||||
}
|
||||
|
||||
@ -233,7 +233,7 @@ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads,
|
||||
ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_calloc(
|
||||
sizeof(ZSTDMT_CCtxPool) + (nbThreads-1)*sizeof(ZSTD_CCtx*), cMem);
|
||||
if (!cctxPool) return NULL;
|
||||
if (pthread_mutex_init(&cctxPool->poolMutex, NULL)) {
|
||||
if (ZSTD_pthread_mutex_init(&cctxPool->poolMutex, NULL)) {
|
||||
ZSTD_free(cctxPool, cMem);
|
||||
return NULL;
|
||||
}
|
||||
@ -249,7 +249,7 @@ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads,
|
||||
/* only works during initialization phase, not during compression */
|
||||
static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool)
|
||||
{
|
||||
pthread_mutex_lock(&cctxPool->poolMutex);
|
||||
ZSTD_pthread_mutex_lock(&cctxPool->poolMutex);
|
||||
{ unsigned const nbThreads = cctxPool->totalCCtx;
|
||||
size_t const poolSize = sizeof(*cctxPool)
|
||||
+ (nbThreads-1)*sizeof(ZSTD_CCtx*);
|
||||
@ -258,7 +258,7 @@ static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool)
|
||||
for (u=0; u<nbThreads; u++) {
|
||||
totalCCtxSize += ZSTD_sizeof_CCtx(cctxPool->cctx[u]);
|
||||
}
|
||||
pthread_mutex_unlock(&cctxPool->poolMutex);
|
||||
ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex);
|
||||
return poolSize + totalCCtxSize;
|
||||
}
|
||||
}
|
||||
@ -266,14 +266,14 @@ static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool)
|
||||
static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* cctxPool)
|
||||
{
|
||||
DEBUGLOG(5, "ZSTDMT_getCCtx");
|
||||
pthread_mutex_lock(&cctxPool->poolMutex);
|
||||
ZSTD_pthread_mutex_lock(&cctxPool->poolMutex);
|
||||
if (cctxPool->availCCtx) {
|
||||
cctxPool->availCCtx--;
|
||||
{ ZSTD_CCtx* const cctx = cctxPool->cctx[cctxPool->availCCtx];
|
||||
pthread_mutex_unlock(&cctxPool->poolMutex);
|
||||
ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex);
|
||||
return cctx;
|
||||
} }
|
||||
pthread_mutex_unlock(&cctxPool->poolMutex);
|
||||
ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex);
|
||||
DEBUGLOG(5, "create one more CCtx");
|
||||
return ZSTD_createCCtx_advanced(cctxPool->cMem); /* note : can be NULL, when creation fails ! */
|
||||
}
|
||||
@ -281,7 +281,7 @@ static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* cctxPool)
|
||||
static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx)
|
||||
{
|
||||
if (cctx==NULL) return; /* compatibility with release on NULL */
|
||||
pthread_mutex_lock(&pool->poolMutex);
|
||||
ZSTD_pthread_mutex_lock(&pool->poolMutex);
|
||||
if (pool->availCCtx < pool->totalCCtx)
|
||||
pool->cctx[pool->availCCtx++] = cctx;
|
||||
else {
|
||||
@ -289,7 +289,7 @@ static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx)
|
||||
DEBUGLOG(5, "CCtx pool overflow : free cctx");
|
||||
ZSTD_freeCCtx(cctx);
|
||||
}
|
||||
pthread_mutex_unlock(&pool->poolMutex);
|
||||
ZSTD_pthread_mutex_unlock(&pool->poolMutex);
|
||||
}
|
||||
|
||||
|
||||
@ -307,8 +307,8 @@ typedef struct {
|
||||
unsigned lastChunk;
|
||||
unsigned jobCompleted;
|
||||
unsigned jobScanned;
|
||||
pthread_mutex_t* jobCompleted_mutex;
|
||||
pthread_cond_t* jobCompleted_cond;
|
||||
ZSTD_pthread_mutex_t* jobCompleted_mutex;
|
||||
ZSTD_pthread_cond_t* jobCompleted_cond;
|
||||
ZSTD_CCtx_params params;
|
||||
const ZSTD_CDict* cdict;
|
||||
ZSTDMT_CCtxPool* cctxPool;
|
||||
@ -375,11 +375,11 @@ _endJob:
|
||||
ZSTDMT_releaseCCtx(job->cctxPool, cctx);
|
||||
ZSTDMT_releaseBuffer(job->bufPool, job->src);
|
||||
job->src = g_nullBuffer; job->srcStart = NULL;
|
||||
PTHREAD_MUTEX_LOCK(job->jobCompleted_mutex);
|
||||
ZSTD_PTHREAD_MUTEX_LOCK(job->jobCompleted_mutex);
|
||||
job->jobCompleted = 1;
|
||||
job->jobScanned = 0;
|
||||
pthread_cond_signal(job->jobCompleted_cond);
|
||||
pthread_mutex_unlock(job->jobCompleted_mutex);
|
||||
ZSTD_pthread_cond_signal(job->jobCompleted_cond);
|
||||
ZSTD_pthread_mutex_unlock(job->jobCompleted_mutex);
|
||||
}
|
||||
|
||||
|
||||
@ -397,8 +397,8 @@ struct ZSTDMT_CCtx_s {
|
||||
ZSTDMT_jobDescription* jobs;
|
||||
ZSTDMT_bufferPool* bufPool;
|
||||
ZSTDMT_CCtxPool* cctxPool;
|
||||
pthread_mutex_t jobCompleted_mutex;
|
||||
pthread_cond_t jobCompleted_cond;
|
||||
ZSTD_pthread_mutex_t jobCompleted_mutex;
|
||||
ZSTD_pthread_cond_t jobCompleted_cond;
|
||||
size_t targetSectionSize;
|
||||
size_t inBuffSize;
|
||||
size_t dictSize;
|
||||
@ -461,11 +461,11 @@ ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, ZSTD_customMem cMem)
|
||||
ZSTDMT_freeCCtx(mtctx);
|
||||
return NULL;
|
||||
}
|
||||
if (pthread_mutex_init(&mtctx->jobCompleted_mutex, NULL)) {
|
||||
if (ZSTD_pthread_mutex_init(&mtctx->jobCompleted_mutex, NULL)) {
|
||||
ZSTDMT_freeCCtx(mtctx);
|
||||
return NULL;
|
||||
}
|
||||
if (pthread_cond_init(&mtctx->jobCompleted_cond, NULL)) {
|
||||
if (ZSTD_pthread_cond_init(&mtctx->jobCompleted_cond, NULL)) {
|
||||
ZSTDMT_freeCCtx(mtctx);
|
||||
return NULL;
|
||||
}
|
||||
@ -505,8 +505,8 @@ size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx)
|
||||
ZSTD_free(mtctx->jobs, mtctx->cMem);
|
||||
ZSTDMT_freeCCtxPool(mtctx->cctxPool);
|
||||
ZSTD_freeCDict(mtctx->cdictLocal);
|
||||
pthread_mutex_destroy(&mtctx->jobCompleted_mutex);
|
||||
pthread_cond_destroy(&mtctx->jobCompleted_cond);
|
||||
ZSTD_pthread_mutex_destroy(&mtctx->jobCompleted_mutex);
|
||||
ZSTD_pthread_cond_destroy(&mtctx->jobCompleted_cond);
|
||||
ZSTD_free(mtctx, mtctx->cMem);
|
||||
return 0;
|
||||
}
|
||||
@ -651,12 +651,12 @@ static size_t ZSTDMT_compress_advanced_internal(
|
||||
unsigned chunkID;
|
||||
for (chunkID=0; chunkID<nbChunks; chunkID++) {
|
||||
DEBUGLOG(5, "waiting for chunk %u ", chunkID);
|
||||
PTHREAD_MUTEX_LOCK(&mtctx->jobCompleted_mutex);
|
||||
ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobCompleted_mutex);
|
||||
while (mtctx->jobs[chunkID].jobCompleted==0) {
|
||||
DEBUGLOG(5, "waiting for jobCompleted signal from chunk %u", chunkID);
|
||||
pthread_cond_wait(&mtctx->jobCompleted_cond, &mtctx->jobCompleted_mutex);
|
||||
ZSTD_pthread_cond_wait(&mtctx->jobCompleted_cond, &mtctx->jobCompleted_mutex);
|
||||
}
|
||||
pthread_mutex_unlock(&mtctx->jobCompleted_mutex);
|
||||
ZSTD_pthread_mutex_unlock(&mtctx->jobCompleted_mutex);
|
||||
DEBUGLOG(5, "ready to write chunk %u ", chunkID);
|
||||
|
||||
mtctx->jobs[chunkID].srcStart = NULL;
|
||||
@ -731,12 +731,12 @@ static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* zcs)
|
||||
DEBUGLOG(4, "ZSTDMT_waitForAllJobsCompleted");
|
||||
while (zcs->doneJobID < zcs->nextJobID) {
|
||||
unsigned const jobID = zcs->doneJobID & zcs->jobIDMask;
|
||||
PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex);
|
||||
ZSTD_PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex);
|
||||
while (zcs->jobs[jobID].jobCompleted==0) {
|
||||
DEBUGLOG(5, "waiting for jobCompleted signal from chunk %u", zcs->doneJobID); /* we want to block when waiting for data to flush */
|
||||
pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex);
|
||||
ZSTD_pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex);
|
||||
}
|
||||
pthread_mutex_unlock(&zcs->jobCompleted_mutex);
|
||||
ZSTD_pthread_mutex_unlock(&zcs->jobCompleted_mutex);
|
||||
zcs->doneJobID++;
|
||||
}
|
||||
}
|
||||
@ -925,13 +925,13 @@ static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsi
|
||||
{
|
||||
unsigned const wJobID = zcs->doneJobID & zcs->jobIDMask;
|
||||
if (zcs->doneJobID == zcs->nextJobID) return 0; /* all flushed ! */
|
||||
PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex);
|
||||
ZSTD_PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex);
|
||||
while (zcs->jobs[wJobID].jobCompleted==0) {
|
||||
DEBUGLOG(5, "waiting for jobCompleted signal from job %u", zcs->doneJobID);
|
||||
if (!blockToFlush) { pthread_mutex_unlock(&zcs->jobCompleted_mutex); return 0; } /* nothing ready to be flushed => skip */
|
||||
pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex); /* block when nothing available to flush */
|
||||
if (!blockToFlush) { ZSTD_pthread_mutex_unlock(&zcs->jobCompleted_mutex); return 0; } /* nothing ready to be flushed => skip */
|
||||
ZSTD_pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex); /* block when nothing available to flush */
|
||||
}
|
||||
pthread_mutex_unlock(&zcs->jobCompleted_mutex);
|
||||
ZSTD_pthread_mutex_unlock(&zcs->jobCompleted_mutex);
|
||||
/* compression job completed : output can be flushed */
|
||||
{ ZSTDMT_jobDescription job = zcs->jobs[wJobID];
|
||||
if (!job.jobScanned) {
|
||||
|
||||
@ -35,7 +35,7 @@
|
||||
* Frames requiring more memory will be rejected.
|
||||
*/
|
||||
#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
|
||||
# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U64)1 << ZSTD_WINDOWLOG_MAX) + 1) /* defined within zstd.h */
|
||||
# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_DEFAULTMAX) + 1)
|
||||
#endif
|
||||
|
||||
|
||||
@ -933,7 +933,7 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e l
|
||||
offset = 0;
|
||||
else {
|
||||
ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1);
|
||||
ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 2);
|
||||
ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5);
|
||||
assert(ofBits <= MaxOff);
|
||||
if (MEM_32bits() && longOffsets) {
|
||||
U32 const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN_32-1);
|
||||
@ -1179,7 +1179,7 @@ seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, ZSTD_longOffset_e const long
|
||||
offset = 0;
|
||||
else {
|
||||
ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1);
|
||||
ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 2);
|
||||
ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5);
|
||||
assert(ofBits <= MaxOff);
|
||||
if (MEM_32bits() && longOffsets) {
|
||||
U32 const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN_32-1);
|
||||
|
||||
@ -711,8 +711,8 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover(
|
||||
* compiled with multithreaded support.
|
||||
*/
|
||||
typedef struct COVER_best_s {
|
||||
pthread_mutex_t mutex;
|
||||
pthread_cond_t cond;
|
||||
ZSTD_pthread_mutex_t mutex;
|
||||
ZSTD_pthread_cond_t cond;
|
||||
size_t liveJobs;
|
||||
void *dict;
|
||||
size_t dictSize;
|
||||
@ -725,8 +725,8 @@ typedef struct COVER_best_s {
|
||||
*/
|
||||
static void COVER_best_init(COVER_best_t *best) {
|
||||
if (best==NULL) return; /* compatible with init on NULL */
|
||||
(void)pthread_mutex_init(&best->mutex, NULL);
|
||||
(void)pthread_cond_init(&best->cond, NULL);
|
||||
(void)ZSTD_pthread_mutex_init(&best->mutex, NULL);
|
||||
(void)ZSTD_pthread_cond_init(&best->cond, NULL);
|
||||
best->liveJobs = 0;
|
||||
best->dict = NULL;
|
||||
best->dictSize = 0;
|
||||
@ -741,11 +741,11 @@ static void COVER_best_wait(COVER_best_t *best) {
|
||||
if (!best) {
|
||||
return;
|
||||
}
|
||||
pthread_mutex_lock(&best->mutex);
|
||||
ZSTD_pthread_mutex_lock(&best->mutex);
|
||||
while (best->liveJobs != 0) {
|
||||
pthread_cond_wait(&best->cond, &best->mutex);
|
||||
ZSTD_pthread_cond_wait(&best->cond, &best->mutex);
|
||||
}
|
||||
pthread_mutex_unlock(&best->mutex);
|
||||
ZSTD_pthread_mutex_unlock(&best->mutex);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -759,8 +759,8 @@ static void COVER_best_destroy(COVER_best_t *best) {
|
||||
if (best->dict) {
|
||||
free(best->dict);
|
||||
}
|
||||
pthread_mutex_destroy(&best->mutex);
|
||||
pthread_cond_destroy(&best->cond);
|
||||
ZSTD_pthread_mutex_destroy(&best->mutex);
|
||||
ZSTD_pthread_cond_destroy(&best->cond);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -771,9 +771,9 @@ static void COVER_best_start(COVER_best_t *best) {
|
||||
if (!best) {
|
||||
return;
|
||||
}
|
||||
pthread_mutex_lock(&best->mutex);
|
||||
ZSTD_pthread_mutex_lock(&best->mutex);
|
||||
++best->liveJobs;
|
||||
pthread_mutex_unlock(&best->mutex);
|
||||
ZSTD_pthread_mutex_unlock(&best->mutex);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -789,7 +789,7 @@ static void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
|
||||
}
|
||||
{
|
||||
size_t liveJobs;
|
||||
pthread_mutex_lock(&best->mutex);
|
||||
ZSTD_pthread_mutex_lock(&best->mutex);
|
||||
--best->liveJobs;
|
||||
liveJobs = best->liveJobs;
|
||||
/* If the new dictionary is better */
|
||||
@ -812,9 +812,9 @@ static void COVER_best_finish(COVER_best_t *best, size_t compressedSize,
|
||||
best->parameters = parameters;
|
||||
best->compressedSize = compressedSize;
|
||||
}
|
||||
pthread_mutex_unlock(&best->mutex);
|
||||
ZSTD_pthread_mutex_unlock(&best->mutex);
|
||||
if (liveJobs == 0) {
|
||||
pthread_cond_broadcast(&best->cond);
|
||||
ZSTD_pthread_cond_broadcast(&best->cond);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
10
lib/zstd.h
10
lib/zstd.h
@ -376,8 +376,8 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
|
||||
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
|
||||
#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* v0.7+ */
|
||||
|
||||
#define ZSTD_WINDOWLOG_MAX_32 27
|
||||
#define ZSTD_WINDOWLOG_MAX_64 27
|
||||
#define ZSTD_WINDOWLOG_MAX_32 30
|
||||
#define ZSTD_WINDOWLOG_MAX_64 31
|
||||
#define ZSTD_WINDOWLOG_MAX ((unsigned)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
|
||||
#define ZSTD_WINDOWLOG_MIN 10
|
||||
#define ZSTD_HASHLOG_MAX MIN(ZSTD_WINDOWLOG_MAX, 30)
|
||||
@ -964,7 +964,9 @@ typedef enum {
|
||||
* Special: value 0 means "do not change cLevel". */
|
||||
ZSTD_p_windowLog, /* Maximum allowed back-reference distance, expressed as power of 2.
|
||||
* Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
|
||||
* Special: value 0 means "do not change windowLog". */
|
||||
* Special: value 0 means "do not change windowLog".
|
||||
* Note: Using a window size greater than ZSTD_MAXWINDOWSIZE_DEFAULT (default: 2^27)
|
||||
* requires setting the maximum window size at least as large during decompression. */
|
||||
ZSTD_p_hashLog, /* Size of the probe table, as a power of 2.
|
||||
* Resulting table size is (1 << (hashLog+2)).
|
||||
* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
|
||||
@ -1032,7 +1034,7 @@ typedef enum {
|
||||
* Larger values increase memory usage and compression ratio, but decrease
|
||||
* compression speed.
|
||||
* Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
|
||||
* (default: 20). */
|
||||
* (default: windowlog - 7). */
|
||||
ZSTD_p_ldmMinMatch, /* Minimum size of searched matches for long distance matcher.
|
||||
* Larger/too small values usually decrease compression ratio.
|
||||
* Must be clamped between ZSTD_LDM_MINMATCH_MIN
|
||||
|
||||
@ -37,6 +37,7 @@
|
||||
# include <io.h>
|
||||
#endif
|
||||
|
||||
#include "bitstream.h"
|
||||
#include "mem.h"
|
||||
#include "fileio.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
|
||||
@ -347,13 +348,16 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName)
|
||||
fileHandle = fopen(fileName, "rb");
|
||||
if (fileHandle==0) EXM_THROW(31, "%s: %s", fileName, strerror(errno));
|
||||
fileSize = UTIL_getFileSize(fileName);
|
||||
if (fileSize > DICTSIZE_MAX)
|
||||
if (fileSize > DICTSIZE_MAX) {
|
||||
EXM_THROW(32, "Dictionary file %s is too large (> %u MB)",
|
||||
fileName, DICTSIZE_MAX >> 20); /* avoid extreme cases */
|
||||
}
|
||||
*bufferPtr = malloc((size_t)fileSize);
|
||||
if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno));
|
||||
{ size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
|
||||
if (readSize!=fileSize) EXM_THROW(35, "Error reading dictionary file %s", fileName); }
|
||||
{ size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
|
||||
if (readSize!=fileSize)
|
||||
EXM_THROW(35, "Error reading dictionary file %s", fileName);
|
||||
}
|
||||
fclose(fileHandle);
|
||||
return (size_t)fileSize;
|
||||
}
|
||||
@ -970,7 +974,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
|
||||
char* dstFileName = (char*)malloc(FNSPACE);
|
||||
size_t const suffixSize = suffix ? strlen(suffix) : 0;
|
||||
U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ;
|
||||
int const isRegularFile = (nbFiles != 1) ? 0 : UTIL_isRegularFile(inFileNamesTable[0]);
|
||||
int const isRegularFile = (nbFiles > 1) ? 0 : UTIL_isRegularFile(inFileNamesTable[0]); /* won't write frame content size when nbFiles > 1 */
|
||||
cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, isRegularFile, comprParams);
|
||||
|
||||
/* init */
|
||||
@ -1167,6 +1171,35 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void FIO_zstdErrorHelp(dRess_t* ress, size_t ret, char const* srcFileName)
|
||||
{
|
||||
ZSTD_frameHeader header;
|
||||
/* No special help for these errors */
|
||||
if (ZSTD_getErrorCode(ret) != ZSTD_error_frameParameter_windowTooLarge)
|
||||
return;
|
||||
/* Try to decode the frame header */
|
||||
ret = ZSTD_getFrameHeader(&header, ress->srcBuffer, ress->srcBufferLoaded);
|
||||
if (ret == 0) {
|
||||
U32 const windowSize = (U32)header.windowSize;
|
||||
U32 const windowLog = BIT_highbit32(windowSize) + ((windowSize & (windowSize - 1)) != 0);
|
||||
U32 const windowMB = (windowSize >> 20) + (windowSize & ((1 MB) - 1));
|
||||
assert(header.windowSize <= (U64)((U32)-1));
|
||||
assert(g_memLimit > 0);
|
||||
DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u\n",
|
||||
srcFileName, header.windowSize, g_memLimit);
|
||||
if (windowLog <= ZSTD_WINDOWLOG_MAX) {
|
||||
DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB\n",
|
||||
srcFileName, windowLog, windowMB);
|
||||
return;
|
||||
}
|
||||
} else if (ZSTD_getErrorCode(ret) != ZSTD_error_frameParameter_windowTooLarge) {
|
||||
DISPLAYLEVEL(1, "%s : Error decoding frame header to read window size : %s\n",
|
||||
srcFileName, ZSTD_getErrorName(ret));
|
||||
return;
|
||||
}
|
||||
DISPLAYLEVEL(1, "%s : Window log larger than ZSTD_WINDOWLOG_MAX=%u not supported\n",
|
||||
srcFileName, ZSTD_WINDOWLOG_MAX);
|
||||
}
|
||||
|
||||
/** FIO_decompressFrame() :
|
||||
* @return : size of decoded zstd frame, or an error code
|
||||
@ -1183,8 +1216,8 @@ unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
|
||||
ZSTD_resetDStream(ress->dctx);
|
||||
if (strlen(srcFileName)>20) srcFileName += strlen(srcFileName)-20; /* display last 20 characters */
|
||||
|
||||
/* Header loading (optional, saves one loop) */
|
||||
{ size_t const toRead = 9;
|
||||
/* Header loading : ensures ZSTD_getFrameHeader() will succeed */
|
||||
{ size_t const toRead = ZSTD_FRAMEHEADERSIZE_MAX;
|
||||
if (ress->srcBufferLoaded < toRead)
|
||||
ress->srcBufferLoaded += fread(((char*)ress->srcBuffer) + ress->srcBufferLoaded, 1, toRead - ress->srcBufferLoaded, finput);
|
||||
}
|
||||
@ -1197,6 +1230,7 @@ unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
|
||||
if (ZSTD_isError(readSizeHint)) {
|
||||
DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n",
|
||||
srcFileName, ZSTD_getErrorName(readSizeHint));
|
||||
FIO_zstdErrorHelp(ress, readSizeHint, srcFileName);
|
||||
return FIO_ERROR_FRAME_DECODING;
|
||||
}
|
||||
|
||||
@ -1700,10 +1734,11 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
|
||||
typedef struct {
|
||||
int numActualFrames;
|
||||
int numSkippableFrames;
|
||||
unsigned long long decompressedSize;
|
||||
U64 decompressedSize;
|
||||
int decompUnavailable;
|
||||
unsigned long long compressedSize;
|
||||
U64 compressedSize;
|
||||
int usesCheck;
|
||||
U32 nbFiles;
|
||||
} fileInfo_t;
|
||||
|
||||
/** getFileInfo() :
|
||||
@ -1720,14 +1755,16 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){
|
||||
DISPLAY("Error: could not open source file %s\n", inFileName);
|
||||
return 3;
|
||||
}
|
||||
info->compressedSize = (unsigned long long)UTIL_getFileSize(inFileName);
|
||||
info->compressedSize = UTIL_getFileSize(inFileName);
|
||||
|
||||
/* begin analyzing frame */
|
||||
for ( ; ; ) {
|
||||
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
|
||||
size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile);
|
||||
if (numBytesRead < ZSTD_frameHeaderSize_min) {
|
||||
if (feof(srcFile) && numBytesRead == 0 && info->compressedSize > 0) {
|
||||
if ( feof(srcFile)
|
||||
&& (numBytesRead == 0)
|
||||
&& (info->compressedSize > 0) ) {
|
||||
break;
|
||||
}
|
||||
else if (feof(srcFile)) {
|
||||
@ -1829,6 +1866,7 @@ static int getFileInfo(fileInfo_t* info, const char* inFileName){
|
||||
}
|
||||
} /* end analyzing frame */
|
||||
fclose(srcFile);
|
||||
info->nbFiles = 1;
|
||||
return detectError;
|
||||
}
|
||||
|
||||
@ -1841,25 +1879,28 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev
|
||||
const char* const checkString = (info->usesCheck ? "XXH64" : "None");
|
||||
if (displayLevel <= 2) {
|
||||
if (!info->decompUnavailable) {
|
||||
DISPLAYOUT("Skippable Non-Skippable Compressed Uncompressed Ratio Check Filename\n");
|
||||
DISPLAYOUT("%9d %13d %7.2f %2s %9.2f %2s %5.3f %5s %s\n",
|
||||
info->numSkippableFrames, info->numActualFrames,
|
||||
DISPLAYOUT("%6d %5d %7.2f %2s %9.2f %2s %5.3f %5s %s\n",
|
||||
info->numSkippableFrames + info->numActualFrames,
|
||||
info->numSkippableFrames,
|
||||
compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
|
||||
ratio, checkString, inFileName);
|
||||
} else {
|
||||
DISPLAYOUT("Skippable Non-Skippable Compressed Check Filename\n");
|
||||
DISPLAYOUT("%9d %13d %7.2f MB %5s %s\n",
|
||||
info->numSkippableFrames, info->numActualFrames,
|
||||
compressedSizeUnit, checkString, inFileName);
|
||||
DISPLAYOUT("%6d %5d %7.2f %2s %5s %s\n",
|
||||
info->numSkippableFrames + info->numActualFrames,
|
||||
info->numSkippableFrames,
|
||||
compressedSizeUnit, unitStr,
|
||||
checkString, inFileName);
|
||||
}
|
||||
} else {
|
||||
DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames);
|
||||
DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames);
|
||||
DISPLAYOUT("Compressed Size: %.2f %2s (%llu B)\n",
|
||||
compressedSizeUnit, unitStr, info->compressedSize);
|
||||
compressedSizeUnit, unitStr,
|
||||
(unsigned long long)info->compressedSize);
|
||||
if (!info->decompUnavailable) {
|
||||
DISPLAYOUT("Decompressed Size: %.2f %2s (%llu B)\n",
|
||||
decompressedSizeUnit, unitStr, info->decompressedSize);
|
||||
decompressedSizeUnit, unitStr,
|
||||
(unsigned long long)info->decompressedSize);
|
||||
DISPLAYOUT("Ratio: %.4f\n", ratio);
|
||||
}
|
||||
DISPLAYOUT("Check: %s\n", checkString);
|
||||
@ -1867,33 +1908,40 @@ static void displayInfo(const char* inFileName, fileInfo_t* info, int displayLev
|
||||
}
|
||||
}
|
||||
|
||||
static fileInfo_t FIO_addFInfo(fileInfo_t fi1, fileInfo_t fi2)
|
||||
{
|
||||
fileInfo_t total;
|
||||
total.numActualFrames = fi1.numActualFrames + fi2.numActualFrames;
|
||||
total.numSkippableFrames = fi1.numSkippableFrames + fi2.numSkippableFrames;
|
||||
total.compressedSize = fi1.compressedSize + fi2.compressedSize;
|
||||
total.decompressedSize = fi1.decompressedSize + fi2.decompressedSize;
|
||||
total.decompUnavailable = fi1.decompUnavailable | fi2.decompUnavailable;
|
||||
total.usesCheck = fi1.usesCheck & fi2.usesCheck;
|
||||
total.nbFiles = fi1.nbFiles + fi2.nbFiles;
|
||||
return total;
|
||||
}
|
||||
|
||||
static int FIO_listFile(const char* inFileName, int displayLevel, unsigned fileNo, unsigned numFiles){
|
||||
static int FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel){
|
||||
/* initialize info to avoid warnings */
|
||||
fileInfo_t info;
|
||||
memset(&info, 0, sizeof(info));
|
||||
DISPLAYOUT("%s (%u/%u):\n", inFileName, fileNo, numFiles);
|
||||
{
|
||||
int const error = getFileInfo(&info, inFileName);
|
||||
{ int const error = getFileInfo(&info, inFileName);
|
||||
if (error == 1) {
|
||||
/* display error, but provide output */
|
||||
DISPLAY("An error occurred with getting file info\n");
|
||||
DISPLAY("An error occurred while getting file info \n");
|
||||
}
|
||||
else if (error == 2) {
|
||||
DISPLAYOUT("File %s not compressed with zstd\n", inFileName);
|
||||
if (displayLevel > 2) {
|
||||
DISPLAYOUT("\n");
|
||||
}
|
||||
DISPLAYOUT("File %s not compressed by zstd \n", inFileName);
|
||||
if (displayLevel > 2) DISPLAYOUT("\n");
|
||||
return 1;
|
||||
}
|
||||
else if (error == 3) {
|
||||
/* error occurred with opening the file */
|
||||
if (displayLevel > 2) {
|
||||
DISPLAYOUT("\n");
|
||||
}
|
||||
/* error occurred while opening the file */
|
||||
if (displayLevel > 2) DISPLAYOUT("\n");
|
||||
return 1;
|
||||
}
|
||||
displayInfo(inFileName, &info, displayLevel);
|
||||
*total = FIO_addFInfo(*total, info);
|
||||
return error;
|
||||
}
|
||||
}
|
||||
@ -1903,15 +1951,36 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
|
||||
DISPLAYOUT("No files given\n");
|
||||
return 0;
|
||||
}
|
||||
DISPLAYOUT("===========================================\n");
|
||||
DISPLAYOUT("Printing information about compressed files\n");
|
||||
DISPLAYOUT("===========================================\n");
|
||||
DISPLAYOUT("Number of files listed: %u\n", numFiles);
|
||||
{
|
||||
int error = 0;
|
||||
DISPLAYOUT("Frames Skips Compressed Uncompressed Ratio Check Filename\n");
|
||||
{ int error = 0;
|
||||
unsigned u;
|
||||
fileInfo_t total;
|
||||
memset(&total, 0, sizeof(total));
|
||||
total.usesCheck = 1;
|
||||
for (u=0; u<numFiles;u++) {
|
||||
error |= FIO_listFile(filenameTable[u], displayLevel, u+1, numFiles);
|
||||
error |= FIO_listFile(&total, filenameTable[u], displayLevel);
|
||||
}
|
||||
if (numFiles > 1) {
|
||||
unsigned const unit = total.compressedSize < (1 MB) ? (1 KB) : (1 MB);
|
||||
const char* const unitStr = total.compressedSize < (1 MB) ? "KB" : "MB";
|
||||
double const compressedSizeUnit = (double)total.compressedSize / unit;
|
||||
double const decompressedSizeUnit = (double)total.decompressedSize / unit;
|
||||
double const ratio = (total.compressedSize == 0) ? 0 : ((double)total.decompressedSize)/total.compressedSize;
|
||||
const char* const checkString = (total.usesCheck ? "XXH64" : "");
|
||||
DISPLAYOUT("----------------------------------------------------------------- \n");
|
||||
if (total.decompUnavailable) {
|
||||
DISPLAYOUT("%6d %5d %7.2f %2s %5s %u files\n",
|
||||
total.numSkippableFrames + total.numActualFrames,
|
||||
total.numSkippableFrames,
|
||||
compressedSizeUnit, unitStr,
|
||||
checkString, total.nbFiles);
|
||||
} else {
|
||||
DISPLAYOUT("%6d %5d %7.2f %2s %9.2f %2s %5.3f %5s %u files\n",
|
||||
total.numSkippableFrames + total.numActualFrames,
|
||||
total.numSkippableFrames,
|
||||
compressedSizeUnit, unitStr, decompressedSizeUnit, unitStr,
|
||||
ratio, checkString, total.nbFiles);
|
||||
}
|
||||
}
|
||||
return error;
|
||||
}
|
||||
|
||||
@ -104,8 +104,11 @@ Display information related to a zstd compressed file, such as size, ratio, and
|
||||
unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\.
|
||||
.
|
||||
.TP
|
||||
\fB\-\-long\fR
|
||||
enables long distance matching\. This increases the window size (\fBwindowLog\fR) and memory usage for both the compressor and decompressor\. This setting is designed to improve the compression ratio for files with long matches at a large distance (up to the maximum window size, 128 MiB)\.
|
||||
\fB\-\-long[=#]\fR
|
||||
enables long distance matching with \fB#\fR \fBwindowLog\fR, if not \fB#\fR is not present it defaults to \fB27\fR\. This increases the window size (\fBwindowLog\fR) and memory usage for both the compressor and decompressor\. This setting is designed to improve the compression ratio for files with long matches at a large distance\.
|
||||
.
|
||||
.IP
|
||||
Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \fB\-\-memory=windowSize\fR needs to be passed to the decompressor\.
|
||||
.
|
||||
.TP
|
||||
\fB\-T#\fR, \fB\-\-threads=#\fR
|
||||
@ -190,6 +193,10 @@ Dictionary saved into \fBfile\fR (default name: dictionary)\.
|
||||
Limit dictionary to specified size (default: 112640)\.
|
||||
.
|
||||
.TP
|
||||
\fB\-B#\fR
|
||||
Split input files in blocks of size # (default: no split)
|
||||
.
|
||||
.TP
|
||||
\fB\-\-dictID=#\fR
|
||||
A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary\. By default, zstd will create a 4\-bytes random number ID\. It\'s possible to give a precise number instead\. Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header, and an ID < 65536 will only need 2 bytes\. This compares favorably to 4 bytes default\. However, it\'s up to the dictionary manager to not assign twice the same ID to 2 different dictionaries\.
|
||||
.
|
||||
@ -267,7 +274,10 @@ There are 8 strategies numbered from 1 to 8, from faster to stronger: 1=ZSTD_fas
|
||||
Specify the maximum number of bits for a match distance\.
|
||||
.
|
||||
.IP
|
||||
The higher number of increases the chance to find a match which usually improves compression ratio\. It also increases memory requirements for the compressor and decompressor\. The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 27 (128 MiB)\.
|
||||
The higher number of increases the chance to find a match which usually improves compression ratio\. It also increases memory requirements for the compressor and decompressor\. The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 30 (1 GiB) on 32\-bit platforms and 31 (2 GiB) on 64\-bit platforms\.
|
||||
.
|
||||
.IP
|
||||
Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \fB\-\-memory=windowSize\fR needs to be passed to the decompressor\.
|
||||
.
|
||||
.TP
|
||||
\fBhashLog\fR=\fIhlog\fR, \fBhlog\fR=\fIhlog\fR
|
||||
@ -340,7 +350,7 @@ Bigger hash tables usually improve compression ratio at the expense of more memo
|
||||
The minimum \fIldmhlog\fR is 6 and the maximum is 26 (default: 20)\.
|
||||
.
|
||||
.TP
|
||||
\fBldmSearchLength\fR=\fIldmslen\fR, \fBldmSlen\fR=\fIldmslen\fR
|
||||
\fBldmSearchLength\fR=\fIldmslen\fR, \fBldmslen\fR=\fIldmslen\fR
|
||||
Specify the minimum searched length of a match for long distance matching\.
|
||||
.
|
||||
.IP
|
||||
|
||||
@ -105,12 +105,16 @@ the last one takes effect.
|
||||
* `--ultra`:
|
||||
unlocks high compression levels 20+ (maximum 22), using a lot more memory.
|
||||
Note that decompression will also require more memory when using these levels.
|
||||
* `--long`:
|
||||
enables long distance matching.
|
||||
* `--long[=#]`:
|
||||
enables long distance matching with `#` `windowLog`, if not `#` is not
|
||||
present it defaults to `27`.
|
||||
This increases the window size (`windowLog`) and memory usage for both the
|
||||
compressor and decompressor. This setting is designed to improve the
|
||||
compression ratio for files with long matches at a large distance
|
||||
(up to the maximum window size, 128 MiB).
|
||||
compressor and decompressor.
|
||||
This setting is designed to improve the compression ratio for files with
|
||||
long matches at a large distance.
|
||||
|
||||
Note: If `windowLog` is set to larger than 27, `--long=windowLog` or
|
||||
`--memory=windowSize` needs to be passed to the decompressor.
|
||||
* `-T#`, `--threads=#`:
|
||||
Compress using `#` threads (default: 1).
|
||||
If `#` is 0, attempt to detect and use the number of physical CPU cores.
|
||||
@ -275,7 +279,11 @@ The list of available _options_:
|
||||
The higher number of increases the chance to find a match which usually
|
||||
improves compression ratio.
|
||||
It also increases memory requirements for the compressor and decompressor.
|
||||
The minimum _wlog_ is 10 (1 KiB) and the maximum is 27 (128 MiB).
|
||||
The minimum _wlog_ is 10 (1 KiB) and the maximum is 30 (1 GiB) on 32-bit
|
||||
platforms and 31 (2 GiB) on 64-bit platforms.
|
||||
|
||||
Note: If `windowLog` is set to larger than 27, `--long=windowLog` or
|
||||
`--memory=windowSize` needs to be passed to the decompressor.
|
||||
|
||||
- `hashLog`=_hlog_, `hlog`=_hlog_:
|
||||
Specify the maximum number of bits for a hash table.
|
||||
|
||||
@ -72,6 +72,7 @@ static const char* g_defaultDictName = "dictionary";
|
||||
static const unsigned g_defaultMaxDictSize = 110 KB;
|
||||
static const int g_defaultDictCLevel = 3;
|
||||
static const unsigned g_defaultSelectivityLevel = 9;
|
||||
static const unsigned g_defaultMaxWindowLog = 27;
|
||||
#define OVERLAP_LOG_DEFAULT 9999
|
||||
#define LDM_PARAM_DEFAULT 9999 /* Default for parameters where 0 is valid */
|
||||
static U32 g_overlapLog = OVERLAP_LOG_DEFAULT;
|
||||
@ -129,7 +130,7 @@ static int usage_advanced(const char* programName)
|
||||
DISPLAY( " -l : print information about zstd compressed files \n");
|
||||
#ifndef ZSTD_NOCOMPRESS
|
||||
DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
|
||||
DISPLAY( "--long : enable long distance matching (requires more memory)\n");
|
||||
DISPLAY( "--long[=#] : enable long distance matching with given window log (default : %u)\n", g_defaultMaxWindowLog);
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
DISPLAY( " -T# : use # threads for compression (default:1) \n");
|
||||
DISPLAY( " -B# : select size of each job (default:0==automatic) \n");
|
||||
@ -459,7 +460,6 @@ int main(int argCount, const char* argv[])
|
||||
if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; }
|
||||
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; g_displayLevel-=(g_displayLevel==2); continue; }
|
||||
if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
|
||||
if (!strcmp(argument, "--long")) { ldmFlag = 1; continue; }
|
||||
if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
|
||||
if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
|
||||
if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
|
||||
@ -514,6 +514,22 @@ int main(int argCount, const char* argv[])
|
||||
if (longCommandWArg(&argument, "--maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--dictID=")) { dictID = readU32FromChar(&argument); continue; }
|
||||
if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) CLEAN_RETURN(badusage(programName)); continue; }
|
||||
if (longCommandWArg(&argument, "--long")) {
|
||||
unsigned ldmWindowLog = 0;
|
||||
ldmFlag = 1;
|
||||
/* Parse optional window log */
|
||||
if (*argument == '=') {
|
||||
++argument;
|
||||
ldmWindowLog = readU32FromChar(&argument);
|
||||
} else if (*argument != 0) {
|
||||
/* Invalid character following --long */
|
||||
CLEAN_RETURN(badusage(programName));
|
||||
}
|
||||
/* Only set windowLog if not already set by --zstd */
|
||||
if (compressionParams.windowLog == 0)
|
||||
compressionParams.windowLog = ldmWindowLog;
|
||||
continue;
|
||||
}
|
||||
/* fall-through, will trigger bad_usage() later on */
|
||||
}
|
||||
|
||||
@ -830,6 +846,13 @@ int main(int argCount, const char* argv[])
|
||||
#endif
|
||||
} else { /* decompression or test */
|
||||
#ifndef ZSTD_NODECOMPRESS
|
||||
if (memLimit == 0) {
|
||||
if (compressionParams.windowLog == 0)
|
||||
memLimit = (U32)1 << g_defaultMaxWindowLog;
|
||||
else {
|
||||
memLimit = (U32)1 << (compressionParams.windowLog & 31);
|
||||
}
|
||||
}
|
||||
FIO_setMemLimit(memLimit);
|
||||
if (filenameIdx==1 && outFileName)
|
||||
operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
|
||||
|
||||
@ -34,27 +34,20 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
|
||||
void *compressed, size_t compressedCapacity,
|
||||
const void *src, size_t srcSize)
|
||||
{
|
||||
int const cLevel = FUZZ_rand(&seed) % kMaxClevel;
|
||||
size_t ret = ZSTD_compressBegin(cctx, cLevel);
|
||||
int const cLevel = FUZZ_rand(&seed) % kMaxClevel;
|
||||
ZSTD_parameters const params = ZSTD_getParams(cLevel, srcSize, 0);
|
||||
size_t ret = ZSTD_compressBegin_advanced(cctx, NULL, 0, params, srcSize);
|
||||
FUZZ_ZASSERT(ret);
|
||||
|
||||
if (ZSTD_isError(ret)) {
|
||||
fprintf(stderr, "ZSTD_compressBegin() error: %s\n",
|
||||
ZSTD_getErrorName(ret));
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = ZSTD_compressBlock(cctx, compressed, compressedCapacity, src, srcSize);
|
||||
if (ZSTD_isError(ret)) {
|
||||
fprintf(stderr, "ZSTD_compressBlock() error: %s\n", ZSTD_getErrorName(ret));
|
||||
return ret;
|
||||
}
|
||||
if (ret == 0) {
|
||||
ret = ZSTD_compressBlock(cctx, compressed, compressedCapacity, src, srcSize);
|
||||
FUZZ_ZASSERT(ret);
|
||||
if (ret == 0) {
|
||||
FUZZ_ASSERT(resultCapacity >= srcSize);
|
||||
memcpy(result, src, srcSize);
|
||||
return srcSize;
|
||||
}
|
||||
ZSTD_decompressBegin(dctx);
|
||||
return ZSTD_decompressBlock(dctx, result, resultCapacity, compressed, ret);
|
||||
}
|
||||
ZSTD_decompressBegin(dctx);
|
||||
return ZSTD_decompressBlock(dctx, result, resultCapacity, compressed, ret);
|
||||
}
|
||||
|
||||
int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
|
||||
@ -87,7 +80,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
|
||||
{
|
||||
size_t const result =
|
||||
roundTripTest(rBuf, neededBufSize, cBuf, neededBufSize, src, size);
|
||||
FUZZ_ASSERT_MSG(!ZSTD_isError(result), ZSTD_getErrorName(result));
|
||||
FUZZ_ZASSERT(result);
|
||||
FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size");
|
||||
FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!");
|
||||
}
|
||||
|
||||
@ -41,21 +41,17 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
|
||||
size_t err;
|
||||
|
||||
ZSTD_CCtx_reset(cctx);
|
||||
FUZZ_setRandomParameters(cctx, &seed);
|
||||
FUZZ_setRandomParameters(cctx, srcSize, &seed);
|
||||
err = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
|
||||
if (err != 0) {
|
||||
return err;
|
||||
}
|
||||
FUZZ_ZASSERT(err);
|
||||
FUZZ_ASSERT(err == 0);
|
||||
cSize = out.pos;
|
||||
} else {
|
||||
int const cLevel = FUZZ_rand(&seed) % kMaxClevel;
|
||||
cSize = ZSTD_compressCCtx(
|
||||
cctx, compressed, compressedCapacity, src, srcSize, cLevel);
|
||||
}
|
||||
if (ZSTD_isError(cSize)) {
|
||||
fprintf(stderr, "Compression error: %s\n", ZSTD_getErrorName(cSize));
|
||||
return cSize;
|
||||
}
|
||||
FUZZ_ZASSERT(cSize);
|
||||
return ZSTD_decompressDCtx(dctx, result, resultCapacity, compressed, cSize);
|
||||
}
|
||||
|
||||
@ -87,7 +83,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
|
||||
{
|
||||
size_t const result =
|
||||
roundTripTest(rBuf, neededBufSize, cBuf, neededBufSize, src, size);
|
||||
FUZZ_ASSERT_MSG(!ZSTD_isError(result), ZSTD_getErrorName(result));
|
||||
FUZZ_ZASSERT(result);
|
||||
FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size");
|
||||
FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!");
|
||||
}
|
||||
|
||||
@ -57,7 +57,7 @@ static size_t compress(uint8_t *dst, size_t capacity,
|
||||
{
|
||||
size_t dstSize = 0;
|
||||
ZSTD_CCtx_reset(cctx);
|
||||
FUZZ_setRandomParameters(cctx, &seed);
|
||||
FUZZ_setRandomParameters(cctx, srcSize, &seed);
|
||||
|
||||
while (srcSize > 0) {
|
||||
ZSTD_inBuffer in = makeInBuffer(&src, &srcSize);
|
||||
@ -85,7 +85,10 @@ static size_t compress(uint8_t *dst, size_t capacity,
|
||||
/* Reset the compressor when the frame is finished */
|
||||
if (ret == 0) {
|
||||
ZSTD_CCtx_reset(cctx);
|
||||
FUZZ_setRandomParameters(cctx, &seed);
|
||||
if ((FUZZ_rand(&seed) & 7) == 0) {
|
||||
size_t const remaining = in.size - in.pos;
|
||||
FUZZ_setRandomParameters(cctx, remaining, &seed);
|
||||
}
|
||||
mode = -1;
|
||||
}
|
||||
break;
|
||||
@ -146,7 +149,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
|
||||
size_t const cSize = compress(cBuf, neededBufSize, src, size);
|
||||
size_t const rSize =
|
||||
ZSTD_decompressDCtx(dctx, rBuf, neededBufSize, cBuf, cSize);
|
||||
FUZZ_ASSERT_MSG(!ZSTD_isError(rSize), ZSTD_getErrorName(rSize));
|
||||
FUZZ_ZASSERT(rSize);
|
||||
FUZZ_ASSERT_MSG(rSize == size, "Incorrect regenerated size");
|
||||
FUZZ_ASSERT_MSG(!memcmp(src, rBuf, size), "Corruption!");
|
||||
}
|
||||
|
||||
@ -13,30 +13,68 @@
|
||||
#include "fuzz_helpers.h"
|
||||
#include "zstd.h"
|
||||
|
||||
static void setRand(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned min,
|
||||
unsigned max, uint32_t *state) {
|
||||
unsigned const value = FUZZ_rand32(state, min, max);
|
||||
FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value));
|
||||
static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned value)
|
||||
{
|
||||
FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value));
|
||||
}
|
||||
|
||||
void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, uint32_t *state)
|
||||
static void setRand(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned min,
|
||||
unsigned max, uint32_t *state) {
|
||||
unsigned const value = FUZZ_rand32(state, min, max);
|
||||
set(cctx, param, value);
|
||||
}
|
||||
|
||||
ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state)
|
||||
{
|
||||
setRand(cctx, ZSTD_p_windowLog, ZSTD_WINDOWLOG_MIN, 23, state);
|
||||
setRand(cctx, ZSTD_p_hashLog, ZSTD_HASHLOG_MIN, 23, state);
|
||||
setRand(cctx, ZSTD_p_chainLog, ZSTD_CHAINLOG_MIN, 24, state);
|
||||
setRand(cctx, ZSTD_p_searchLog, ZSTD_SEARCHLOG_MIN, 9, state);
|
||||
setRand(cctx, ZSTD_p_minMatch, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX,
|
||||
state);
|
||||
setRand(cctx, ZSTD_p_targetLength, ZSTD_TARGETLENGTH_MIN,
|
||||
ZSTD_TARGETLENGTH_MAX, state);
|
||||
setRand(cctx, ZSTD_p_compressionStrategy, ZSTD_fast, ZSTD_btultra, state);
|
||||
/* Select compression parameters */
|
||||
ZSTD_compressionParameters cParams;
|
||||
cParams.windowLog = FUZZ_rand32(state, ZSTD_WINDOWLOG_MIN, 15);
|
||||
cParams.hashLog = FUZZ_rand32(state, ZSTD_HASHLOG_MIN, 15);
|
||||
cParams.chainLog = FUZZ_rand32(state, ZSTD_CHAINLOG_MIN, 16);
|
||||
cParams.searchLog = FUZZ_rand32(state, ZSTD_SEARCHLOG_MIN, 9);
|
||||
cParams.searchLength = FUZZ_rand32(state, ZSTD_SEARCHLENGTH_MIN,
|
||||
ZSTD_SEARCHLENGTH_MAX);
|
||||
cParams.targetLength = FUZZ_rand32(state, ZSTD_TARGETLENGTH_MIN,
|
||||
ZSTD_TARGETLENGTH_MAX);
|
||||
cParams.strategy = FUZZ_rand32(state, ZSTD_fast, ZSTD_btultra);
|
||||
return ZSTD_adjustCParams(cParams, srcSize, 0);
|
||||
}
|
||||
|
||||
ZSTD_frameParameters FUZZ_randomFParams(uint32_t *state)
|
||||
{
|
||||
/* Select frame parameters */
|
||||
ZSTD_frameParameters fParams;
|
||||
fParams.contentSizeFlag = FUZZ_rand32(state, 0, 1);
|
||||
fParams.checksumFlag = FUZZ_rand32(state, 0, 1);
|
||||
fParams.noDictIDFlag = FUZZ_rand32(state, 0, 1);
|
||||
return fParams;
|
||||
}
|
||||
|
||||
ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state)
|
||||
{
|
||||
ZSTD_parameters params;
|
||||
params.cParams = FUZZ_randomCParams(srcSize, state);
|
||||
params.fParams = FUZZ_randomFParams(state);
|
||||
return params;
|
||||
}
|
||||
|
||||
void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state)
|
||||
{
|
||||
ZSTD_compressionParameters cParams = FUZZ_randomCParams(srcSize, state);
|
||||
set(cctx, ZSTD_p_windowLog, cParams.windowLog);
|
||||
set(cctx, ZSTD_p_hashLog, cParams.hashLog);
|
||||
set(cctx, ZSTD_p_chainLog, cParams.chainLog);
|
||||
set(cctx, ZSTD_p_searchLog, cParams.searchLog);
|
||||
set(cctx, ZSTD_p_minMatch, cParams.searchLength);
|
||||
set(cctx, ZSTD_p_targetLength, cParams.targetLength);
|
||||
set(cctx, ZSTD_p_compressionStrategy, cParams.strategy);
|
||||
/* Select frame parameters */
|
||||
setRand(cctx, ZSTD_p_contentSizeFlag, 0, 1, state);
|
||||
setRand(cctx, ZSTD_p_checksumFlag, 0, 1, state);
|
||||
setRand(cctx, ZSTD_p_dictIDFlag, 0, 1, state);
|
||||
/* Select long distance matchig parameters */
|
||||
setRand(cctx, ZSTD_p_enableLongDistanceMatching, 0, 1, state);
|
||||
setRand(cctx, ZSTD_p_ldmHashLog, ZSTD_HASHLOG_MIN, 24, state);
|
||||
setRand(cctx, ZSTD_p_ldmHashLog, ZSTD_HASHLOG_MIN, 16, state);
|
||||
setRand(cctx, ZSTD_p_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN,
|
||||
ZSTD_LDM_MINMATCH_MAX, state);
|
||||
setRand(cctx, ZSTD_p_ldmBucketSizeLog, 0, ZSTD_LDM_BUCKETSIZELOG_MAX,
|
||||
|
||||
@ -21,7 +21,11 @@
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, uint32_t *state);
|
||||
void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, uint32_t *state);
|
||||
|
||||
ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, uint32_t *state);
|
||||
ZSTD_frameParameters FUZZ_randomFParams(uint32_t *state);
|
||||
ZSTD_parameters FUZZ_randomParams(size_t srcSize, uint32_t *state);
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
@ -13,11 +13,16 @@ roundTripTest() {
|
||||
cLevel="$2"
|
||||
proba=""
|
||||
fi
|
||||
if [ -n "$4" ]; then
|
||||
dLevel="$4"
|
||||
else
|
||||
dLevel="$cLevel"
|
||||
fi
|
||||
|
||||
rm -f tmp1 tmp2
|
||||
$ECHO "roundTripTest: ./datagen $1 $proba | $ZSTD -v$cLevel | $ZSTD -d"
|
||||
$ECHO "roundTripTest: ./datagen $1 $proba | $ZSTD -v$cLevel | $ZSTD -d$dLevel"
|
||||
./datagen $1 $proba | $MD5SUM > tmp1
|
||||
./datagen $1 $proba | $ZSTD --ultra -v$cLevel | $ZSTD -d | $MD5SUM > tmp2
|
||||
./datagen $1 $proba | $ZSTD --ultra -v$cLevel | $ZSTD -d$dLevel | $MD5SUM > tmp2
|
||||
$DIFF -q tmp1 tmp2
|
||||
}
|
||||
|
||||
@ -29,12 +34,17 @@ fileRoundTripTest() {
|
||||
local_c="$2"
|
||||
local_p=""
|
||||
fi
|
||||
if [ -n "$4" ]; then
|
||||
local_d="$4"
|
||||
else
|
||||
local_d="$local_c"
|
||||
fi
|
||||
|
||||
rm -f tmp.zstd tmp.md5.1 tmp.md5.2
|
||||
$ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c tmp | $ZSTD -d"
|
||||
$ECHO "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c tmp | $ZSTD -d$local_d"
|
||||
./datagen $1 $local_p > tmp
|
||||
cat tmp | $MD5SUM > tmp.md5.1
|
||||
$ZSTD --ultra -v$local_c -c tmp | $ZSTD -d | $MD5SUM > tmp.md5.2
|
||||
$ZSTD --ultra -v$local_c -c tmp | $ZSTD -d$local_d | $MD5SUM > tmp.md5.2
|
||||
$DIFF -q tmp.md5.1 tmp.md5.2
|
||||
}
|
||||
|
||||
@ -673,16 +683,24 @@ roundTripTest -g140000000 -P60 "5 --long"
|
||||
roundTripTest -g70000000 -P70 "8 --long"
|
||||
roundTripTest -g18000001 -P80 "18 --long"
|
||||
fileRoundTripTest -g4100M -P99 "1 --long"
|
||||
# Test large window logs
|
||||
roundTripTest -g4100M -P50 "1 --long=30"
|
||||
roundTripTest -g4100M -P50 "1 --long --zstd=wlog=30,clog=30"
|
||||
# Test parameter parsing
|
||||
roundTripTest -g1M -P50 "1 --long=30" " --memory=1024MB"
|
||||
roundTripTest -g1M -P50 "1 --long=30 --zstd=wlog=29" " --memory=512MB"
|
||||
roundTripTest -g1M -P50 "1 --long=30" " --long=29 --memory=1024MB"
|
||||
roundTripTest -g1M -P50 "1 --long=30" " --zstd=wlog=29 --memory=1024MB"
|
||||
|
||||
|
||||
if [ -n "$hasMT" ]
|
||||
then
|
||||
$ECHO "\n**** zstdmt long round-trip tests **** "
|
||||
roundTripTest -g99000000 -P99 "20 -T2"
|
||||
roundTripTest -g6000000000 -P99 "1 -T2"
|
||||
roundTripTest -g1500000000 -P97 "1 -T999"
|
||||
fileRoundTripTest -g4195M -P98 " -T0"
|
||||
roundTripTest -g1500000000 -P97 "1 --long -T999"
|
||||
roundTripTest -g99000000 -P99 "20 -T2" " "
|
||||
roundTripTest -g6000000000 -P99 "1 -T2" " "
|
||||
roundTripTest -g1500000000 -P97 "1 -T999" " "
|
||||
fileRoundTripTest -g4195M -P98 " -T0" " "
|
||||
roundTripTest -g1500000000 -P97 "1 --long=23 -T2" " "
|
||||
else
|
||||
$ECHO "\n**** no multithreading, skipping zstdmt tests **** "
|
||||
fi
|
||||
|
||||
@ -50,6 +50,7 @@ static const U32 g_cLevelMax_smallTests = 10;
|
||||
#define COMPRESSIBLE_NOISE_LENGTH (10 MB)
|
||||
#define FUZ_COMPRESSIBILITY_DEFAULT 50
|
||||
static const U32 prime32 = 2654435761U;
|
||||
static const U32 windowLogMax = 27;
|
||||
|
||||
|
||||
/*-************************************
|
||||
@ -1380,6 +1381,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
|
||||
|
||||
/* mess with compression parameters */
|
||||
cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
|
||||
cParams.windowLog = MIN(windowLogMax, cParams.windowLog);
|
||||
cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
|
||||
cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
|
||||
cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user