From 89dc856caee4a9c10740b6ee8b20aaa74f1a2b7b Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 24 Aug 2017 16:48:32 -0700 Subject: [PATCH 1/7] [pool] Fix formatting --- lib/common/pool.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/common/pool.c b/lib/common/pool.c index a227044f..0f848901 100644 --- a/lib/common/pool.c +++ b/lib/common/pool.c @@ -25,8 +25,8 @@ /* A job is a function and an opaque argument */ typedef struct POOL_job_s { - POOL_function function; - void *opaque; + POOL_function function; + void *opaque; } POOL_job; struct POOL_ctx_s { @@ -214,13 +214,13 @@ void POOL_add(void* ctxVoid, POOL_function function, void *opaque) { /* We don't need any data, but if it is empty malloc() might return NULL. */ struct POOL_ctx_s { - int data; + int data; }; POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { - (void)numThreads; - (void)queueSize; - return (POOL_ctx*)malloc(sizeof(POOL_ctx)); + (void)numThreads; + (void)queueSize; + return (POOL_ctx*)malloc(sizeof(POOL_ctx)); } void POOL_free(POOL_ctx* ctx) { @@ -228,8 +228,8 @@ void POOL_free(POOL_ctx* ctx) { } void POOL_add(void* ctx, POOL_function function, void* opaque) { - (void)ctx; - function(opaque); + (void)ctx; + function(opaque); } size_t POOL_sizeof(POOL_ctx* ctx) { From 26dc040a7b0b527a65da93020a8f4c15652e17ee Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 24 Aug 2017 17:01:41 -0700 Subject: [PATCH 2/7] [pool] Accept custom allocators --- lib/common/pool.c | 32 ++++++++++++++++++++++---------- lib/common/pool.h | 3 +++ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/lib/common/pool.c b/lib/common/pool.c index 0f848901..ada9b169 100644 --- a/lib/common/pool.c +++ b/lib/common/pool.c @@ -30,6 +30,7 @@ typedef struct POOL_job_s { } POOL_job; struct POOL_ctx_s { + ZSTD_customMem customMem; /* Keep track of the threads */ pthread_t *threads; size_t numThreads; @@ -98,11 +99,15 @@ static void* POOL_thread(void* opaque) { } POOL_ctx *POOL_create(size_t numThreads, size_t queueSize) { + return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); +} + +POOL_ctx *POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) { POOL_ctx *ctx; /* Check the parameters */ if (!numThreads) { return NULL; } /* Allocate the context and zero initialize */ - ctx = (POOL_ctx *)calloc(1, sizeof(POOL_ctx)); + ctx = (POOL_ctx *)ZSTD_calloc(sizeof(POOL_ctx), customMem); if (!ctx) { return NULL; } /* Initialize the job queue. * It needs one extra space since one space is wasted to differentiate empty @@ -119,8 +124,9 @@ POOL_ctx *POOL_create(size_t numThreads, size_t queueSize) { (void)pthread_cond_init(&ctx->queuePopCond, NULL); ctx->shutdown = 0; /* Allocate space for the thread handles */ - ctx->threads = (pthread_t*)malloc(numThreads * sizeof(pthread_t)); + ctx->threads = (pthread_t*)ZSTD_malloc(numThreads * sizeof(pthread_t), customMem); ctx->numThreads = 0; + ctx->customMem = customMem; /* Check for errors */ if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; } /* Initialize the threads */ @@ -160,9 +166,9 @@ void POOL_free(POOL_ctx *ctx) { pthread_mutex_destroy(&ctx->queueMutex); pthread_cond_destroy(&ctx->queuePushCond); pthread_cond_destroy(&ctx->queuePopCond); - if (ctx->queue) free(ctx->queue); - if (ctx->threads) free(ctx->threads); - free(ctx); + ZSTD_free(ctx->queue, ctx->customMem); + ZSTD_free(ctx->threads, ctx->customMem); + ZSTD_free(ctx, ctx->customMem); } size_t POOL_sizeof(POOL_ctx *ctx) { @@ -213,18 +219,23 @@ void POOL_add(void* ctxVoid, POOL_function function, void *opaque) { /* No multi-threading support */ /* We don't need any data, but if it is empty malloc() might return NULL. */ -struct POOL_ctx_s { - int data; -}; +struct POOL_ctx_s {}; +static POOL_ctx g_ctx; POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { + return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); +} + +POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) { (void)numThreads; (void)queueSize; - return (POOL_ctx*)malloc(sizeof(POOL_ctx)); + (void)customMem; + return &g_ctx; } void POOL_free(POOL_ctx* ctx) { - free(ctx); + assert(ctx == &g_ctx); + (void)ctx; } void POOL_add(void* ctx, POOL_function function, void* opaque) { @@ -234,6 +245,7 @@ void POOL_add(void* ctx, POOL_function function, void* opaque) { size_t POOL_sizeof(POOL_ctx* ctx) { if (ctx==NULL) return 0; /* supports sizeof NULL */ + assert(ctx == &g_ctx); return sizeof(*ctx); } diff --git a/lib/common/pool.h b/lib/common/pool.h index 264c5c9c..411b73e1 100644 --- a/lib/common/pool.h +++ b/lib/common/pool.h @@ -16,6 +16,7 @@ extern "C" { #include /* size_t */ +#include "zstd_internal.h" /* ZSTD_customMem */ typedef struct POOL_ctx_s POOL_ctx; @@ -27,6 +28,8 @@ typedef struct POOL_ctx_s POOL_ctx; */ POOL_ctx *POOL_create(size_t numThreads, size_t queueSize); +POOL_ctx *POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem); + /*! POOL_free() : Free a thread pool returned by POOL_create(). */ From de6c6bce859193ac7dc50a96b01c6de4ae9e328b Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 24 Aug 2017 18:09:50 -0700 Subject: [PATCH 3/7] Fix zstd_internal.h for C++ mode --- lib/common/zstd_common.c | 3 +-- lib/common/zstd_internal.h | 8 ++++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c index 08384cab..1155c60c 100644 --- a/lib/common/zstd_common.c +++ b/lib/common/zstd_common.c @@ -15,8 +15,7 @@ #include /* malloc, calloc, free */ #include /* memset */ #include "error_private.h" -#define ZSTD_STATIC_LINKING_ONLY -#include "zstd.h" +#include "zstd_internal.h" /*-**************************************** diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index ac1f3989..fce75723 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -29,6 +29,11 @@ #include "xxhash.h" /* XXH_reset, update, digest */ +#if defined (__cplusplus) +extern "C" { +#endif + + /*-************************************* * Debug ***************************************/ @@ -334,5 +339,8 @@ typedef struct { size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr); +#if defined (__cplusplus) +} +#endif #endif /* ZSTD_CCOMMON_H_MODULE */ From db3f5372dfc636c848f8ad5b74c392272b48b326 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 24 Aug 2017 18:12:28 -0700 Subject: [PATCH 4/7] [zstdmt] Use POOL_create_advanced() --- lib/compress/zstdmt_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 8564bc43..84a56889 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -426,7 +426,7 @@ ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, ZSTD_customMem cMem) mtctx->allJobsCompleted = 1; mtctx->sectionSize = 0; mtctx->overlapLog = ZSTDMT_OVERLAPLOG_DEFAULT; - mtctx->factory = POOL_create(nbThreads, 0); + mtctx->factory = POOL_create_advanced(nbThreads, 0, cMem); mtctx->jobs = ZSTDMT_allocJobsTable(&nbJobs, cMem); mtctx->jobIDMask = nbJobs - 1; mtctx->bufPool = ZSTDMT_createBufferPool(nbThreads, cMem); From 7c365eb02c2eda99fde29a9e0894b667611556b2 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 25 Aug 2017 17:44:32 -0700 Subject: [PATCH 5/7] [threading] Fix ERROR macro after including windows.h --- lib/common/error_private.h | 3 ++- lib/common/threading.h | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/common/error_private.h b/lib/common/error_private.h index 9dd9a87c..f8e68016 100644 --- a/lib/common/error_private.h +++ b/lib/common/error_private.h @@ -51,7 +51,8 @@ typedef ZSTD_ErrorCode ERR_enum; #ifdef ERROR # undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ #endif -#define ERROR(name) ((size_t)-PREFIX(name)) +#define ERROR(name) ZSTD_ERROR(name) +#define ZSTD_ERROR(name) ((size_t)-PREFIX(name)) ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } diff --git a/lib/common/threading.h b/lib/common/threading.h index ab09977a..0a9fcf85 100644 --- a/lib/common/threading.h +++ b/lib/common/threading.h @@ -37,7 +37,15 @@ extern "C" { # define WIN32_LEAN_AND_MEAN #endif +#ifdef ERROR +# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ +#endif #include +#ifdef ERROR +# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ +#endif +#define ERROR(name) ZSTD_ERROR(name) + /* mutex */ #define pthread_mutex_t CRITICAL_SECTION From 02033be08c581ec59d4ef26d46defc0687d15f81 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 28 Aug 2017 17:19:01 -0700 Subject: [PATCH 6/7] [pool] Visual Studios disallows empty structs --- lib/common/pool.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/common/pool.c b/lib/common/pool.c index ada9b169..9567d112 100644 --- a/lib/common/pool.c +++ b/lib/common/pool.c @@ -219,7 +219,9 @@ void POOL_add(void* ctxVoid, POOL_function function, void *opaque) { /* No multi-threading support */ /* We don't need any data, but if it is empty malloc() might return NULL. */ -struct POOL_ctx_s {}; +struct POOL_ctx_s { + int dummy; +}; static POOL_ctx g_ctx; POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { From 9822f97721e472d711f8e28c7304dac379ccdfb6 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Tue, 29 Aug 2017 11:54:38 -0700 Subject: [PATCH 7/7] [error] Don't guard undef X with ifdef X --- lib/common/error_private.h | 4 +--- lib/common/threading.h | 8 ++------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/lib/common/error_private.h b/lib/common/error_private.h index f8e68016..a55bce84 100644 --- a/lib/common/error_private.h +++ b/lib/common/error_private.h @@ -48,9 +48,7 @@ typedef ZSTD_ErrorCode ERR_enum; /*-**************************************** * Error codes handling ******************************************/ -#ifdef ERROR -# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ -#endif +#undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ #define ERROR(name) ZSTD_ERROR(name) #define ZSTD_ERROR(name) ((size_t)-PREFIX(name)) diff --git a/lib/common/threading.h b/lib/common/threading.h index 0a9fcf85..bd4b654c 100644 --- a/lib/common/threading.h +++ b/lib/common/threading.h @@ -37,13 +37,9 @@ extern "C" { # define WIN32_LEAN_AND_MEAN #endif -#ifdef ERROR -# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ -#endif +#undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ #include -#ifdef ERROR -# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ -#endif +#undef ERROR #define ERROR(name) ZSTD_ERROR(name)