From 0d6ecc72a3158b41446970cfcf2f1f1d45b72f41 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 11 Sep 2017 14:09:34 -0700 Subject: [PATCH] makes it possible to compile libzstd in single-thread mode without zstdmt_compress.c (#819) --- NEWS | 1 + circle.yml | 2 +- lib/.gitignore | 1 + lib/Makefile | 11 ++++++++- lib/compress/zstd_compress.c | 43 ++++++++++++++++++++-------------- lib/compress/zstd_compress.h | 6 ++++- lib/compress/zstdmt_compress.h | 16 +++++++++++++ 7 files changed, 60 insertions(+), 20 deletions(-) diff --git a/NEWS b/NEWS index c659e1f4..1300c80d 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,7 @@ license : changed /examples license to BSD + GPLv2 license : fix a few header files to reflect new license (#825) fix : 32-bits build can now decode large offsets (levels 21+) fix : a rare compression bug when compression generates very large distances (only possible at --ultra -22) +build: fix : no-multithread variant compiles without pool.c dependency, reported by Mitchell Blank Jr (@mitchblank) (#819) build: better compatibility with reproducible builds, by Bernhard M. Wiedemann (@bmwiedemann) (#818) v1.3.1 diff --git a/circle.yml b/circle.yml index 8c2bd30d..e89d548a 100644 --- a/circle.yml +++ b/circle.yml @@ -9,7 +9,7 @@ dependencies: test: override: - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then cc -v; make all && make clean; fi && + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then cc -v; make all && make clean && make -C lib libzstd-nomt && make clean; fi && if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu90build && make clean; fi : parallel: true diff --git a/lib/.gitignore b/lib/.gitignore index b43a8543..4cd50ac6 100644 --- a/lib/.gitignore +++ b/lib/.gitignore @@ -1,2 +1,3 @@ # make install artefact libzstd.pc +libzstd-nomt diff --git a/lib/Makefile b/lib/Makefile index 6de1450b..cdfdc5cd 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -101,10 +101,19 @@ lib-release lib-release-mt: DEBUGFLAGS := lib-release: lib lib-release-mt: lib-mt +# Special case : building library in single-thread mode _and_ without zstdmt_compress.c +ZSTDMT_FILES = compress/zstdmt_compress.c +ZSTD_NOMT_FILES = $(filter-out $(ZSTDMT_FILES),$(ZSTD_FILES)) +libzstd-nomt: LDFLAGS += -shared -fPIC -fvisibility=hidden +libzstd-nomt: $(ZSTD_NOMT_FILES) + @echo compiling single-thread dynamic library $(LIBVER) + @echo files : $(ZSTD_NOMT_FILES) + @$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@ + clean: @$(RM) -r *.dSYM # Mac OS-X specific @$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc - @$(RM) dll/libzstd.dll dll/libzstd.lib + @$(RM) dll/libzstd.dll dll/libzstd.lib libzstd-nomt* @$(RM) common/*.o compress/*.o decompress/*.o dictBuilder/*.o legacy/*.o deprecated/*.o @echo Cleaning library completed diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 63dfc36e..616a3d58 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -112,23 +112,37 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) cctx->workSpace = NULL; ZSTD_freeCDict(cctx->cdictLocal); cctx->cdictLocal = NULL; +#ifdef ZSTD_MULTITHREAD ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; +#endif ZSTD_free(cctx, cctx->customMem); return 0; /* reserved as a potential error code in the future */ } + +static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + return ZSTDMT_sizeof_CCtx(cctx->mtctx); +#else + (void) cctx; + return 0; +#endif +} + + size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) { if (cctx==NULL) return 0; /* support sizeof on NULL */ DEBUGLOG(5, "sizeof(*cctx) : %u", (U32)sizeof(*cctx)); DEBUGLOG(5, "workSpaceSize : %u", (U32)cctx->workSpaceSize); DEBUGLOG(5, "streaming buffers : %u", (U32)(cctx->outBuffSize + cctx->inBuffSize)); - DEBUGLOG(5, "inner MTCTX : %u", (U32)ZSTDMT_sizeof_CCtx(cctx->mtctx)); + DEBUGLOG(5, "inner MTCTX : %u", (U32)ZSTD_sizeof_mtctx(cctx)); return sizeof(*cctx) + cctx->workSpaceSize + ZSTD_sizeof_CDict(cctx->cdictLocal) + cctx->outBuffSize + cctx->inBuffSize - + ZSTDMT_sizeof_CCtx(cctx->mtctx); + + ZSTD_sizeof_mtctx(cctx); } size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) @@ -238,10 +252,6 @@ static ZSTD_CCtx_params ZSTD_assignParamsToCCtxParams( return ERROR(parameter_outOfBound); \ } } -size_t ZSTDMT_CCtxParam_setMTCtxParameter( - ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, unsigned value); -size_t ZSTDMT_initializeCCtxParameters(ZSTD_CCtx_params* params, unsigned nbThreads); - size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value) { if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); @@ -379,16 +389,25 @@ size_t ZSTD_CCtxParam_setParameter( if (value == 0) return 0; #ifndef ZSTD_MULTITHREAD if (value > 1) return ERROR(parameter_unsupported); -#endif +#else return ZSTDMT_initializeCCtxParameters(params, value); +#endif case ZSTD_p_jobSize : +#ifndef ZSTD_MULTITHREAD + return ERROR(parameter_unsupported); +#else if (params->nbThreads <= 1) return ERROR(parameter_unsupported); return ZSTDMT_CCtxParam_setMTCtxParameter(params, ZSTDMT_p_sectionSize, value); +#endif case ZSTD_p_overlapSizeLog : +#ifndef ZSTD_MULTITHREAD + return ERROR(parameter_unsupported); +#else if (params->nbThreads <= 1) return ERROR(parameter_unsupported); return ZSTDMT_CCtxParam_setMTCtxParameter(params, ZSTDMT_p_overlapSectionLog, value); +#endif default: return ERROR(parameter_unsupported); } @@ -2531,16 +2550,6 @@ size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuf return ZSTD_compressStream_generic(zcs, output, input, ZSTD_e_continue); } -/*! ZSTDMT_initCStream_internal() : - * Private use only. Init streaming operation. - * expects params to be valid. - * must receive dict, or cdict, or none, but not both. - * @return : 0, or an error code */ -size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, - const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode, - const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); - size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, ZSTD_outBuffer* output, diff --git a/lib/compress/zstd_compress.h b/lib/compress/zstd_compress.h index 0408be8f..4da0317a 100644 --- a/lib/compress/zstd_compress.h +++ b/lib/compress/zstd_compress.h @@ -16,7 +16,9 @@ * Dependencies ***************************************/ #include "zstd_internal.h" -#include "zstdmt_compress.h" +#ifdef ZSTD_MULTITHREAD +# include "zstdmt_compress.h" +#endif #if defined (__cplusplus) extern "C" { @@ -90,7 +92,9 @@ struct ZSTD_CCtx_s { ZSTD_prefixDict prefixDict; /* single-usage dictionary */ /* Multi-threading */ +#ifdef ZSTD_MULTITHREAD ZSTDMT_CCtx* mtctx; +#endif }; diff --git a/lib/compress/zstdmt_compress.h b/lib/compress/zstdmt_compress.h index 11880dfd..8c59c684 100644 --- a/lib/compress/zstdmt_compress.h +++ b/lib/compress/zstdmt_compress.h @@ -108,6 +108,22 @@ ZSTDLIB_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, ZSTD_EndDirective endOp); +/* === Private definitions; never ever use directly === */ + +size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, unsigned value); + +size_t ZSTDMT_initializeCCtxParameters(ZSTD_CCtx_params* params, unsigned nbThreads); + +/*! ZSTDMT_initCStream_internal() : + * Private use only. Init streaming operation. + * expects params to be valid. + * must receive dict, or cdict, or none, but not both. + * @return : 0, or an error code */ +size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, + const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); + #if defined (__cplusplus) }