From cb18fffe6544eda6e73b53dd9870e7393a267b3c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 24 Sep 2019 17:50:58 -0700 Subject: [PATCH 01/59] enforce C90 compatibility for zlibWrapper --- lib/common/zstd_internal.h | 2 +- lib/zstd.h | 10 +++++----- zlibWrapper/Makefile | 2 +- zlibWrapper/examples/fitblk.c | 8 ++++---- zlibWrapper/gzclose.c | 2 +- zlibWrapper/gzlib.c | 16 ++++++++-------- zlibWrapper/gzread.c | 16 ++++++++-------- zlibWrapper/gzwrite.c | 16 ++++++++-------- zlibWrapper/zstd_zlibwrapper.c | 16 ++++++++++------ 9 files changed, 46 insertions(+), 42 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index f791c5b3..d292f8a7 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -202,7 +202,7 @@ static void ZSTD_copy16(void* dst, const void* src) { memcpy(dst, src, 16); } typedef enum { ZSTD_no_overlap, - ZSTD_overlap_src_before_dst, + ZSTD_overlap_src_before_dst /* ZSTD_overlap_dst_before_src, */ } ZSTD_overlap_e; diff --git a/lib/zstd.h b/lib/zstd.h index 42d4188c..08c851b2 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -398,7 +398,7 @@ typedef enum { ZSTD_c_experimentalParam4=1001, ZSTD_c_experimentalParam5=1002, ZSTD_c_experimentalParam6=1003, - ZSTD_c_experimentalParam7=1004, + ZSTD_c_experimentalParam7=1004 } ZSTD_cParameter; typedef struct { @@ -1106,7 +1106,7 @@ typedef enum { typedef enum { ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ - ZSTD_dlm_byRef = 1, /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ + ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ } ZSTD_dictLoadMethod_e; typedef enum { @@ -1120,7 +1120,7 @@ typedef enum { * This question could be kept for later, when there are actually multiple formats to support, * but there is also the question of pinning enum values, and pinning value `0` is especially important */ ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ - ZSTD_f_zstd1_magicless = 1, /* Variant of zstd frame format, without initial 4-bytes magic number. + ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number. * Useful to save 4 bytes per generated frame. * Decoder cannot recognise automatically this format, requiring this instruction. */ } ZSTD_format_e; @@ -1154,7 +1154,7 @@ typedef enum { */ ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ - ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ + ZSTD_dictForceCopy = 2 /* Always copy the dictionary. */ } ZSTD_dictAttachPref_e; typedef enum { @@ -1163,7 +1163,7 @@ typedef enum { * levels will be compressed. */ ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be * emitted if Huffman compression is not profitable. */ - ZSTD_lcm_uncompressed = 2, /**< Always emit uncompressed literals. */ + ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */ } ZSTD_literalCompressionMode_e; diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index d4fc33b5..9222141a 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -20,7 +20,7 @@ TEST_FILE = ../doc/zstd_compression_format.md CPPFLAGS = -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) \ -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -CFLAGS ?= $(MOREFLAGS) -O3 -std=gnu99 +CFLAGS ?= $(MOREFLAGS) -O3 -std=c90 -pedantic -Wno-long-long -Wno-variadic-macros CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ -Wstrict-aliasing=1 diff --git a/zlibWrapper/examples/fitblk.c b/zlibWrapper/examples/fitblk.c index 6418ca38..6b587cee 100644 --- a/zlibWrapper/examples/fitblk.c +++ b/zlibWrapper/examples/fitblk.c @@ -180,8 +180,8 @@ int main(int argc, char **argv) if (ret == Z_STREAM_END && def.avail_out >= EXCESS) { /* write block to stdout */ have = size + EXCESS - def.avail_out; - // if (fwrite(blk, 1, have, stdout) != have || ferror(stdout)) - // quit("error writing output"); + /* if (fwrite(blk, 1, have, stdout) != have || ferror(stdout)) + * quit("error writing output"); */ /* clean up and print results to stderr */ ret = deflateEnd(&def); @@ -237,8 +237,8 @@ int main(int argc, char **argv) /* done -- write block to stdout */ have = size - def.avail_out; -// if (fwrite(blk, 1, have, stdout) != have || ferror(stdout)) -// quit("error writing output"); + /* if (fwrite(blk, 1, have, stdout) != have || ferror(stdout)) + * quit("error writing output"); */ /* clean up and print results to stderr */ free(tmp); diff --git a/zlibWrapper/gzclose.c b/zlibWrapper/gzclose.c index d4493d01..25d3789b 100644 --- a/zlibWrapper/gzclose.c +++ b/zlibWrapper/gzclose.c @@ -19,7 +19,7 @@ int ZEXPORT gzclose(file) if (file == NULL) return Z_STREAM_ERROR; - state = (gz_statep)file; + state.file = file; return state.state->mode == GZ_READ ? gzclose_r(file) : gzclose_w(file); #else diff --git a/zlibWrapper/gzlib.c b/zlibWrapper/gzlib.c index 3070dd8b..b004e4f3 100644 --- a/zlibWrapper/gzlib.c +++ b/zlibWrapper/gzlib.c @@ -325,7 +325,7 @@ int ZEXPORT gzbuffer(file, size) /* get internal structure and check integrity */ if (file == NULL) return -1; - state = (gz_statep)file; + state.file = file; if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) return -1; @@ -351,7 +351,7 @@ int ZEXPORT gzrewind(file) /* get internal structure */ if (file == NULL) return -1; - state = (gz_statep)file; + state.file = file; /* check that we're reading and that there's no error */ if (state.state->mode != GZ_READ || @@ -378,7 +378,7 @@ z_off64_t ZEXPORT gzseek64(file, offset, whence) /* get internal structure and check integrity */ if (file == NULL) return -1; - state = (gz_statep)file; + state.file = file; if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) return -1; @@ -463,7 +463,7 @@ z_off64_t ZEXPORT gztell64(file) /* get internal structure and check integrity */ if (file == NULL) return -1; - state = (gz_statep)file; + state.file = file; if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) return -1; @@ -491,7 +491,7 @@ z_off64_t ZEXPORT gzoffset64(file) /* get internal structure and check integrity */ if (file == NULL) return -1; - state = (gz_statep)file; + state.file = file; if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) return -1; @@ -523,7 +523,7 @@ int ZEXPORT gzeof(file) /* get internal structure and check integrity */ if (file == NULL) return 0; - state = (gz_statep)file; + state.file = file; if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) return 0; @@ -541,7 +541,7 @@ const char * ZEXPORT gzerror(file, errnum) /* get internal structure and check integrity */ if (file == NULL) return NULL; - state = (gz_statep)file; + state.file = file; if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) return NULL; @@ -561,7 +561,7 @@ void ZEXPORT gzclearerr(file) /* get internal structure and check integrity */ if (file == NULL) return; - state = (gz_statep)file; + state.file = file; if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) return; diff --git a/zlibWrapper/gzread.c b/zlibWrapper/gzread.c index 88fc06c7..bcac9700 100644 --- a/zlibWrapper/gzread.c +++ b/zlibWrapper/gzread.c @@ -386,7 +386,7 @@ int ZEXPORT gzread(file, buf, len) /* get internal structure */ if (file == NULL) return -1; - state = (gz_statep)file; + state.file = file; /* check that we're reading and that there's no (serious) error */ if (state.state->mode != GZ_READ || @@ -424,7 +424,7 @@ z_size_t ZEXPORT gzfread(buf, size, nitems, file) /* get internal structure */ if (file == NULL) return 0; - state = (gz_statep)file; + state.file = file; /* check that we're reading and that there's no (serious) error */ if (state.state->mode != GZ_READ || @@ -470,7 +470,7 @@ int ZEXPORT gzgetc(file) /* get internal structure */ if (file == NULL) return -1; - state = (gz_statep)file; + state.file = file; /* check that we're reading and that there's no (serious) error */ if (state.state->mode != GZ_READ || @@ -485,7 +485,7 @@ int ZEXPORT gzgetc(file) } /* nothing there -- try gz_read() */ - ret = (unsigned)gz_read(state, buf, 1); + ret = (int)gz_read(state, buf, 1); return ret < 1 ? -1 : buf[0]; } @@ -505,7 +505,7 @@ int ZEXPORT gzungetc(c, file) /* get internal structure */ if (file == NULL) return -1; - state = (gz_statep)file; + state.file = file; /* check that we're reading and that there's no (serious) error */ if (state.state->mode != GZ_READ || @@ -569,7 +569,7 @@ char * ZEXPORT gzgets(file, buf, len) /* check parameters and get internal structure */ if (file == NULL || buf == NULL || len < 1) return NULL; - state = (gz_statep)file; + state.file = file; /* check that we're reading and that there's no (serious) error */ if (state.state->mode != GZ_READ || @@ -628,7 +628,7 @@ int ZEXPORT gzdirect(file) /* get internal structure */ if (file == NULL) return 0; - state = (gz_statep)file; + state.file = file; /* if the state is not known, but we can find out, then do so (this is mainly for right after a gzopen() or gzdopen()) */ @@ -649,7 +649,7 @@ int ZEXPORT gzclose_r(file) /* get internal structure */ if (file == NULL) return Z_STREAM_ERROR; - state = (gz_statep)file; + state.file = file; /* check that we're reading */ if (state.state->mode != GZ_READ) diff --git a/zlibWrapper/gzwrite.c b/zlibWrapper/gzwrite.c index 21d5f847..422ff17d 100644 --- a/zlibWrapper/gzwrite.c +++ b/zlibWrapper/gzwrite.c @@ -258,7 +258,7 @@ int ZEXPORT gzwrite(file, buf, len) /* get internal structure */ if (file == NULL) return 0; - state = (gz_statep)file; + state.file = file; /* check that we're writing and that there's no error */ if (state.state->mode != GZ_WRITE || state.state->err != Z_OK) @@ -289,7 +289,7 @@ z_size_t ZEXPORT gzfwrite(buf, size, nitems, file) assert(size != 0); if (file == NULL) return 0; - state = (gz_statep)file; + state.file = file; /* check that we're writing and that there's no error */ if (state.state->mode != GZ_WRITE || state.state->err != Z_OK) @@ -319,7 +319,7 @@ int ZEXPORT gzputc(file, c) /* get internal structure */ if (file == NULL) return -1; - state = (gz_statep)file; + state.file = file; strm = &(state.state->strm); /* check that we're writing and that there's no error */ @@ -366,7 +366,7 @@ int ZEXPORT gzputs(file, str) /* get internal structure */ if (file == NULL) return -1; - state = (gz_statep)file; + state.file = file; /* check that we're writing and that there's no error */ if (state.state->mode != GZ_WRITE || state.state->err != Z_OK) @@ -393,7 +393,7 @@ int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va) /* get internal structure */ if (file == NULL) return Z_STREAM_ERROR; - state = (gz_statep)file; + state.file = file; strm = &(state.state->strm); /* check that we're writing and that there's no error */ @@ -565,7 +565,7 @@ int ZEXPORT gzflush(file, flush) /* get internal structure */ if (file == NULL) return Z_STREAM_ERROR; - state = (gz_statep)file; + state.file = file; /* check that we're writing and that there's no error */ if (state.state->mode != GZ_WRITE || state.state->err != Z_OK) @@ -599,7 +599,7 @@ int ZEXPORT gzsetparams(file, level, strategy) /* get internal structure */ if (file == NULL) return Z_STREAM_ERROR; - state = (gz_statep)file; + state.file = file; strm = &(state.state->strm); /* check that we're writing and that there's no error */ @@ -639,7 +639,7 @@ int ZEXPORT gzclose_w(file) /* get internal structure */ if (file == NULL) return Z_STREAM_ERROR; - state = (gz_statep)file; + state.file = file; /* check that we're writing */ if (state.state->mode != GZ_WRITE) diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 0ee5a310..d07ed953 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -54,7 +54,7 @@ int ZWRAP_isUsingZSTDcompression(void) { return g_ZWRAP_useZSTDcompression; } static ZWRAP_decompress_type g_ZWRAPdecompressionType = ZWRAP_AUTO; -void ZWRAP_setDecompressionType(ZWRAP_decompress_type type) { g_ZWRAPdecompressionType = type; }; +void ZWRAP_setDecompressionType(ZWRAP_decompress_type type) { g_ZWRAPdecompressionType = type; } ZWRAP_decompress_type ZWRAP_getDecompressionType(void) { return g_ZWRAPdecompressionType; } @@ -121,8 +121,10 @@ static ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm) if (zwc==NULL) return NULL; memset(zwc, 0, sizeof(ZWRAP_CCtx)); memcpy(&zwc->allocFunc, strm, sizeof(z_stream)); - { ZSTD_customMem const ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwc->allocFunc }; - zwc->customMem = ZWRAP_customMem; } + { ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, NULL }; + ZWRAP_customMem.opaque = &zwc->allocFunc; + zwc->customMem = ZWRAP_customMem; + } } else { zwc = (ZWRAP_CCtx*)calloc(1, sizeof(*zwc)); if (zwc==NULL) return NULL; @@ -462,8 +464,10 @@ static ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm) if (zwd==NULL) return NULL; memset(zwd, 0, sizeof(ZWRAP_DCtx)); zwd->allocFunc = *strm; /* just to copy zalloc, zfree & opaque */ - { ZSTD_customMem const ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, &zwd->allocFunc }; - zwd->customMem = ZWRAP_customMem; } + { ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, NULL }; + ZWRAP_customMem.opaque = &zwd->allocFunc; + zwd->customMem = ZWRAP_customMem; + } } else { zwd = (ZWRAP_DCtx*)calloc(1, sizeof(*zwd)); if (zwd==NULL) return NULL; @@ -1003,7 +1007,7 @@ ZEXTERN int ZEXPORT z_inflateBackEnd OF((z_streamp strm)) } -ZEXTERN uLong ZEXPORT z_zlibCompileFlags OF((void)) { return zlibCompileFlags(); }; +ZEXTERN uLong ZEXPORT z_zlibCompileFlags OF((void)) { return zlibCompileFlags(); } From 0582b27caeda307eddf8965bfc33f863b716cd3d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 26 Sep 2019 13:08:25 -0700 Subject: [PATCH 02/59] added c++-compat build flag ensure code can be compiled "as is" in C++ mode also : restructured flags so that they can be individually changed / disabled on command line --- zlibWrapper/Makefile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index 9222141a..a61980c2 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -18,12 +18,14 @@ EXAMPLE_PATH = examples PROGRAMS_PATH = ../programs TEST_FILE = ../doc/zstd_compression_format.md -CPPFLAGS = -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) \ +CPPFLAGS = -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) \ -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -CFLAGS ?= $(MOREFLAGS) -O3 -std=c90 -pedantic -Wno-long-long -Wno-variadic-macros -CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ +STDFLAGS = -std=c90 -pedantic -Wno-long-long -Wno-variadic-macros -Wc++-compat +DEBUGFLAGS=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ -Wstrict-aliasing=1 +CFLAGS ?= -O3 +CFLAGS += $(STDFLAGS) $(DEBUGFLAGS) $(MOREFLAGS) # Define *.exe as extension for Windows systems From 69c9401932e1b84af7381e2f5be73bf258225408 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 26 Sep 2019 15:01:29 -0700 Subject: [PATCH 03/59] simplified Makefile and fixed a few c++-compat issues --- zlibWrapper/Makefile | 23 +++++++++++------------ zlibWrapper/examples/fitblk.c | 4 ++-- zlibWrapper/examples/zwrapbench.c | 24 ++++++++++++------------ zlibWrapper/zstd_zlibwrapper.c | 6 +++--- 4 files changed, 28 insertions(+), 29 deletions(-) diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index a61980c2..4ef18028 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -70,35 +70,34 @@ valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench $(VALGRIND) ./zwrapbench -rqi1b1e5 ../lib ../programs ../tests #.c.o: -# $(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $< +# $(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@ -minigzip: $(EXAMPLE_PATH)/minigzip.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) +minigzip: $(EXAMPLE_PATH)/minigzip.o zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZSTDLIBRARY) $(ZLIB_LIBRARY) -o $@ -minigzip_zstd: $(EXAMPLE_PATH)/minigzip.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) +minigzip_zstd: $(EXAMPLE_PATH)/minigzip.o zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZSTDLIBRARY) $(ZLIB_LIBRARY) -o $@ -example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) +example: $(EXAMPLE_PATH)/example.o zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@ -example_zstd: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) +example_zstd: $(EXAMPLE_PATH)/example.o zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@ -fitblk: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY) +fitblk: $(EXAMPLE_PATH)/fitblk.o zstd_zlibwrapper.o $(ZSTDLIBRARY) $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@ -fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY) +fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY) $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@ -zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(PROGRAMS_PATH)/util.o $(PROGRAMS_PATH)/timefn.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY) +zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o zstd_zlibwrapper.o $(PROGRAMS_PATH)/util.o $(PROGRAMS_PATH)/timefn.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY) $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@ -$(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h - $(CC) $(CFLAGS) $(CPPFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c +zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h -$(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h - $(CC) $(CFLAGS) $(CPPFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c +zstdTurnedOn_zlibwrapper.o: CPPFLAGS += -DZWRAP_USE_ZSTD=1 +zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h $(ZSTDLIBDIR)/libzstd.a: $(MAKE) -C $(ZSTDLIBDIR) libzstd.a diff --git a/zlibWrapper/examples/fitblk.c b/zlibWrapper/examples/fitblk.c index 6b587cee..669b176e 100644 --- a/zlibWrapper/examples/fitblk.c +++ b/zlibWrapper/examples/fitblk.c @@ -159,7 +159,7 @@ int main(int argc, char **argv) if (ZWRAP_isUsingZSTDcompression()) printf("zstd version %s\n", zstdVersion()); /* allocate memory for buffers and compression engine */ - blk = malloc(size + EXCESS); + blk = (unsigned char*)malloc(size + EXCESS); def.zalloc = Z_NULL; def.zfree = Z_NULL; def.opaque = Z_NULL; @@ -200,7 +200,7 @@ int main(int argc, char **argv) inf.avail_in = 0; inf.next_in = Z_NULL; ret = inflateInit(&inf); - tmp = malloc(size + EXCESS); + tmp = (unsigned char*)malloc(size + EXCESS); if (ret != Z_OK || tmp == NULL) quit("out of memory"); ret = deflateReset(&def); diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index 61031b9d..35893a4b 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -311,14 +311,14 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize, ret = deflateReset(&def); if (ret != Z_OK) EXM_THROW(1, "deflateReset failure"); if (useSetDict) { - ret = deflateSetDictionary(&def, dictBuffer, dictBufferSize); + ret = deflateSetDictionary(&def, (const z_Bytef*)dictBuffer, dictBufferSize); if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure"); if (ZWRAP_isUsingZSTDcompression()) useSetDict = 0; /* zstd doesn't require deflateSetDictionary after ZWRAP_deflateReset_keepDict */ } - def.next_in = (z_const void*) blockTable[blockNb].srcPtr; + def.next_in = (z_const z_Bytef*) blockTable[blockNb].srcPtr; def.avail_in = (uInt)blockTable[blockNb].srcSize; def.total_in = 0; - def.next_out = (void*) blockTable[blockNb].cPtr; + def.next_out = (z_Bytef*) blockTable[blockNb].cPtr; def.avail_out = (uInt)blockTable[blockNb].cRoom; def.total_out = 0; ret = deflate(&def, Z_FINISH); @@ -343,13 +343,13 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize, ret = deflateInit(&def, cLevel); if (ret != Z_OK) EXM_THROW(1, "deflateInit failure"); if (dictBuffer) { - ret = deflateSetDictionary(&def, dictBuffer, dictBufferSize); + ret = deflateSetDictionary(&def, (const z_Bytef*)dictBuffer, dictBufferSize); if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure"); } - def.next_in = (z_const void*) blockTable[blockNb].srcPtr; + def.next_in = (z_const z_Bytef*) blockTable[blockNb].srcPtr; def.avail_in = (uInt)blockTable[blockNb].srcSize; def.total_in = 0; - def.next_out = (void*) blockTable[blockNb].cPtr; + def.next_out = (z_Bytef*) blockTable[blockNb].cPtr; def.avail_out = (uInt)blockTable[blockNb].cRoom; def.total_out = 0; ret = deflate(&def, Z_FINISH); @@ -451,15 +451,15 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize, else ret = inflateReset(&inf); if (ret != Z_OK) EXM_THROW(1, "inflateReset failure"); - inf.next_in = (z_const void*) blockTable[blockNb].cPtr; + inf.next_in = (z_const z_Bytef*) blockTable[blockNb].cPtr; inf.avail_in = (uInt)blockTable[blockNb].cSize; inf.total_in = 0; - inf.next_out = (void*) blockTable[blockNb].resPtr; + inf.next_out = (z_Bytef*) blockTable[blockNb].resPtr; inf.avail_out = (uInt)blockTable[blockNb].srcSize; inf.total_out = 0; ret = inflate(&inf, Z_FINISH); if (ret == Z_NEED_DICT) { - ret = inflateSetDictionary(&inf, dictBuffer, dictBufferSize); + ret = inflateSetDictionary(&inf, (const z_Bytef*)dictBuffer, dictBufferSize); if (ret != Z_OK) EXM_THROW(1, "inflateSetDictionary failure"); ret = inflate(&inf, Z_FINISH); } @@ -483,15 +483,15 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize, inf.opaque = Z_NULL; ret = inflateInit(&inf); if (ret != Z_OK) EXM_THROW(1, "inflateInit failure"); - inf.next_in = (z_const void*) blockTable[blockNb].cPtr; + inf.next_in = (z_const z_Bytef*) blockTable[blockNb].cPtr; inf.avail_in = (uInt)blockTable[blockNb].cSize; inf.total_in = 0; - inf.next_out = (void*) blockTable[blockNb].resPtr; + inf.next_out = (z_Bytef*) blockTable[blockNb].resPtr; inf.avail_out = (uInt)blockTable[blockNb].srcSize; inf.total_out = 0; ret = inflate(&inf, Z_FINISH); if (ret == Z_NEED_DICT) { - ret = inflateSetDictionary(&inf, dictBuffer, dictBufferSize); + ret = inflateSetDictionary(&inf, (const z_Bytef*) dictBuffer, dictBufferSize); if (ret != Z_OK) EXM_THROW(1, "inflateSetDictionary failure"); ret = inflate(&inf, Z_FINISH); } diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index d07ed953..cb6afa78 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -99,7 +99,7 @@ typedef struct { unsigned long long pledgedSrcSize; } ZWRAP_CCtx; -typedef ZWRAP_CCtx internal_state; +/* typedef ZWRAP_CCtx internal_state; */ @@ -513,7 +513,7 @@ static int ZWRAPD_finishWithErrorMsg(z_streamp strm, char* message) ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, - const char *version, int stream_size)) + const char* version, int stream_size)) { if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB) { strm->reserved = ZWRAP_ZLIB_STREAM; @@ -524,7 +524,7 @@ ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, LOG_WRAPPERD("- inflateInit\n"); if (zwd == NULL) return ZWRAPD_finishWithError(zwd, strm, 0); - zwd->version = ZSTD_malloc(strlen(version)+1, zwd->customMem); + zwd->version = (char*)ZSTD_malloc(strlen(version)+1, zwd->customMem); if (zwd->version == NULL) return ZWRAPD_finishWithError(zwd, strm, 0); strcpy(zwd->version, version); From c69ed0f8d7a2db6b2ebb287ce28beff109af75ef Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 2 Oct 2019 17:32:19 -0700 Subject: [PATCH 04/59] updated tests for zlibwrapper C90 strict compatibility --- zlibWrapper/Makefile | 23 ++++++++++++++--------- zlibWrapper/gzlib.c | 2 +- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index 4ef18028..f8ade828 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -18,15 +18,15 @@ EXAMPLE_PATH = examples PROGRAMS_PATH = ../programs TEST_FILE = ../doc/zstd_compression_format.md -CPPFLAGS = -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) \ - -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -STDFLAGS = -std=c90 -pedantic -Wno-long-long -Wno-variadic-macros -Wc++-compat -DEBUGFLAGS=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ - -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ - -Wstrict-aliasing=1 -CFLAGS ?= -O3 -CFLAGS += $(STDFLAGS) $(DEBUGFLAGS) $(MOREFLAGS) - +CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) \ + -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) +STDCFLAGS = -std=c90 -pedantic -Wno-long-long -Wno-variadic-macros -Wc++-compat \ + -DNO_snprintf -DNO_vsnprintf # strict ISO C90 is missing these prototypes +DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ + -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ + -Wstrict-aliasing=1 +CFLAGS ?= -O3 +CFLAGS += $(STDFLAGS) $(DEBUGFLAGS) $(MOREFLAGS) # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) @@ -35,6 +35,11 @@ else EXT = endif +default : release + +release : STDFLAGS = +release : STDCPPFLAGS = +release : all all: fitblk example zwrapbench minigzip diff --git a/zlibWrapper/gzlib.c b/zlibWrapper/gzlib.c index b004e4f3..b1fb9851 100644 --- a/zlibWrapper/gzlib.c +++ b/zlibWrapper/gzlib.c @@ -216,7 +216,7 @@ local gzFile gz_open(path, fd, mode) #if !defined(NO_snprintf) && !defined(NO_vsnprintf) (void)snprintf(state.state->path, len + 1, "%s", (const char *)path); #else - strcpy(state.state->path, path); + strcpy(state.state->path, (const char*)path); #endif /* compute the flags for open() */ From cadff8cdc4c339aa0a85ef505c5a4a2130ff96af Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 2 Oct 2019 17:41:19 -0700 Subject: [PATCH 05/59] zlibwrapper : fixed flag-variable name release doesn't use specific std/debug flags --- zlibWrapper/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index f8ade828..f63291c9 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -20,7 +20,7 @@ TEST_FILE = ../doc/zstd_compression_format.md CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) \ -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -STDCFLAGS = -std=c90 -pedantic -Wno-long-long -Wno-variadic-macros -Wc++-compat \ +STDFLAGS = -std=c90 -pedantic -Wno-long-long -Wno-variadic-macros -Wc++-compat \ -DNO_snprintf -DNO_vsnprintf # strict ISO C90 is missing these prototypes DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ @@ -38,7 +38,7 @@ endif default : release release : STDFLAGS = -release : STDCPPFLAGS = +release : DEBUGFLAGS = release : all all: fitblk example zwrapbench minigzip From e0d413d648790fa3b34432a6bc2678a3e3361d58 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 4 Oct 2019 15:09:52 -0700 Subject: [PATCH 06/59] fixed init warning --- programs/util.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/programs/util.c b/programs/util.c index 3988295d..c5d344ec 100644 --- a/programs/util.c +++ b/programs/util.c @@ -69,7 +69,8 @@ int UTIL_setFileStat(const char *filename, stat_t *statbuf) #else { /* (atime, mtime) */ - struct timespec timebuf[2] = { {0, UTIME_NOW}, statbuf->st_mtim }; + struct timespec timebuf[2] = { {0, UTIME_NOW} }; + timebuf[1] = statbuf->st_mtim; res += utimensat(AT_FDCWD, filename, timebuf, 0); } #endif From dc1fb684bfbd5bd1400226cb2c9ee69084782a90 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 12 Sep 2019 18:24:44 -0400 Subject: [PATCH 07/59] Remove Unused MEM_SKIP_MSAN Macro --- lib/common/mem.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/common/mem.h b/lib/common/mem.h index 2b115ddb..acc7231e 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -74,12 +74,6 @@ void __msan_poison(const volatile void *a, size_t size); intptr_t __msan_test_shadow(const volatile void *x, size_t size); #endif -#if defined (MEMORY_SANITIZER) -# define MEM_SKIP_MSAN __attribute__((no_sanitize("memory"))) -#else -# define MEM_SKIP_MSAN -#endif - /*-************************************************************** * Basic Types From edb6d884a593bb80c66825c8a476e47ded2d617a Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 12 Sep 2019 18:32:22 -0400 Subject: [PATCH 08/59] Detect Whether We're Being Compiled with ASAN --- lib/common/mem.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/common/mem.h b/lib/common/mem.h index acc7231e..983b55ed 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -74,6 +74,19 @@ void __msan_poison(const volatile void *a, size_t size); intptr_t __msan_test_shadow(const volatile void *x, size_t size); #endif +/* detects whether we are being compiled under asan */ +#if defined (__has_feature) +# if __has_feature(address_sanitizer) +# define ADDRESS_SANITIZER 1 +# endif +#elif defined(__SANITIZE_ADDRESS__) +# define ADDRESS_SANITIZER 1 +#endif + +#if defined (ADDRESS_SANITIZER) +# include +#endif + /*-************************************************************** * Basic Types From 35c30d6ca7219a602dc4c54d33835e4e1be6fee5 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 12 Sep 2019 18:33:00 -0400 Subject: [PATCH 09/59] Poison Unused Workspace Memory --- lib/compress/zstd_cwksp.h | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 39d064c4..6d775f90 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -214,6 +214,11 @@ MEM_STATIC void* ZSTD_cwksp_reserve_internal( ws->tableValidEnd = alloc; } ws->allocStart = alloc; + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + __asan_unpoison_memory_region(alloc, bytes); +#endif + return alloc; } @@ -254,6 +259,11 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { return NULL; } ws->tableEnd = end; + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + __asan_unpoison_memory_region(alloc, bytes); +#endif + return alloc; } @@ -279,6 +289,11 @@ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { ws->objectEnd = end; ws->tableEnd = end; ws->tableValidEnd = end; + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + __asan_unpoison_memory_region(start, bytes); +#endif + return start; } @@ -331,6 +346,14 @@ MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) { */ MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { DEBUGLOG(4, "cwksp: clearing tables!"); + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + { + size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd; + __asan_poison_memory_region(ws->objectEnd, size); + } +#endif + ws->tableEnd = ws->objectEnd; ZSTD_cwksp_assert_internal_consistency(ws); } @@ -353,6 +376,13 @@ MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { } #endif +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + { + size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->objectEnd; + __asan_poison_memory_region(ws->objectEnd, size); + } +#endif + ws->tableEnd = ws->objectEnd; ws->allocStart = ws->workspaceEnd; ws->allocFailed = 0; From da88c35d41f53255f644a8b2ca27f4fa017048e6 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 12 Sep 2019 18:39:46 -0400 Subject: [PATCH 10/59] Stop Assuming Tables are Adjacent --- lib/compress/zstd_compress.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 2f0736b2..02011c94 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1649,12 +1649,13 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, /* copy tables */ { size_t const chainSize = (cdict_cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cdict_cParams->chainLog); size_t const hSize = (size_t)1 << cdict_cParams->hashLog; - size_t const tableSpace = (chainSize + hSize) * sizeof(U32); - assert((U32*)cctx->blockState.matchState.chainTable == (U32*)cctx->blockState.matchState.hashTable + hSize); /* chainTable must follow hashTable */ - assert((U32*)cctx->blockState.matchState.hashTable3 == (U32*)cctx->blockState.matchState.chainTable + chainSize); - assert((U32*)cdict->matchState.chainTable == (U32*)cdict->matchState.hashTable + hSize); /* chainTable must follow hashTable */ - assert((U32*)cdict->matchState.hashTable3 == (U32*)cdict->matchState.chainTable + chainSize); - memcpy(cctx->blockState.matchState.hashTable, cdict->matchState.hashTable, tableSpace); /* presumes all tables follow each other */ + + memcpy(cctx->blockState.matchState.hashTable, + cdict->matchState.hashTable, + hSize * sizeof(U32)); + memcpy(cctx->blockState.matchState.chainTable, + cdict->matchState.chainTable, + chainSize * sizeof(U32)); } /* Zero the hashTable3, since the cdict never fills it */ @@ -1741,10 +1742,16 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog; int const h3log = srcCCtx->blockState.matchState.hashLog3; size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; - size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); - assert((U32*)dstCCtx->blockState.matchState.chainTable == (U32*)dstCCtx->blockState.matchState.hashTable + hSize); /* chainTable must follow hashTable */ - assert((U32*)dstCCtx->blockState.matchState.hashTable3 == (U32*)dstCCtx->blockState.matchState.chainTable + chainSize); - memcpy(dstCCtx->blockState.matchState.hashTable, srcCCtx->blockState.matchState.hashTable, tableSpace); /* presumes all tables follow each other */ + + memcpy(dstCCtx->blockState.matchState.hashTable, + srcCCtx->blockState.matchState.hashTable, + hSize * sizeof(U32)); + memcpy(dstCCtx->blockState.matchState.chainTable, + srcCCtx->blockState.matchState.chainTable, + chainSize * sizeof(U32)); + memcpy(dstCCtx->blockState.matchState.hashTable3, + srcCCtx->blockState.matchState.hashTable3, + h3Size * sizeof(U32)); } ZSTD_cwksp_mark_tables_clean(&dstCCtx->workspace); From 19a0955ec90b30d249e253b47160d40f5b34a5ea Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 12 Sep 2019 19:40:38 -0400 Subject: [PATCH 11/59] Add `ZSTD_cwksp_alloc_size()` to Help Calculate Needed Workspace Size --- lib/compress/zstd_compress.c | 58 ++++++++++++++++++++++-------------- lib/compress/zstd_cwksp.h | 25 ++++++++++++++++ lib/compress/zstd_ldm.c | 6 ++-- 3 files changed, 64 insertions(+), 25 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 02011c94..8fbddf8b 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1078,9 +1078,16 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, size_t const hSize = ((size_t)1) << cParams->hashLog; U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; size_t const h3Size = ((size_t)1) << hashLog3; - size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); - size_t const optPotentialSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<strategy >= ZSTD_btopt)) ? optPotentialSpace : 0; @@ -1097,20 +1104,21 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); U32 const divider = (cParams.minMatch==3) ? 3 : 4; size_t const maxNbSeq = blockSize / divider; - size_t const tokenSpace = WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq; - size_t const entropySpace = HUF_WORKSPACE_SIZE; - size_t const blockStateSpace = 2 * sizeof(ZSTD_compressedBlockState_t); + size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq); + size_t const entropySpace = ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE); + size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 1); size_t const ldmSpace = ZSTD_ldm_getTableSize(params->ldmParams); - size_t const ldmSeqSpace = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize) * sizeof(rawSeq); + size_t const ldmSeqSpace = ZSTD_cwksp_alloc_size(ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize) * sizeof(rawSeq)); size_t const neededSpace = entropySpace + blockStateSpace + tokenSpace + matchStateSize + ldmSpace + ldmSeqSpace; + size_t const cctxSpace = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)); - DEBUGLOG(5, "sizeof(ZSTD_CCtx) : %u", (U32)sizeof(ZSTD_CCtx)); + DEBUGLOG(5, "sizeof(ZSTD_CCtx) : %u", (U32)cctxSpace); DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace); - return sizeof(ZSTD_CCtx) + neededSpace; + return cctxSpace + neededSpace; } } @@ -1393,7 +1401,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); U32 const divider = (params.cParams.minMatch==3) ? 3 : 4; size_t const maxNbSeq = blockSize / divider; - size_t const tokenSpace = WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq; + size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq); size_t const buffOutSize = (zbuff==ZSTDb_buffered) ? ZSTD_compressBound(blockSize)+1 : 0; size_t const buffInSize = (zbuff==ZSTDb_buffered) ? windowSize + blockSize : 0; size_t const matchStateSize = ZSTD_sizeof_matchState(¶ms.cParams, /* forCCtx */ 1); @@ -1408,12 +1416,12 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_cwksp_bump_oversized_duration(ws, 0); /* Check if workspace is large enough, alloc a new one if needed */ - { size_t const cctxSpace = zc->staticSize ? sizeof(ZSTD_CCtx) : 0; - size_t const entropySpace = HUF_WORKSPACE_SIZE; - size_t const blockStateSpace = 2 * sizeof(ZSTD_compressedBlockState_t); - size_t const bufferSpace = buffInSize + buffOutSize; + { size_t const cctxSpace = zc->staticSize ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0; + size_t const entropySpace = ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE); + size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); + size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize) + ZSTD_cwksp_alloc_size(buffOutSize); size_t const ldmSpace = ZSTD_ldm_getTableSize(params.ldmParams); - size_t const ldmSeqSpace = maxNbLdmSeq * sizeof(rawSeq); + size_t const ldmSeqSpace = ZSTD_cwksp_alloc_size(maxNbLdmSeq * sizeof(rawSeq)); size_t const neededSpace = cctxSpace + @@ -3060,8 +3068,11 @@ size_t ZSTD_estimateCDictSize_advanced( ZSTD_dictLoadMethod_e dictLoadMethod) { DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict)); - return sizeof(ZSTD_CDict) + HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) - + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_cwksp_align(dictSize, sizeof(void *))); + return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void *)))); } size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel) @@ -3141,11 +3152,11 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, if (!customMem.customAlloc ^ !customMem.customFree) return NULL; { size_t const workspaceSize = - sizeof(ZSTD_CDict) + - HUF_WORKSPACE_SIZE + + ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 - : ZSTD_cwksp_align(dictSize, sizeof(void*))); + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))); void* const workspace = ZSTD_malloc(workspaceSize, customMem); ZSTD_cwksp ws; ZSTD_CDict* cdict; @@ -3224,8 +3235,11 @@ const ZSTD_CDict* ZSTD_initStaticCDict( ZSTD_compressionParameters cParams) { size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0); - size_t const neededSize = sizeof(ZSTD_CDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : ZSTD_cwksp_align(dictSize, sizeof(void*))) - + HUF_WORKSPACE_SIZE + matchStateSize; + size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + + matchStateSize; ZSTD_CDict* cdict; if ((size_t)workspace & 7) return NULL; /* 8-aligned */ diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 6d775f90..5dce662d 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -34,6 +34,17 @@ extern "C" { * In which case, resize it down to free some memory */ #define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 +/* Since the workspace is effectively its own little malloc implementation / + * arena, when we run under ASAN, we should similarly insert redzones between + * each internal element of the workspace, so ASAN will catch overruns that + * reach outside an object but that stay inside the workspace. + * + * This defines the size of that redzone. + */ +#ifndef ZSTD_CWKSP_ASAN_REDZONE_SIZE +#define ZSTD_CWKSP_ASAN_REDZONE_SIZE 8 +#endif + /*-************************************* * Structures ***************************************/ @@ -166,6 +177,20 @@ MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) { return (size + mask) & ~mask; } +/** + * Use this to determine how much space in the workspace we will consume to + * allocate this object. (Normally it should be exactly the size of the object, + * but under special conditions, like ASAN, where we pad each object, it might + * be larger.) + */ +MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) { +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + return size + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#else + return size; +#endif +} + MEM_STATIC void ZSTD_cwksp_internal_advance_phase( ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) { assert(phase >= ws->phase); diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c index fc3f4694..c3312ad3 100644 --- a/lib/compress/zstd_ldm.c +++ b/lib/compress/zstd_ldm.c @@ -49,9 +49,9 @@ size_t ZSTD_ldm_getTableSize(ldmParams_t params) { size_t const ldmHSize = ((size_t)1) << params.hashLog; size_t const ldmBucketSizeLog = MIN(params.bucketSizeLog, params.hashLog); - size_t const ldmBucketSize = - ((size_t)1) << (params.hashLog - ldmBucketSizeLog); - size_t const totalSize = ldmBucketSize + ldmHSize * sizeof(ldmEntry_t); + size_t const ldmBucketSize = ((size_t)1) << (params.hashLog - ldmBucketSizeLog); + size_t const totalSize = ZSTD_cwksp_alloc_size(ldmBucketSize) + + ZSTD_cwksp_alloc_size(ldmHSize * sizeof(ldmEntry_t)); return params.enableLdm ? totalSize : 0; } From 143b296cf6833c57934981e9c28d090a52caea17 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 12 Sep 2019 19:41:45 -0400 Subject: [PATCH 12/59] Surround Workspace Allocs with Dead Zone --- lib/compress/zstd_cwksp.h | 49 ++++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 5dce662d..e35d8c1c 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -226,8 +226,14 @@ MEM_STATIC void* ZSTD_cwksp_reserve_internal( void* bottom = ws->tableEnd; ZSTD_cwksp_internal_advance_phase(ws, phase); alloc = (BYTE *)ws->allocStart - bytes; - DEBUGLOG(5, "cwksp: reserving %zd bytes, %zd bytes remaining", - bytes, ZSTD_cwksp_available_space(ws) - bytes); + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* over-reserve space */ + alloc = (BYTE *)alloc - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#endif + + DEBUGLOG(5, "cwksp: reserving %p %zd bytes, %zd bytes remaining", + alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); ZSTD_cwksp_assert_internal_consistency(ws); assert(alloc >= bottom); if (alloc < bottom) { @@ -241,6 +247,9 @@ MEM_STATIC void* ZSTD_cwksp_reserve_internal( ws->allocStart = alloc; #if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on + * either size. */ + alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; __asan_unpoison_memory_region(alloc, bytes); #endif @@ -272,8 +281,14 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { void* alloc = ws->tableEnd; void* end = (BYTE *)alloc + bytes; void* top = ws->allocStart; - DEBUGLOG(5, "cwksp: reserving table %zd bytes, %zd bytes remaining", - bytes, ZSTD_cwksp_available_space(ws) - bytes); + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* over-reserve space */ + end = (BYTE *)end + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#endif + + DEBUGLOG(5, "cwksp: reserving %p table %zd bytes, %zd bytes remaining", + alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); assert((bytes & (sizeof(U32)-1)) == 0); ZSTD_cwksp_internal_advance_phase(ws, phase); ZSTD_cwksp_assert_internal_consistency(ws); @@ -286,6 +301,9 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { ws->tableEnd = end; #if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on + * either size. */ + alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; __asan_unpoison_memory_region(alloc, bytes); #endif @@ -297,12 +315,18 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { */ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); - void* start = ws->objectEnd; - void* end = (BYTE*)start + roundedBytes; + void* alloc = ws->objectEnd; + void* end = (BYTE*)alloc + roundedBytes; + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* over-reserve space */ + end = (BYTE *)end + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#endif + DEBUGLOG(5, - "cwksp: reserving object %zd bytes (rounded to %zd), %zd bytes remaining", - bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); - assert(((size_t)start & (sizeof(void*)-1)) == 0); + "cwksp: reserving %p object %zd bytes (rounded to %zd), %zd bytes remaining", + alloc, bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); + assert(((size_t)alloc & (sizeof(void*)-1)) == 0); assert((bytes & (sizeof(void*)-1)) == 0); ZSTD_cwksp_assert_internal_consistency(ws); /* we must be in the first phase, no advance is possible */ @@ -316,10 +340,13 @@ MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { ws->tableValidEnd = end; #if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) - __asan_unpoison_memory_region(start, bytes); + /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on + * either size. */ + alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; + __asan_unpoison_memory_region(alloc, bytes); #endif - return start; + return alloc; } MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) { From ef0b5707c525f2bc97ba3c25119aaafc4d3896ab Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 16 Sep 2019 17:43:05 -0400 Subject: [PATCH 13/59] Refactor Freeing CCtxes / CDicts Inside Workspaces --- lib/compress/zstd_compress.c | 20 +++++++++----------- lib/compress/zstd_cwksp.h | 7 +++++++ 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 8fbddf8b..431307f0 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -130,24 +130,23 @@ static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) { assert(cctx != NULL); assert(cctx->staticSize == 0); - /* Only free workspace if cctx not in workspace, otherwise the workspace - * will be freed when the cctx itself is freed. */ - if ((void*)cctx->workspace.workspace != (void*)cctx) { - ZSTD_cwksp_free(&cctx->workspace, cctx->customMem); - } ZSTD_clearAllDicts(cctx); #ifdef ZSTD_MULTITHREAD ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; #endif + ZSTD_cwksp_free(&cctx->workspace, cctx->customMem); } size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) { + int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx); if (cctx==NULL) return 0; /* support free on NULL */ RETURN_ERROR_IF(cctx->staticSize, memory_allocation, "not compatible with static CCtx"); ZSTD_freeCCtxContent(cctx); - ZSTD_free(cctx, cctx->customMem); + if (!cctxInWorkspace) { + ZSTD_free(cctx, cctx->customMem); + } return 0; } @@ -3204,12 +3203,11 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) { if (cdict==NULL) return 0; /* support free on NULL */ { ZSTD_customMem const cMem = cdict->customMem; - /* Only free workspace if cdict not in workspace, otherwise the - * workspace will be freed when the cdict itself is freed. */ - if ((void*)cdict->workspace.workspace != (void*)cdict) { - ZSTD_cwksp_free(&cdict->workspace, cMem); + int cdictInWorkspace = ZSTD_cwksp_owns_buffer(&cdict->workspace, cdict); + ZSTD_cwksp_free(&cdict->workspace, cMem); + if (!cdictInWorkspace) { + ZSTD_free(cdict, cMem); } - ZSTD_free(cdict, cMem); return 0; } } diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index e35d8c1c..f6068127 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -217,6 +217,13 @@ MEM_STATIC void ZSTD_cwksp_internal_advance_phase( } } +/** + * Returns whether this object/buffer/etc was allocated in this workspace. + */ +MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr) { + return (ptr != NULL) && (ws->workspace <= ptr) && (ptr <= ws->workspaceEnd); +} + /** * Internal function. Do not use directly. */ From 8cffd6ed082b996040ae8c64302e594b1368d272 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 16 Sep 2019 17:43:55 -0400 Subject: [PATCH 14/59] Avoid ASAN Failure in ZSTD_cwksp_free() --- lib/compress/zstd_cwksp.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index f6068127..bae5b654 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -478,9 +478,10 @@ MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem } MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { + void *ptr = ws->workspace; DEBUGLOG(4, "cwksp: freeing workspace"); - ZSTD_free(ws->workspace, customMem); memset(ws, 0, sizeof(ZSTD_cwksp)); + ZSTD_free(ptr, customMem); } /** From b6c0a02a17bcab8aab56f58563f244c52cb1cab8 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 16 Sep 2019 17:45:40 -0400 Subject: [PATCH 15/59] Fix ZSTD_sizeof_matchState() Calculation --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 431307f0..becdc728 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1076,7 +1076,7 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); size_t const hSize = ((size_t)1) << cParams->hashLog; U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; - size_t const h3Size = ((size_t)1) << hashLog3; + size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; size_t const tableSpace = ZSTD_cwksp_alloc_size(chainSize * sizeof(U32)) + ZSTD_cwksp_alloc_size(hSize * sizeof(U32)) + ZSTD_cwksp_alloc_size(h3Size * sizeof(U32)); From 0cc481ef66b951a681ba9af70e06c13b91075ada Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 16 Sep 2019 17:47:29 -0400 Subject: [PATCH 16/59] Fix Workspace Size Calculation --- lib/compress/zstd_compress.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index becdc728..5c739290 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1400,7 +1400,9 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); U32 const divider = (params.cParams.minMatch==3) ? 3 : 4; size_t const maxNbSeq = blockSize / divider; - size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq); + size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) + + ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(seqDef)) + + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); size_t const buffOutSize = (zbuff==ZSTDb_buffered) ? ZSTD_compressBound(blockSize)+1 : 0; size_t const buffInSize = (zbuff==ZSTDb_buffered) ? windowSize + blockSize : 0; size_t const matchStateSize = ZSTD_sizeof_matchState(¶ms.cParams, /* forCCtx */ 1); From a07037b7846f473bb27018a860b9ad1c1c9d46b5 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 16 Sep 2019 17:56:28 -0400 Subject: [PATCH 17/59] Don't Try to Redzone the Tables --- lib/compress/zstd_cwksp.h | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index bae5b654..2fca8cff 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -289,11 +289,6 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { void* end = (BYTE *)alloc + bytes; void* top = ws->allocStart; -#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) - /* over-reserve space */ - end = (BYTE *)end + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; -#endif - DEBUGLOG(5, "cwksp: reserving %p table %zd bytes, %zd bytes remaining", alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); assert((bytes & (sizeof(U32)-1)) == 0); @@ -308,9 +303,6 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { ws->tableEnd = end; #if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) - /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on - * either size. */ - alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; __asan_unpoison_memory_region(alloc, bytes); #endif From 0ffae7e44061bd50aa76cff8b310d6da2f802ddc Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 16 Sep 2019 18:06:16 -0400 Subject: [PATCH 18/59] Stop Allocating Extra Space for Table Redzones --- lib/compress/zstd_compress.c | 6 +++--- lib/compress/zstd_cwksp.h | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 5c739290..11006673 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1077,9 +1077,9 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, size_t const hSize = ((size_t)1) << cParams->hashLog; U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; - size_t const tableSpace = ZSTD_cwksp_alloc_size(chainSize * sizeof(U32)) - + ZSTD_cwksp_alloc_size(hSize * sizeof(U32)) - + ZSTD_cwksp_alloc_size(h3Size * sizeof(U32)); + size_t const tableSpace = chainSize * sizeof(U32) + + hSize * sizeof(U32) + + h3Size * sizeof(U32); size_t const optPotentialSpace = ZSTD_cwksp_alloc_size((MaxML+1) * sizeof(U32)) + ZSTD_cwksp_alloc_size((MaxLL+1) * sizeof(U32)) diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 2fca8cff..15b48b47 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -182,6 +182,10 @@ MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) { * allocate this object. (Normally it should be exactly the size of the object, * but under special conditions, like ASAN, where we pad each object, it might * be larger.) + * + * Since tables aren't currently redzoned, you don't need to call through this + * to figure out how much space you need for the matchState tables. Everything + * else is though. */ MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) { #if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) From b6987acbbf1d565feac9473eb095eebade7f6f1d Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Mon, 16 Sep 2019 19:04:05 -0400 Subject: [PATCH 19/59] Declare the ASAN Functions We Need, Don't Include the Header --- lib/common/mem.h | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/lib/common/mem.h b/lib/common/mem.h index 983b55ed..530d30c8 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -84,7 +84,40 @@ intptr_t __msan_test_shadow(const volatile void *x, size_t size); #endif #if defined (ADDRESS_SANITIZER) -# include +/* Not all platforms that support asan provide sanitizers/asan_interface.h. + * We therefore declare the functions we need ourselves, rather than trying to + * include the header file... */ + +/** + * Marks a memory region ([addr, addr+size)) as unaddressable. + * + * This memory must be previously allocated by your program. Instrumented + * code is forbidden from accessing addresses in this region until it is + * unpoisoned. This function is not guaranteed to poison the entire region - + * it could poison only a subregion of [addr, addr+size) due to ASan + * alignment restrictions. + * + * \note This function is not thread-safe because no two threads can poison or + * unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_poison_memory_region(void const volatile *addr, size_t size); + +/** + * Marks a memory region ([addr, addr+size)) as addressable. + * + * This memory must be previously allocated by your program. Accessing + * addresses in this region is allowed until this region is poisoned again. + * This function could unpoison a super-region of [addr, addr+size) due + * to ASan alignment restrictions. + * + * \note This function is not thread-safe because no two threads can + * poison or unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_unpoison_memory_region(void const volatile *addr, size_t size); #endif From 2c80a9f8ac73ad6634dfe68ae1624eb1af5e3d61 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 17 Sep 2019 11:35:49 -0400 Subject: [PATCH 20/59] Check if CCtx in Workspace after Null Check --- lib/compress/zstd_compress.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 11006673..4afb9cdc 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -139,13 +139,15 @@ static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) { - int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx); if (cctx==NULL) return 0; /* support free on NULL */ RETURN_ERROR_IF(cctx->staticSize, memory_allocation, "not compatible with static CCtx"); - ZSTD_freeCCtxContent(cctx); - if (!cctxInWorkspace) { - ZSTD_free(cctx, cctx->customMem); + { + int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx); + ZSTD_freeCCtxContent(cctx); + if (!cctxInWorkspace) { + ZSTD_free(cctx, cctx->customMem); + } } return 0; } @@ -1077,6 +1079,8 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, size_t const hSize = ((size_t)1) << cParams->hashLog; U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; + /* We don't use ZSTD_cwksp_alloc_size() here because the tables aren't + * surrounded by redzones in ASAN. */ size_t const tableSpace = chainSize * sizeof(U32) + hSize * sizeof(U32) + h3Size * sizeof(U32); From bd6a20b8a0cc16fbd6efa974bf60d59bb79f6398 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 10 Oct 2019 13:45:55 -0400 Subject: [PATCH 21/59] Expand Default Redzone Size --- lib/compress/zstd_cwksp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h index 15b48b47..18492acb 100644 --- a/lib/compress/zstd_cwksp.h +++ b/lib/compress/zstd_cwksp.h @@ -42,7 +42,7 @@ extern "C" { * This defines the size of that redzone. */ #ifndef ZSTD_CWKSP_ASAN_REDZONE_SIZE -#define ZSTD_CWKSP_ASAN_REDZONE_SIZE 8 +#define ZSTD_CWKSP_ASAN_REDZONE_SIZE 128 #endif /*-************************************* From ede31da2ea317d72eb0525efe865e7b2c3ce44b5 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Thu, 10 Oct 2019 15:02:08 -0400 Subject: [PATCH 22/59] Fix CCtx Size Estimation --- lib/compress/zstd_compress.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 4afb9cdc..7af657d7 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1107,7 +1107,9 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); U32 const divider = (cParams.minMatch==3) ? 3 : 4; size_t const maxNbSeq = blockSize / divider; - size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq); + size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) + + ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(seqDef)) + + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); size_t const entropySpace = ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE); size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 1); @@ -1157,7 +1159,8 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize; size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1; - size_t const streamingSize = inBuffSize + outBuffSize; + size_t const streamingSize = ZSTD_cwksp_alloc_size(inBuffSize) + + ZSTD_cwksp_alloc_size(outBuffSize); return CCtxSize + streamingSize; } From 6309be677c21c7fef359569a3e747a3934ab35c3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 15 Oct 2019 16:09:18 -0700 Subject: [PATCH 23/59] minor comments & refactoring --- programs/benchzstd.c | 39 +++++++++++++++++++-------------------- programs/benchzstd.h | 1 - 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/programs/benchzstd.c b/programs/benchzstd.c index 263dc088..7439677c 100644 --- a/programs/benchzstd.c +++ b/programs/benchzstd.c @@ -88,7 +88,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; #endif #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } -#define EXM_THROW_INT(errorNum, ...) { \ +#define RETURN_ERROR_INT(errorNum, ...) { \ DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \ DISPLAYLEVEL(1, "Error %i : ", errorNum); \ DISPLAYLEVEL(1, __VA_ARGS__); \ @@ -401,9 +401,9 @@ BMK_benchMemAdvancedNoAlloc( BMK_initCCtxArgs cctxprep; BMK_initDCtxArgs dctxprep; - cbp.benchFn = local_defaultCompress; + cbp.benchFn = local_defaultCompress; /* ZSTD_compress2 */ cbp.benchPayload = cctx; - cbp.initFn = local_initCCtx; + cbp.initFn = local_initCCtx; /* BMK_initCCtx */ cbp.initPayload = &cctxprep; cbp.errorFn = ZSTD_isError; cbp.blockCount = nbBlocks; @@ -534,8 +534,8 @@ BMK_benchMemAdvancedNoAlloc( if (u==srcSize-1) { /* should never happen */ DISPLAY("no difference detected\n"); } - } - } + } /* for (u=0; umode == BMK_both) && (crcOrig!=crcCheck)) */ } /* CRC Checking */ if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */ @@ -754,8 +754,7 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize, size_t pos = 0, totalSize = 0; unsigned n; for (n=0; n bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ - { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); - if (readSize != (size_t)fileSize) EXM_THROW_INT(11, "could not read %s", fileNamesTable[n]); - pos += readSize; - } - fileSizes[n] = (size_t)fileSize; - totalSize += (size_t)fileSize; - fclose(f); - } + { FILE* const f = fopen(fileNamesTable[n], "rb"); + if (f==NULL) RETURN_ERROR_INT(10, "impossible to open file %s", fileNamesTable[n]); + DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]); + if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ + { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); + if (readSize != (size_t)fileSize) RETURN_ERROR_INT(11, "could not read %s", fileNamesTable[n]); + pos += readSize; + } + fileSizes[n] = (size_t)fileSize; + totalSize += (size_t)fileSize; + fclose(f); + } } - if (totalSize == 0) EXM_THROW_INT(12, "no data to bench"); + if (totalSize == 0) RETURN_ERROR_INT(12, "no data to bench"); return 0; } diff --git a/programs/benchzstd.h b/programs/benchzstd.h index 2c762771..ef7d9fb1 100644 --- a/programs/benchzstd.h +++ b/programs/benchzstd.h @@ -205,7 +205,6 @@ BMK_benchOutcome_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, - #endif /* BENCH_ZSTD_H_3242387 */ #if defined (__cplusplus) From 2d5201b0ab16b74e6aa1b8fa3079d3a5eb4f7eaf Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 16 Oct 2019 14:51:33 -0700 Subject: [PATCH 24/59] removed wildcopy8() which is no longer used, noticed by @davidbolvansky --- doc/zstd_manual.html | 27 +++++++++++++++++++++++++++ lib/common/zstd_internal.h | 14 -------------- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index 79b9d023..7a1b457d 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -866,6 +866,24 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
+
typedef struct {
+    unsigned int matchPos; /* Match pos in dst */
+    /* If seqDef.offset > 3, then this is seqDef.offset - 3
+     * If seqDef.offset < 3, then this is the corresponding repeat offset
+     * But if seqDef.offset < 3 and litLength == 0, this is the
+     *   repeat offset before the corresponding repeat offset
+     * And if seqDef.offset == 3 and litLength == 0, this is the
+     *   most recent repeat offset - 1
+     */
+    unsigned int offset;
+    unsigned int litLength; /* Literal length */
+    unsigned int matchLength; /* Match length */
+    /* 0 when seq not rep and seqDef.offset otherwise
+     * when litLength == 0 this will be <= 4, otherwise <= 3 like normal
+     */
+    unsigned int rep;
+} ZSTD_Sequence;
+

typedef struct {
     unsigned windowLog;       /**< largest match distance : larger == more compression, more memory needed during decompression */
     unsigned chainLog;        /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
@@ -1001,6 +1019,15 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
            or an error code (if srcSize is too small)

+
size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
+    size_t outSeqsSize, const void* src, size_t srcSize);
+
 Extract sequences from the sequence store
+ zc can be used to insert custom compression params.
+ This function invokes ZSTD_compress2
+ @return : number of sequences extracted
+ 
+

+

Memory management


 
 
size_t ZSTD_estimateCCtxSize(int compressionLevel);
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index f791c5b3..13420bd8 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -247,20 +247,6 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e
     }
 }
 
-/*! ZSTD_wildcopy8() :
- *  The same as ZSTD_wildcopy(), but it can only overwrite 8 bytes, and works for
- *  overlapping buffers that are at least 8 bytes apart.
- */
-MEM_STATIC void ZSTD_wildcopy8(void* dst, const void* src, ptrdiff_t length)
-{
-    const BYTE* ip = (const BYTE*)src;
-    BYTE* op = (BYTE*)dst;
-    BYTE* const oend = (BYTE*)op + length;
-    do {
-        COPY8(op, ip);
-    } while (op < oend);
-}
-
 
 /*-*******************************************
 *  Private declarations

From 6323966e53955619e53f9d1d2993bbcae4c28629 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 16 Oct 2019 16:14:04 -0700
Subject: [PATCH 25/59] updated erroneous comments using ZSTD_dm_*

instead of the current ZSTD_dct_*,
reported by @nigeltao (#1822)
---
 doc/zstd_manual.html         | 35 +++++++++++++++++++++++++++++++----
 lib/compress/zstd_compress.c |  2 +-
 lib/zstd.h                   |  8 ++++----
 tests/fuzzer.c               |  8 ++++----
 4 files changed, 40 insertions(+), 13 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 79b9d023..4a0d3ee3 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -796,7 +796,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
   Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
            It's a CPU consuming operation, with non-negligible impact on latency.
            If there is a need to use the same prefix multiple times, consider loadDictionary instead.
-  Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent).
+  Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
            Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. 
 


 
@@ -840,7 +840,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
   Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
            Prefix buffer must remain unmodified up to the end of frame,
            reached when ZSTD_decompressStream() returns 0.
-  Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
+  Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
            Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
   Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
            A full dictionary is more costly, as it requires building tables.
@@ -866,6 +866,24 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
  
 

 
+
typedef struct {
+    unsigned int matchPos; /* Match pos in dst */
+    /* If seqDef.offset > 3, then this is seqDef.offset - 3
+     * If seqDef.offset < 3, then this is the corresponding repeat offset
+     * But if seqDef.offset < 3 and litLength == 0, this is the
+     *   repeat offset before the corresponding repeat offset
+     * And if seqDef.offset == 3 and litLength == 0, this is the
+     *   most recent repeat offset - 1
+     */
+    unsigned int offset;
+    unsigned int litLength; /* Literal length */
+    unsigned int matchLength; /* Match length */
+    /* 0 when seq not rep and seqDef.offset otherwise
+     * when litLength == 0 this will be <= 4, otherwise <= 3 like normal
+     */
+    unsigned int rep;
+} ZSTD_Sequence;
+


 
typedef struct {
     unsigned windowLog;       /**< largest match distance : larger == more compression, more memory needed during decompression */
     unsigned chainLog;        /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
@@ -1001,6 +1019,15 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
            or an error code (if srcSize is too small) 
 


 
+
size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
+    size_t outSeqsSize, const void* src, size_t srcSize);
+
 Extract sequences from the sequence store
+ zc can be used to insert custom compression params.
+ This function invokes ZSTD_compress2
+ @return : number of sequences extracted
+ 
+


+
 
Memory management

 
 
size_t ZSTD_estimateCCtxSize(int compressionLevel);
@@ -1322,7 +1349,7 @@ size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigne
  *
  * Creates of an internal CDict (incompatible with static CCtx), except if
  * dict == NULL or dictSize < 8, in which case no dict is used.
- * Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if
+ * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
  * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
  */
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
@@ -1337,7 +1364,7 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di
  *     ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
  *
  * pledgedSrcSize must be correct. If srcSize is not known at init time, use
- * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy.
+ * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
  */
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
                                              ZSTD_parameters params, unsigned long long pledgedSrcSize);
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 7facbeff..11354e53 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3518,7 +3518,7 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
 /* ZSTD_initCStream_advanced() :
  * pledgedSrcSize must be exact.
  * if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN.
- * dict is loaded with default parameters ZSTD_dm_auto and ZSTD_dlm_byCopy. */
+ * dict is loaded with default parameters ZSTD_dct_auto and ZSTD_dlm_byCopy. */
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
                                  const void* dict, size_t dictSize,
                                  ZSTD_parameters params, unsigned long long pss)
diff --git a/lib/zstd.h b/lib/zstd.h
index 66784562..a710a510 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -928,7 +928,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
  *  Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters.
  *           It's a CPU consuming operation, with non-negligible impact on latency.
  *           If there is a need to use the same prefix multiple times, consider loadDictionary instead.
- *  Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent).
+ *  Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent).
  *           Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */
 ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
                                  const void* prefix, size_t prefixSize);
@@ -972,7 +972,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
  *  Note 2 : Prefix buffer is referenced. It **must** outlive decompression.
  *           Prefix buffer must remain unmodified up to the end of frame,
  *           reached when ZSTD_decompressStream() returns 0.
- *  Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent).
+ *  Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent).
  *           Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section)
  *  Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost.
  *           A full dictionary is more costly, as it requires building tables.
@@ -1670,7 +1670,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLe
  *
  * Creates of an internal CDict (incompatible with static CCtx), except if
  * dict == NULL or dictSize < 8, in which case no dict is used.
- * Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if
+ * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if
  * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.
  */
 ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
@@ -1685,7 +1685,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dic
  *     ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
  *
  * pledgedSrcSize must be correct. If srcSize is not known at init time, use
- * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy.
+ * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy.
  */
 ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
                                              ZSTD_parameters params, unsigned long long pledgedSrcSize);
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 1f46363a..a109a440 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -1387,7 +1387,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         }
         DISPLAYLEVEL(3, "OK \n");
 
-        DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a good dictionary : ", testNb++);
+        DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a good dictionary : ", testNb++);
         {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
             ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
             if (cdict==NULL) goto _output_error;
@@ -1395,7 +1395,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         }
         DISPLAYLEVEL(3, "OK \n");
 
-        DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a rawContent (must fail) : ", testNb++);
+        DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a rawContent (must fail) : ", testNb++);
         {   ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
             ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
             if (cdict!=NULL) goto _output_error;
@@ -1403,7 +1403,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         }
         DISPLAYLEVEL(3, "OK \n");
 
-        DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_auto should fail : ", testNb++);
+        DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++);
         {
             size_t ret;
             MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY);
@@ -1417,7 +1417,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
         }
         DISPLAYLEVEL(3, "OK \n");
 
-        DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_rawContent should pass : ", testNb++);
+        DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++);
         {
             size_t ret;
             MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY);

From 83749411a65bda3c2075253ac740b3ffb5ea6c5f Mon Sep 17 00:00:00 2001
From: Bimba Shrestha 
Date: Wed, 16 Oct 2019 16:26:46 -0700
Subject: [PATCH 26/59] Removing unnecessary check from decode side

---
 doc/educational_decoder/zstd_decompress.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c
index f3e1b848..53ac52e4 100644
--- a/doc/educational_decoder/zstd_decompress.c
+++ b/doc/educational_decoder/zstd_decompress.c
@@ -856,8 +856,7 @@ static size_t decode_literals_compressed(frame_context_t *const ctx,
         // Impossible
         IMPOSSIBLE();
     }
-    if (regenerated_size > MAX_LITERALS_SIZE ||
-        compressed_size >= regenerated_size) {
+    if (regenerated_size > MAX_LITERALS_SIZE) {
         CORRUPTION();
     }
 

From 25ce9ac401ad153697ba34f6cd65c69dd9817add Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 16 Oct 2019 17:27:03 -0700
Subject: [PATCH 27/59] removed UNALIGNED() macro from educational decoder

as this name collides with existing macro in mingw64+clang9.
---
 doc/educational_decoder/zstd_decompress.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c
index 53ac52e4..64e1b873 100644
--- a/doc/educational_decoder/zstd_decompress.c
+++ b/doc/educational_decoder/zstd_decompress.c
@@ -1529,7 +1529,7 @@ void free_dictionary(dictionary_t *const dict) {
 /******* END DICTIONARY PARSING ***********************************************/
 
 /******* IO STREAM OPERATIONS *************************************************/
-#define UNALIGNED() ERROR("Attempting to operate on a non-byte aligned stream")
+
 /// Reads `num` bits from a bitstream, and updates the internal offset
 static inline u64 IO_read_bits(istream_t *const in, const int num_bits) {
     if (num_bits > 64 || num_bits <= 0) {
@@ -1608,7 +1608,7 @@ static inline const u8 *IO_get_read_ptr(istream_t *const in, size_t len) {
         INP_SIZE();
     }
     if (in->bit_offset != 0) {
-        UNALIGNED();
+        ERROR("Attempting to operate on a non-byte aligned stream");
     }
     const u8 *const ptr = in->ptr;
     in->ptr += len;
@@ -1634,7 +1634,7 @@ static inline void IO_advance_input(istream_t *const in, size_t len) {
          INP_SIZE();
     }
     if (in->bit_offset != 0) {
-        UNALIGNED();
+        ERROR("Attempting to operate on a non-byte aligned stream");
     }
 
     in->ptr += len;

From 303261f659959bcd665ce3ff3bc2e3d4c88ccc23 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 10:50:39 -0700
Subject: [PATCH 28/59] ignore build artifact from educational decoder test

---
 doc/educational_decoder/.gitignore | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 doc/educational_decoder/.gitignore

diff --git a/doc/educational_decoder/.gitignore b/doc/educational_decoder/.gitignore
new file mode 100644
index 00000000..b801306f
--- /dev/null
+++ b/doc/educational_decoder/.gitignore
@@ -0,0 +1,2 @@
+# Build artifacts
+harness

From 000404311f60b39d45874bc7bd04672f54bad840 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 10:56:14 -0700
Subject: [PATCH 29/59] fix incorrect dictName/FileName comparison on Windows

inode identification does not seem to work on Windows,
even with on a msys2 posix layer.
---
 programs/util.c | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/programs/util.c b/programs/util.c
index 0908a43a..735606fc 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -106,20 +106,23 @@ int UTIL_compareStr(const void *p1, const void *p2) {
     return strcmp(* (char * const *) p1, * (char * const *) p2);
 }
 
-int UTIL_isSameFile(const char* file1, const char* file2)
+int UTIL_isSameFile(const char* fName1, const char* fName2)
 {
-#if defined(_MSC_VER)
+    assert(fName1 != NULL); assert(fName2 != NULL);
+#if defined(_MSC_VER) || defined(_WIN32)
     /* note : Visual does not support file identification by inode.
+     *        inode does not work on Windows, even with a posix layer, like msys2.
      *        The following work-around is limited to detecting exact name repetition only,
      *        aka `filename` is considered different from `subdir/../filename` */
     return !strcmp(file1, file2);
 #else
-    stat_t file1Stat;
-    stat_t file2Stat;
-    return UTIL_getFileStat(file1, &file1Stat)
-        && UTIL_getFileStat(file2, &file2Stat)
-        && (file1Stat.st_dev == file2Stat.st_dev)
-        && (file1Stat.st_ino == file2Stat.st_ino);
+    {   stat_t file1Stat;
+        stat_t file2Stat;
+        return UTIL_getFileStat(fName1, &file1Stat)
+            && UTIL_getFileStat(fName2, &file2Stat)
+            && (file1Stat.st_dev == file2Stat.st_dev)
+            && (file1Stat.st_ino == file2Stat.st_ino);
+    }
 #endif
 }
 

From a71256a2ee4fc219a85ca0c93260c30434b026ba Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 11:01:20 -0700
Subject: [PATCH 30/59] fix several cast

---
 programs/util.c | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/programs/util.c b/programs/util.c
index 735606fc..321da066 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -243,19 +243,20 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
     DIR *dir;
     struct dirent *entry;
     char* path;
-    int dirLength, fnameLength, pathLength, nbFiles = 0;
+    size_t dirLength, fnameLength, pathLength;
+    int nbFiles = 0;
 
     if (!(dir = opendir(dirName))) {
         UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
         return 0;
     }
 
-    dirLength = (int)strlen(dirName);
+    dirLength = strlen(dirName);
     errno = 0;
     while ((entry = readdir(dir)) != NULL) {
         if (strcmp (entry->d_name, "..") == 0 ||
             strcmp (entry->d_name, ".") == 0) continue;
-        fnameLength = (int)strlen(entry->d_name);
+        fnameLength = strlen(entry->d_name);
         path = (char*) malloc(dirLength + fnameLength + 2);
         if (!path) { closedir(dir); return 0; }
         memcpy(path, dirName, dirLength);
@@ -277,7 +278,8 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
         } else {
             if (*bufStart + *pos + pathLength >= *bufEnd) {
                 ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
-                *bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
+                assert(newListSize >= 0);
+                *bufStart = (char*)UTIL_realloc(*bufStart, (size_t)newListSize);
                 *bufEnd = *bufStart + newListSize;
                 if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
             }
@@ -335,7 +337,8 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
             size_t const len = strlen(inputNames[i]);
             if (buf + pos + len >= bufend) {
                 ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
-                buf = (char*)UTIL_realloc(buf, newListSize);
+                assert(newListSize >= 0);
+                buf = (char*)UTIL_realloc(buf, (size_t)newListSize);
                 bufend = buf + newListSize;
                 if (!buf) return NULL;
             }
@@ -345,7 +348,7 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
                 nbFiles++;
             }
         } else {
-            nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks);
+            nbFiles += (unsigned)UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks);
             if (buf == NULL) return NULL;
     }   }
 

From 1a18f1484fac179125912c58573d743814c83084 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 13:01:18 -0700
Subject: [PATCH 31/59] force compression during tests

to erase potentially remaining artifacts from previous runs
---
 doc/educational_decoder/Makefile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/educational_decoder/Makefile b/doc/educational_decoder/Makefile
index b2ed9f33..a7dc2130 100644
--- a/doc/educational_decoder/Makefile
+++ b/doc/educational_decoder/Makefile
@@ -29,17 +29,17 @@ harness: $(HARNESS_FILES)
 	$(CC) $(FLAGS) $^ -o $@
 
 clean:
-	@$(RM) -f harness
+	@$(RM) harness
 	@$(RM) -rf harness.dSYM
 
 test: harness
 	#
 	# Testing single-file decompression with educational decoder
 	#
-	@$(ZSTD) README.md -o tmp.zst
+	@$(ZSTD) -f README.md -o tmp.zst
 	@./harness tmp.zst tmp
 	@$(DIFF) -s tmp README.md
-	@$(RM) -f tmp*
+	@$(RM) tmp*
 	#
 	# Testing dictionary decompression with education decoder
 	#
@@ -47,8 +47,8 @@ test: harness
 	@$(ZSTD) --train harness.c zstd_decompress.c zstd_decompress.h README.md \
                   harness.c zstd_decompress.c zstd_decompress.h README.md \
                   harness.c zstd_decompress.c zstd_decompress.h README.md
-	@$(ZSTD) -D dictionary README.md -o tmp.zst
+	@$(ZSTD) -f README.md -D dictionary -o tmp.zst
 	@./harness tmp.zst tmp dictionary
 	@$(DIFF) -s tmp README.md
-	@$(RM) -f tmp* dictionary
+	@$(RM) tmp* dictionary
 	@$(MAKE) clean

From bfd829f2544a1518a896ed474408575461453a5b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 14:03:20 -0700
Subject: [PATCH 32/59] minor: DIFF determination

use gdiff on SunOS
---
 doc/educational_decoder/Makefile  | 14 +++++++++---
 doc/educational_decoder/harness.c | 38 ++++++++++++++++---------------
 2 files changed, 31 insertions(+), 21 deletions(-)

diff --git a/doc/educational_decoder/Makefile b/doc/educational_decoder/Makefile
index a7dc2130..704f8676 100644
--- a/doc/educational_decoder/Makefile
+++ b/doc/educational_decoder/Makefile
@@ -7,8 +7,15 @@
 # in the COPYING file in the root directory of this source tree).
 # ################################################################
 
-ZSTD ?= zstd   # requires zstd installation on local system
+ZSTD ?= zstd   # note: requires zstd installation on local system
+
+UNAME?= $(shell uname)
+ifeq ($(UNAME), SunOS)
+DIFF ?= gdiff
+else
 DIFF ?= diff
+endif
+
 HARNESS_FILES=*.c
 
 MULTITHREAD_LDFLAGS = -pthread
@@ -30,7 +37,7 @@ harness: $(HARNESS_FILES)
 
 clean:
 	@$(RM) harness
-	@$(RM) -rf harness.dSYM
+	@$(RM) -rf harness.dSYM  # MacOS specific
 
 test: harness
 	#
@@ -46,7 +53,8 @@ test: harness
 	# note : files are presented multiple for training, to reach minimum threshold
 	@$(ZSTD) --train harness.c zstd_decompress.c zstd_decompress.h README.md \
                   harness.c zstd_decompress.c zstd_decompress.h README.md \
-                  harness.c zstd_decompress.c zstd_decompress.h README.md
+                  harness.c zstd_decompress.c zstd_decompress.h README.md \
+                  -o dictionary
 	@$(ZSTD) -f README.md -D dictionary -o tmp.zst
 	@./harness tmp.zst tmp dictionary
 	@$(DIFF) -s tmp README.md
diff --git a/doc/educational_decoder/harness.c b/doc/educational_decoder/harness.c
index 36f3967a..df4c0af9 100644
--- a/doc/educational_decoder/harness.c
+++ b/doc/educational_decoder/harness.c
@@ -25,28 +25,29 @@ u8 *input;
 u8 *output;
 u8 *dict;
 
-size_t read_file(const char *path, u8 **ptr) {
-    FILE *f = fopen(path, "rb");
+static size_t read_file(const char *path, u8 **ptr)
+{
+    FILE* const f = fopen(path, "rb");
     if (!f) {
-        fprintf(stderr, "failed to open file %s\n", path);
+        fprintf(stderr, "failed to open file %s \n", path);
         exit(1);
     }
 
     fseek(f, 0L, SEEK_END);
-    size_t size = (size_t)ftell(f);
+    size_t const size = (size_t)ftell(f);
     rewind(f);
 
     *ptr = malloc(size);
     if (!ptr) {
-        fprintf(stderr, "failed to allocate memory to hold %s\n", path);
+        fprintf(stderr, "failed to allocate memory to hold %s \n", path);
         exit(1);
     }
 
     size_t pos = 0;
     while (!feof(f)) {
-        size_t read = fread(&(*ptr)[pos], 1, size, f);
+        size_t const read = fread(*ptr + pos, 1, size, f);
         if (ferror(f)) {
-            fprintf(stderr, "error while reading file %s\n", path);
+            fprintf(stderr, "error while reading file %s \n", path);
             exit(1);
         }
         pos += read;
@@ -57,30 +58,30 @@ size_t read_file(const char *path, u8 **ptr) {
     return pos;
 }
 
-void write_file(const char *path, const u8 *ptr, size_t size) {
-    FILE *f = fopen(path, "wb");
+static void write_file(const char *path, const u8 *ptr, size_t size)
+{
+    FILE* const f = fopen(path, "wb");
 
     size_t written = 0;
     while (written < size) {
-        written += fwrite(&ptr[written], 1, size, f);
+        written += fwrite(ptr+written, 1, size, f);
         if (ferror(f)) {
             fprintf(stderr, "error while writing file %s\n", path);
             exit(1);
-        }
-    }
+    }   }
 
     fclose(f);
 }
 
 int main(int argc, char **argv) {
     if (argc < 3) {
-        fprintf(stderr, "usage: %s   [dictionary]\n",
+        fprintf(stderr, "usage: %s   [dictionary] \n",
                 argv[0]);
 
         return 1;
     }
 
-    size_t input_size = read_file(argv[1], &input);
+    size_t const input_size = read_file(argv[1], &input);
     size_t dict_size = 0;
     if (argc >= 4) {
         dict_size = read_file(argv[3], &dict);
@@ -92,17 +93,17 @@ int main(int argc, char **argv) {
         fprintf(stderr, "WARNING: Compressed data does not contain "
                         "decompressed size, going to assume the compression "
                         "ratio is at most %d (decompressed size of at most "
-                        "%zu)\n",
-                MAX_COMPRESSION_RATIO, decompressed_size);
+                        "%u) \n",
+                MAX_COMPRESSION_RATIO, (unsigned)decompressed_size);
     }
     if (decompressed_size > MAX_OUTPUT_SIZE) {
         fprintf(stderr,
-                "Required output size too large for this implementation\n");
+                "Required output size too large for this implementation \n");
         return 1;
     }
     output = malloc(decompressed_size);
     if (!output) {
-        fprintf(stderr, "failed to allocate memory\n");
+        fprintf(stderr, "failed to allocate memory \n");
         return 1;
     }
 
@@ -122,4 +123,5 @@ int main(int argc, char **argv) {
     free(output);
     free(dict);
     input = output = dict = NULL;
+    return 0;
 }

From b062b6fb2d2c5e58c47fdac4355449051445fd6c Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 14:11:54 -0700
Subject: [PATCH 33/59] minor refactoring of harness

---
 doc/educational_decoder/harness.c | 37 ++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 16 deletions(-)

diff --git a/doc/educational_decoder/harness.c b/doc/educational_decoder/harness.c
index df4c0af9..33839491 100644
--- a/doc/educational_decoder/harness.c
+++ b/doc/educational_decoder/harness.c
@@ -21,10 +21,6 @@ typedef unsigned char u8;
 // Protect against allocating too much memory for output
 #define MAX_OUTPUT_SIZE ((size_t)1024 * 1024 * 1024)
 
-u8 *input;
-u8 *output;
-u8 *dict;
-
 static size_t read_file(const char *path, u8 **ptr)
 {
     FILE* const f = fopen(path, "rb");
@@ -61,6 +57,10 @@ static size_t read_file(const char *path, u8 **ptr)
 static void write_file(const char *path, const u8 *ptr, size_t size)
 {
     FILE* const f = fopen(path, "wb");
+    if (!f) {
+        fprintf(stderr, "failed to open file %s \n", path);
+        exit(1);
+    }
 
     size_t written = 0;
     while (written < size) {
@@ -73,7 +73,8 @@ static void write_file(const char *path, const u8 *ptr, size_t size)
     fclose(f);
 }
 
-int main(int argc, char **argv) {
+int main(int argc, char **argv)
+{
     if (argc < 3) {
         fprintf(stderr, "usage: %s   [dictionary] \n",
                 argv[0]);
@@ -81,27 +82,31 @@ int main(int argc, char **argv) {
         return 1;
     }
 
+    u8* input;
     size_t const input_size = read_file(argv[1], &input);
+
+    u8* dict;
     size_t dict_size = 0;
     if (argc >= 4) {
         dict_size = read_file(argv[3], &dict);
     }
 
-    size_t decompressed_size = ZSTD_get_decompressed_size(input, input_size);
-    if (decompressed_size == (size_t)-1) {
-        decompressed_size = MAX_COMPRESSION_RATIO * input_size;
+    size_t out_capacity = ZSTD_get_decompressed_size(input, input_size);
+    if (out_capacity == (size_t)-1) {
+        out_capacity = MAX_COMPRESSION_RATIO * input_size;
         fprintf(stderr, "WARNING: Compressed data does not contain "
                         "decompressed size, going to assume the compression "
                         "ratio is at most %d (decompressed size of at most "
                         "%u) \n",
-                MAX_COMPRESSION_RATIO, (unsigned)decompressed_size);
+                MAX_COMPRESSION_RATIO, (unsigned)out_capacity);
     }
-    if (decompressed_size > MAX_OUTPUT_SIZE) {
+    if (out_capacity > MAX_OUTPUT_SIZE) {
         fprintf(stderr,
                 "Required output size too large for this implementation \n");
         return 1;
     }
-    output = malloc(decompressed_size);
+
+    u8* const output = malloc(out_capacity);
     if (!output) {
         fprintf(stderr, "failed to allocate memory \n");
         return 1;
@@ -111,17 +116,17 @@ int main(int argc, char **argv) {
     if (dict) {
         parse_dictionary(parsed_dict, dict, dict_size);
     }
-    size_t decompressed =
-        ZSTD_decompress_with_dict(output, decompressed_size,
-                                  input, input_size, parsed_dict);
+    size_t const decompressed_size =
+        ZSTD_decompress_with_dict(output, out_capacity,
+                                  input, input_size,
+                                  parsed_dict);
 
     free_dictionary(parsed_dict);
 
-    write_file(argv[2], output, decompressed);
+    write_file(argv[2], output, decompressed_size);
 
     free(input);
     free(output);
     free(dict);
-    input = output = dict = NULL;
     return 0;
 }

From a0c041612d6ec63bf3ad028d928f7ee346956ed7 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 14:15:00 -0700
Subject: [PATCH 34/59] fixed dict ptr init

---
 doc/educational_decoder/harness.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/educational_decoder/harness.c b/doc/educational_decoder/harness.c
index 33839491..de2e1235 100644
--- a/doc/educational_decoder/harness.c
+++ b/doc/educational_decoder/harness.c
@@ -85,7 +85,7 @@ int main(int argc, char **argv)
     u8* input;
     size_t const input_size = read_file(argv[1], &input);
 
-    u8* dict;
+    u8* dict = NULL;
     size_t dict_size = 0;
     if (argc >= 4) {
         dict_size = read_file(argv[3], &dict);

From 5b8e873357a32e53edb6ef3b7cbe8bc58a9c75fc Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 14:29:48 -0700
Subject: [PATCH 35/59] fix harness test

---
 doc/educational_decoder/harness.c | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/doc/educational_decoder/harness.c b/doc/educational_decoder/harness.c
index de2e1235..a704f6bd 100644
--- a/doc/educational_decoder/harness.c
+++ b/doc/educational_decoder/harness.c
@@ -39,19 +39,15 @@ static size_t read_file(const char *path, u8 **ptr)
         exit(1);
     }
 
-    size_t pos = 0;
-    while (!feof(f)) {
-        size_t const read = fread(*ptr + pos, 1, size, f);
-        if (ferror(f)) {
-            fprintf(stderr, "error while reading file %s \n", path);
-            exit(1);
-        }
-        pos += read;
+    size_t const read = fread(*ptr, 1, size, f);
+    if (read != size) {  /* must read everything in one pass */
+        fprintf(stderr, "error while reading file %s \n", path);
+        exit(1);
     }
 
     fclose(f);
 
-    return pos;
+    return read;
 }
 
 static void write_file(const char *path, const u8 *ptr, size_t size)

From 157479af0cc0611bedf3c5c1987971b761620eb1 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 14:31:42 -0700
Subject: [PATCH 36/59] fixed isSameFile()

---
 programs/util.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/util.c b/programs/util.c
index 321da066..dbdd1cee 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -114,7 +114,7 @@ int UTIL_isSameFile(const char* fName1, const char* fName2)
      *        inode does not work on Windows, even with a posix layer, like msys2.
      *        The following work-around is limited to detecting exact name repetition only,
      *        aka `filename` is considered different from `subdir/../filename` */
-    return !strcmp(file1, file2);
+    return !strcmp(fName1, fName2);
 #else
     {   stat_t file1Stat;
         stat_t file2Stat;

From ba7e2b6da73ca9dc9279357d8649e075c35030ce Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 15:07:47 -0700
Subject: [PATCH 37/59] tests: can override isTerminal with environment
 variable

---
 tests/playTests.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/playTests.sh b/tests/playTests.sh
index 06de4f8a..f63a7f69 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -64,11 +64,12 @@ PRGDIR="$SCRIPT_DIR/../programs"
 TESTDIR="$SCRIPT_DIR/../tests"
 UNAME=$(uname)
 
-isTerminal=false
+detectedTerminal=false
 if [ -t 0 ] && [ -t 1 ]
 then
-    isTerminal=true
+    detectedTerminal=true
 fi
+isTerminal=${isTerminal:-$detectedTerminal}
 
 isWindows=false
 INTOVOID="/dev/null"

From 7f86ae2867204c8f12fa8cfe858da2d62371370d Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 15:27:25 -0700
Subject: [PATCH 38/59] fixed multiple implicit casts

---
 programs/zstdcli.c | 52 +++++++++++++++++++++++-----------------------
 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index ae53d2c3..6c6c10f2 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -287,7 +287,7 @@ static unsigned readU32FromChar(const char** stringPtr) {
  *  If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
  * @return 0 and doesn't modify *stringPtr otherwise.
  */
-static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
+static int longCommandWArg(const char** stringPtr, const char* longCommand)
 {
     size_t const comSize = strlen(longCommand);
     int const result = !strncmp(*stringPtr, longCommand, comSize);
@@ -430,8 +430,8 @@ static ZDICT_fastCover_params_t defaultFastCoverParams(void)
 static unsigned parseAdaptParameters(const char* stringPtr, int* adaptMinPtr, int* adaptMaxPtr)
 {
     for ( ; ;) {
-        if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
-        if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        if (longCommandWArg(&stringPtr, "min=")) { *adaptMinPtr = (int)readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
+        if (longCommandWArg(&stringPtr, "max=")) { *adaptMaxPtr = (int)readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
         DISPLAYLEVEL(4, "invalid compression parameter \n");
         return 0;
     }
@@ -526,7 +526,7 @@ static int init_cLevel(void) {
                 DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large\n", ENV_CLEVEL, env);
                 return ZSTDCLI_CLEVEL_DEFAULT;
             } else if (*ptr == 0) {
-                return sign * absLevel;
+                return sign * (int)absLevel;
             }
         }
 
@@ -584,7 +584,7 @@ int main(int argCount, const char* argv[])
     int cLevelLast = -1000000000;
     unsigned recursive = 0;
     unsigned memLimit = 0;
-    const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*));   /* argCount >= 1 */
+    const char** filenameTable = (const char**)malloc((size_t)argCount * sizeof(const char*));   /* argCount >= 1 */
     unsigned filenameIdx = 0;
     const char* programName = argv[0];
     const char* outFileName = NULL;
@@ -745,7 +745,7 @@ int main(int argCount, const char* argv[])
                       continue;
                     }
 #endif
-                    if (longCommandWArg(&argument, "--threads=")) { nbWorkers = readU32FromChar(&argument); continue; }
+                    if (longCommandWArg(&argument, "--threads=")) { nbWorkers = (int)readU32FromChar(&argument); continue; }
                     if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
                     if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
                     if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
@@ -807,7 +807,7 @@ int main(int argCount, const char* argv[])
 #ifndef ZSTD_NOCOMPRESS
                     /* compression Level */
                     if ((*argument>='0') && (*argument<='9')) {
-                        dictCLevel = cLevel = readU32FromChar(&argument);
+                        dictCLevel = cLevel = (int)readU32FromChar(&argument);
                         continue;
                     }
 #endif
@@ -856,7 +856,7 @@ int main(int argCount, const char* argv[])
 
                         /* destination file name */
                     case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break;
-                   
+
                         /* limit decompression memory */
                     case 'M':
                         argument++;
@@ -879,7 +879,7 @@ int main(int argCount, const char* argv[])
                     case 'e':
                         /* compression Level */
                         argument++;
-                        cLevelLast = readU32FromChar(&argument);
+                        cLevelLast = (int)readU32FromChar(&argument);
                         break;
 
                         /* Modify Nb Iterations (benchmark only) */
@@ -905,7 +905,7 @@ int main(int argCount, const char* argv[])
                         /* nb of threads (hidden option) */
                     case 'T':
                         argument++;
-                        nbWorkers = readU32FromChar(&argument);
+                        nbWorkers = (int)readU32FromChar(&argument);
                         break;
 
                         /* Dictionary Selection level */
@@ -1042,16 +1042,16 @@ int main(int argCount, const char* argv[])
 #ifndef ZSTD_NOBENCH
         benchParams.blockSize = blockSize;
         benchParams.nbWorkers = nbWorkers;
-        benchParams.realTime = setRealTimePrio;
+        benchParams.realTime = (unsigned)setRealTimePrio;
         benchParams.nbSeconds = bench_nbSeconds;
         benchParams.ldmFlag = ldmFlag;
-        benchParams.ldmMinMatch = g_ldmMinMatch;
-        benchParams.ldmHashLog = g_ldmHashLog;
+        benchParams.ldmMinMatch = (int)g_ldmMinMatch;
+        benchParams.ldmHashLog = (int)g_ldmHashLog;
         if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
-            benchParams.ldmBucketSizeLog = g_ldmBucketSizeLog;
+            benchParams.ldmBucketSizeLog = (int)g_ldmBucketSizeLog;
         }
         if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) {
-            benchParams.ldmHashRateLog = g_ldmHashRateLog;
+            benchParams.ldmHashRateLog = (int)g_ldmHashRateLog;
         }
         benchParams.literalCompressionMode = literalCompressionMode;
 
@@ -1092,16 +1092,16 @@ int main(int argCount, const char* argv[])
 #ifndef ZSTD_NODICT
         ZDICT_params_t zParams;
         zParams.compressionLevel = dictCLevel;
-        zParams.notificationLevel = g_displayLevel;
+        zParams.notificationLevel = (unsigned)g_displayLevel;
         zParams.dictID = dictID;
         if (dict == cover) {
             int const optimize = !coverParams.k || !coverParams.d;
-            coverParams.nbThreads = nbWorkers;
+            coverParams.nbThreads = (unsigned)nbWorkers;
             coverParams.zParams = zParams;
             operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, NULL, optimize);
         } else if (dict == fastCover) {
             int const optimize = !fastCoverParams.k || !fastCoverParams.d;
-            fastCoverParams.nbThreads = nbWorkers;
+            fastCoverParams.nbThreads = (unsigned)nbWorkers;
             fastCoverParams.zParams = zParams;
             operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, NULL, &fastCoverParams, optimize);
         } else {
@@ -1156,14 +1156,14 @@ int main(int argCount, const char* argv[])
     if (operation==zom_compress) {
 #ifndef ZSTD_NOCOMPRESS
         FIO_setNbWorkers(prefs, nbWorkers);
-        FIO_setBlockSize(prefs, (U32)blockSize);
-        if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, g_overlapLog);
-        FIO_setLdmFlag(prefs, ldmFlag);
-        FIO_setLdmHashLog(prefs, g_ldmHashLog);
-        FIO_setLdmMinMatch(prefs, g_ldmMinMatch);
-        if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, g_ldmBucketSizeLog);
-        if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, g_ldmHashRateLog);
-        FIO_setAdaptiveMode(prefs, adapt);
+        FIO_setBlockSize(prefs, (int)blockSize);
+        if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, (int)g_overlapLog);
+        FIO_setLdmFlag(prefs, (unsigned)ldmFlag);
+        FIO_setLdmHashLog(prefs, (int)g_ldmHashLog);
+        FIO_setLdmMinMatch(prefs, (int)g_ldmMinMatch);
+        if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, (int)g_ldmBucketSizeLog);
+        if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, (int)g_ldmHashRateLog);
+        FIO_setAdaptiveMode(prefs, (unsigned)adapt);
         FIO_setAdaptMin(prefs, adaptMin);
         FIO_setAdaptMax(prefs, adaptMax);
         FIO_setRsyncable(prefs, rsyncable);

From 1795133c450e839aa7b766d30a8c33989b7aad56 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 15:32:03 -0700
Subject: [PATCH 39/59] refactored FIO_compressMultipleFilenames() prototype

for consistency
---
 lib/compress/zstd_cwksp.h |  2 +-
 programs/fileio.c         | 17 +++++++++--------
 programs/fileio.h         |  9 +++++----
 programs/zstdcli.c        |  2 +-
 4 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/lib/compress/zstd_cwksp.h b/lib/compress/zstd_cwksp.h
index 39d064c4..a6a4a3fb 100644
--- a/lib/compress/zstd_cwksp.h
+++ b/lib/compress/zstd_cwksp.h
@@ -404,7 +404,7 @@ MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) {
 }
 
 MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) {
-    return (BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace;
+    return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace);
 }
 
 MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) {
diff --git a/programs/fileio.c b/programs/fileio.c
index eecdf0dd..f0527528 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -648,7 +648,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
         DISPLAY("Unable to malloc new str array, not checking for name collisions\n");
         return 1;
     }
-    
+
     for (u = 0; u < nbFiles; ++u) {
         filename = strrchr(filenameTable[u], c[0]);
         if (filename == NULL) {
@@ -715,7 +715,7 @@ FIO_createFilename_fromOutDir(const char* srcFilename, const char* outDirName, c
     strcpy(result, outDirName);
     if (outDirName[strlen(outDirName)-1] == c[0]) {
         strcat(result, filename);
-    } else {  
+    } else {
         strcat(result, c);
         strcat(result, filename);
     }
@@ -1493,7 +1493,7 @@ FIO_determineCompressedName(const char* srcFileName, const char* outDirName, con
         sfnSize = strlen(outDirFilename);
         assert(outDirFilename != NULL);
     }
-    
+
     if (dfnbCapacity <= sfnSize+suffixSize+1) {
         /* resize buffer for dstName */
         free(dstFileNameBuffer);
@@ -1522,9 +1522,10 @@ FIO_determineCompressedName(const char* srcFileName, const char* outDirName, con
  * or into one file each (outFileName == NULL, but suffix != NULL),
  * or into a destination folder (specified with -O)
  */
-int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable,
-                                  const char* outDirName, unsigned nbFiles, 
-                                  const char* outFileName, const char* suffix, 
+int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
+                                  const char** inFileNamesTable, unsigned nbFiles,
+                                  const char* outDirName,
+                                  const char* outFileName, const char* suffix,
                                   const char* dictFileName, int compressionLevel,
                                   ZSTD_compressionParameters comprParams)
 {
@@ -2278,7 +2279,7 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName)
     char* outDirFilename = NULL;
     size_t sfnSize = strlen(srcFileName);
     size_t suffixSize;
-    
+
     const char* const suffixPtr = strrchr(srcFileName, '.');
     if (suffixPtr == NULL) {
         DISPLAYLEVEL(1, "zstd: %s: unknown suffix -- ignored \n",
@@ -2328,7 +2329,7 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName)
         dfnbCapacity = sfnSize + 20;
         dstFileNameBuffer = (char*)malloc(dfnbCapacity);
         if (dstFileNameBuffer==NULL)
-            EXM_THROW(74, "%s : not enough memory for dstFileName", strerror(errno)); 
+            EXM_THROW(74, "%s : not enough memory for dstFileName", strerror(errno));
     }
 
     /* return dst name == src name truncated from suffix */
diff --git a/programs/fileio.h b/programs/fileio.h
index ded1c037..841e0a3c 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -104,10 +104,11 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
 ***************************************/
 /** FIO_compressMultipleFilenames() :
     @return : nb of missing files */
-int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable,
-                                  const char* outDirName, unsigned nbFiles, 
-                                  const char* outFileName, const char* suffix, 
-                                  const char* dictFileName, int compressionLevel, 
+int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
+                                  const char** inFileNamesTable, unsigned nbFiles,
+                                  const char* outDirName,
+                                  const char* outFileName, const char* suffix,
+                                  const char* dictFileName, int compressionLevel,
                                   ZSTD_compressionParameters comprParams);
 
 /** FIO_decompressMultipleFilenames() :
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 6c6c10f2..4d1fad70 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -1177,7 +1177,7 @@ int main(int argCount, const char* argv[])
         if ((filenameIdx==1) && outFileName)
           operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams);
         else
-          operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
+          operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams);
 #else
         (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */
         DISPLAY("Compression not supported \n");

From 0ee360982d1b8e4f959f6da32bf1608935a867e8 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 16:09:53 -0700
Subject: [PATCH 40/59] improved test mode `-t`

The test mode do no longer open a file `/dev/null` nor write anything to output.

This is supposed to be more efficient than writing to `/dev/null`,
and more universal too : the previous method was failing on Windows.
---
 programs/fileio.c  | 91 ++++++++++++++++++++++++++++------------------
 programs/fileio.h  | 13 ++++---
 programs/zstdcli.c |  2 +-
 3 files changed, 64 insertions(+), 42 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index f0527528..9efb4aae 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -284,9 +284,10 @@ void FIO_addAbortHandler()
 
 
 /*-*************************************
-*  Parameters: Typedefs
+*  Parameters: FIO_prefs_t
 ***************************************/
 
+/* typedef'd to FIO_prefs_t within fileio.h */
 struct FIO_prefs_s {
 
     /* Algorithm preferences */
@@ -308,6 +309,7 @@ struct FIO_prefs_s {
     size_t streamSrcSize;
     size_t targetCBlockSize;
     int srcSizeHint;
+    int testMode;
     ZSTD_literalCompressionMode_e literalCompressionMode;
 
     /* IO preferences */
@@ -355,6 +357,7 @@ FIO_prefs_t* FIO_createPreferences(void)
     ret->streamSrcSize = 0;
     ret->targetCBlockSize = 0;
     ret->srcSizeHint = 0;
+    ret->testMode = 0;
     ret->literalCompressionMode = ZSTD_lcm_auto;
     return ret;
 }
@@ -435,6 +438,10 @@ void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint) {
     prefs->srcSizeHint = (int)MIN((size_t)INT_MAX, srcSizeHint);
 }
 
+void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode) {
+    prefs->testMode = (testMode!=0);
+}
+
 void FIO_setLiteralCompressionMode(
         FIO_prefs_t* const prefs,
         ZSTD_literalCompressionMode_e mode) {
@@ -1618,7 +1625,11 @@ static void FIO_freeDResources(dRess_t ress)
 
 /** FIO_fwriteSparse() :
 *   @return : storedSkips, to be provided to next call to FIO_fwriteSparse() of LZ4IO_fwriteSparseEnd() */
-static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips)
+static unsigned
+FIO_fwriteSparse(const FIO_prefs_t* const prefs,
+                 FILE* file,
+                 const void* buffer, size_t bufferSize,
+                 unsigned storedSkips)
 {
     const size_t* const bufferT = (const size_t*)buffer;   /* Buffer is supposed malloc'ed, hence aligned on size_t */
     size_t bufferSizeT = bufferSize / sizeof(size_t);
@@ -1626,6 +1637,8 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
     const size_t* ptrT = bufferT;
     static const size_t segmentSizeT = (32 KB) / sizeof(size_t);   /* 0-test re-attempted every 32 KB */
 
+    if (prefs->testMode) return 0;  /* do not output anything in test mode */
+
     if (!prefs->sparseFileSupport) {  /* normal write */
         size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
         if (sizeCheck != bufferSize)
@@ -1690,8 +1703,9 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
 }
 
 static void
-FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
+FIO_fwriteSparseEnd(const FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
 {
+    if (prefs->testMode) assert(storedSkips == 0);
     if (storedSkips>0) {
         assert(prefs->sparseFileSupport > 0);  /* storedSkips>0 implies sparse support is enabled */
         (void)prefs;   /* assert can be disabled, in which case prefs becomes unused */
@@ -1708,7 +1722,7 @@ FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
 
 /** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode
     @return : 0 (no error) */
-static int FIO_passThrough(FIO_prefs_t* const prefs,
+static int FIO_passThrough(const FIO_prefs_t* const prefs,
                            FILE* foutput, FILE* finput,
                            void* buffer, size_t bufferSize,
                            size_t alreadyLoaded)
@@ -1748,7 +1762,10 @@ static unsigned FIO_highbit64(unsigned long long v)
 
 /* FIO_zstdErrorHelp() :
  * detailed error message when requested window size is too large */
-static void FIO_zstdErrorHelp(FIO_prefs_t* const prefs, dRess_t* ress, size_t err, char const* srcFileName)
+static void
+FIO_zstdErrorHelp(const FIO_prefs_t* const prefs,
+                  const dRess_t* ress,
+                  size_t err, const char* srcFileName)
 {
     ZSTD_frameHeader header;
 
@@ -1780,12 +1797,10 @@ static void FIO_zstdErrorHelp(FIO_prefs_t* const prefs, dRess_t* ress, size_t er
  *  @return : size of decoded zstd frame, or an error code
 */
 #define FIO_ERROR_FRAME_DECODING   ((unsigned long long)(-2))
-static unsigned long long FIO_decompressZstdFrame(
-                                       FIO_prefs_t* const prefs,
-                                       dRess_t* ress,
-                                       FILE* finput,
-                                       const char* srcFileName,
-                                       U64 alreadyDecoded)
+static unsigned long long
+FIO_decompressZstdFrame(const FIO_prefs_t* const prefs,
+                        dRess_t* ress, FILE* finput,
+                        const char* srcFileName, U64 alreadyDecoded)
 {
     U64 frameSize = 0;
     U32 storedSkips = 0;
@@ -1849,13 +1864,16 @@ static unsigned long long FIO_decompressZstdFrame(
 
 
 #ifdef ZSTD_GZDECOMPRESS
-static unsigned long long FIO_decompressGzFrame(dRess_t* ress,
-                                    FILE* srcFile, const char* srcFileName)
+static unsigned long long
+FIO_decompressGzFrame(const FIO_prefs_t* const prefs,
+                      dRess_t* ress, FILE* srcFile,
+                      const char* srcFileName)
 {
     unsigned long long outFileSize = 0;
     z_stream strm;
     int flush = Z_NO_FLUSH;
     int decodingError = 0;
+    unsigned storedSkips = 0;
 
     strm.zalloc = Z_NULL;
     strm.zfree = Z_NULL;
@@ -1890,10 +1908,7 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress,
         }
         {   size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
             if (decompBytes) {
-                if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) {
-                    DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno));
-                    decodingError = 1; break;
-                }
+                storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decompBytes, storedSkips);
                 outFileSize += decompBytes;
                 strm.next_out = (Bytef*)ress->dstBuffer;
                 strm.avail_out = (uInt)ress->dstBufferSize;
@@ -1910,19 +1925,24 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress,
         DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName);
         decodingError = 1;
     }
+    FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips);
     return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize;
 }
 #endif
 
 
 #ifdef ZSTD_LZMADECOMPRESS
-static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName, int plain_lzma)
+static unsigned long long
+FIO_decompressLzmaFrame(const FIO_prefs_t* const prefs,
+                        dRess_t* ress, FILE* srcFile,
+                        const char* srcFileName, int plain_lzma)
 {
     unsigned long long outFileSize = 0;
     lzma_stream strm = LZMA_STREAM_INIT;
     lzma_action action = LZMA_RUN;
     lzma_ret initRet;
     int decodingError = 0;
+    unsigned storedSkips = 0;
 
     strm.next_in = 0;
     strm.avail_in = 0;
@@ -1965,10 +1985,7 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile,
         }
         {   size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
             if (decompBytes) {
-                if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) {
-                    DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno));
-                    decodingError = 1; break;
-                }
+                storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decompBytes, storedSkips);
                 outFileSize += decompBytes;
                 strm.next_out = (BYTE*)ress->dstBuffer;
                 strm.avail_out = ress->dstBufferSize;
@@ -1980,19 +1997,23 @@ static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile,
         memmove(ress->srcBuffer, strm.next_in, strm.avail_in);
     ress->srcBufferLoaded = strm.avail_in;
     lzma_end(&strm);
+    FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips);
     return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize;
 }
 #endif
 
 #ifdef ZSTD_LZ4DECOMPRESS
-static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
-                                    FILE* srcFile, const char* srcFileName)
+static unsigned long long
+FIO_decompressLz4Frame(const FIO_prefs_t* const prefs,
+                       dRess_t* ress, FILE* srcFile,
+                       const char* srcFileName)
 {
     unsigned long long filesize = 0;
     LZ4F_errorCode_t nextToLoad;
     LZ4F_decompressionContext_t dCtx;
     LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
     int decodingError = 0;
+    unsigned storedSkips = 0;
 
     if (LZ4F_isError(errorCode)) {
         DISPLAYLEVEL(1, "zstd: failed to create lz4 decompression context \n");
@@ -2036,10 +2057,7 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
 
             /* Write Block */
             if (decodedBytes) {
-                if (fwrite(ress->dstBuffer, 1, decodedBytes, ress->dstFile) != decodedBytes) {
-                    DISPLAYLEVEL(1, "zstd: fwrite error: %s \n", strerror(errno));
-                    decodingError = 1; nextToLoad = 0; break;
-                }
+                storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, decodedBytes, storedSkips);
                 filesize += decodedBytes;
                 DISPLAYUPDATE(2, "\rDecompressed : %u MB  ", (unsigned)(filesize>>20));
             }
@@ -2060,6 +2078,7 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
 
     LZ4F_freeDecompressionContext(dCtx);
     ress->srcBufferLoaded = 0; /* LZ4F will reach exact frame boundary */
+    FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips);
 
     return decodingError ? FIO_ERROR_FRAME_DECODING : filesize;
 }
@@ -2073,8 +2092,9 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
  * @return : 0 : OK
  *           1 : error
  */
-static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* srcFile,
-                        const char* dstFileName, const char* srcFileName)
+static int FIO_decompressFrames(const FIO_prefs_t* const prefs,
+                                dRess_t ress, FILE* srcFile,
+                          const char* dstFileName, const char* srcFileName)
 {
     unsigned readSomething = 0;
     unsigned long long filesize = 0;
@@ -2106,7 +2126,7 @@ static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* sr
             filesize += frameSize;
         } else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */
 #ifdef ZSTD_GZDECOMPRESS
-            unsigned long long const frameSize = FIO_decompressGzFrame(&ress, srcFile, srcFileName);
+            unsigned long long const frameSize = FIO_decompressGzFrame(prefs, &ress, srcFile, srcFileName);
             if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
             filesize += frameSize;
 #else
@@ -2116,7 +2136,7 @@ static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* sr
         } else if ((buf[0] == 0xFD && buf[1] == 0x37)  /* xz magic number */
                 || (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */
 #ifdef ZSTD_LZMADECOMPRESS
-            unsigned long long const frameSize = FIO_decompressLzmaFrame(&ress, srcFile, srcFileName, buf[0] != 0xFD);
+            unsigned long long const frameSize = FIO_decompressLzmaFrame(prefs, &ress, srcFile, srcFileName, buf[0] != 0xFD);
             if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
             filesize += frameSize;
 #else
@@ -2125,7 +2145,7 @@ static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* sr
 #endif
         } else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) {
 #ifdef ZSTD_LZ4DECOMPRESS
-            unsigned long long const frameSize = FIO_decompressLz4Frame(&ress, srcFile, srcFileName);
+            unsigned long long const frameSize = FIO_decompressLz4Frame(prefs, &ress, srcFile, srcFileName);
             if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
             filesize += frameSize;
 #else
@@ -2165,11 +2185,11 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs,
     int transfer_permissions = 0;
     int releaseDstFile = 0;
 
-    if (ress.dstFile == NULL) {
+    if ((ress.dstFile == NULL) && (prefs->testMode==0)) {
         releaseDstFile = 1;
 
         ress.dstFile = FIO_openDstFile(prefs, srcFileName, dstFileName);
-        if (ress.dstFile==0) return 1;
+        if (ress.dstFile==NULL) return 1;
 
         /* Must only be added after FIO_openDstFile() succeeds.
          * Otherwise we may delete the destination file if it already exists,
@@ -2182,7 +2202,6 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs,
             transfer_permissions = 1;
     }
 
-
     result = FIO_decompressFrames(prefs, ress, srcFile, dstFileName, srcFileName);
 
     if (releaseDstFile) {
diff --git a/programs/fileio.h b/programs/fileio.h
index 841e0a3c..cb3d64c5 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -74,6 +74,7 @@ void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable);
 void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize);
 void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize);
 void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint);
+void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode);
 void FIO_setLiteralCompressionMode(
         FIO_prefs_t* const prefs,
         ZSTD_literalCompressionMode_e mode);
@@ -85,14 +86,14 @@ void FIO_setNotificationLevel(int level);
 *  Single File functions
 ***************************************/
 /** FIO_compressFilename() :
-    @return : 0 == ok;  1 == pb with src file. */
+ * @return : 0 == ok;  1 == pb with src file. */
 int FIO_compressFilename (FIO_prefs_t* const prefs,
                           const char* outfilename, const char* infilename,
                           const char* dictFileName, int compressionLevel,
                           ZSTD_compressionParameters comprParams);
 
 /** FIO_decompressFilename() :
-    @return : 0 == ok;  1 == pb with src file. */
+ * @return : 0 == ok;  1 == pb with src file. */
 int FIO_decompressFilename (FIO_prefs_t* const prefs,
                             const char* outfilename, const char* infilename, const char* dictFileName);
 
@@ -103,7 +104,7 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
 *  Multiple File functions
 ***************************************/
 /** FIO_compressMultipleFilenames() :
-    @return : nb of missing files */
+ * @return : nb of missing files */
 int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
                                   const char** inFileNamesTable, unsigned nbFiles,
                                   const char* outDirName,
@@ -112,7 +113,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
                                   ZSTD_compressionParameters comprParams);
 
 /** FIO_decompressMultipleFilenames() :
-    @return : nb of missing or skipped files */
+ * @return : nb of missing or skipped files */
 int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
                                     const char** srcNamesTable, unsigned nbFiles,
                                     const char* outDirName,
@@ -120,10 +121,12 @@ int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
                                     const char* dictFileName);
 
 /* FIO_checkFilenameCollisions() :
- * Checks for and warns if thereå are any files that would have the same output path
+ * Checks for and warns if there are any files that would have the same output path
  */
 int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles);
 
+
+
 /*-*************************************
 *  Advanced stuff (should actually be hosted elsewhere)
 ***************************************/
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 4d1fad70..aed9ee86 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -1120,7 +1120,7 @@ int main(int argCount, const char* argv[])
     }
 
 #ifndef ZSTD_NODECOMPRESS
-    if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); }  /* test mode */
+    if (operation==zom_test) { FIO_setTestMode(prefs, 1); outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); }  /* test mode */
 #endif
 
     /* No input filename ==> use stdin and stdout */

From 0a24d4ef1847cb2665507c26ec072c4dc9064969 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 16:39:47 -0700
Subject: [PATCH 41/59] removed regular file test on Windows

since it does not work well on this platform
(tested with MinGW).

Note : could be an issue within UTIL_isRegularFile()
---
 programs/fileio.c | 6 ++++++
 programs/fileio.h | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 9efb4aae..85c04d48 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -538,6 +538,8 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
  * @result : FILE* to `dstFileName`, or NULL if it fails */
 static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName, const char* dstFileName)
 {
+    if (prefs->testMode) return NULL;  /* do not open file in test mode */
+
     assert(dstFileName != NULL);
     if (!strcmp (dstFileName, stdoutmark)) {
         DISPLAYLEVEL(4,"Using stdout for output \n");
@@ -562,10 +564,14 @@ static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName,
     if (UTIL_isRegularFile(dstFileName)) {
         /* Check if destination file already exists */
         FILE* const fCheck = fopen( dstFileName, "rb" );
+#if !defined(_WIN32)
+        /* this test does not work on Windows :
+         * `NUL` and `nul` are detected as regular files */
         if (!strcmp(dstFileName, nulmark)) {
             EXM_THROW(40, "%s is unexpectedly categorized as a regular file",
                         dstFileName);
         }
+#endif
         if (fCheck != NULL) {  /* dst file exists, authorization prompt */
             fclose(fCheck);
             if (!prefs->overwrite) {
diff --git a/programs/fileio.h b/programs/fileio.h
index cb3d64c5..4b0143be 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -26,7 +26,7 @@ extern "C" {
 #define stdinmark  "/*stdin*\\"
 #define stdoutmark "/*stdout*\\"
 #ifdef _WIN32
-#  define nulmark "nul"
+#  define nulmark "NUL"
 #else
 #  define nulmark "/dev/null"
 #endif

From caf40d0ae4e5b25bfc48921679d99935445833fe Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 17 Oct 2019 16:58:49 -0700
Subject: [PATCH 42/59] fix : no output file opened in test mode

also : redistributed error code within fileio.c
for more precise diagnosis.
---
 programs/fileio.c | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 85c04d48..948310e5 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -536,7 +536,9 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
 /** FIO_openDstFile() :
  *  condition : `dstFileName` must be non-NULL.
  * @result : FILE* to `dstFileName`, or NULL if it fails */
-static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName, const char* dstFileName)
+static FILE*
+FIO_openDstFile(FIO_prefs_t* const prefs,
+          const char* srcFileName, const char* dstFileName)
 {
     if (prefs->testMode) return NULL;  /* do not open file in test mode */
 
@@ -936,14 +938,14 @@ FIO_compressLzmaFrame(cRess_t* ress,
     if (plain_lzma) {
         lzma_options_lzma opt_lzma;
         if (lzma_lzma_preset(&opt_lzma, compressionLevel))
-            EXM_THROW(71, "zstd: %s: lzma_lzma_preset error", srcFileName);
+            EXM_THROW(81, "zstd: %s: lzma_lzma_preset error", srcFileName);
         ret = lzma_alone_encoder(&strm, &opt_lzma); /* LZMA */
         if (ret != LZMA_OK)
-            EXM_THROW(71, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret);
+            EXM_THROW(82, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret);
     } else {
         ret = lzma_easy_encoder(&strm, compressionLevel, LZMA_CHECK_CRC64); /* XZ */
         if (ret != LZMA_OK)
-            EXM_THROW(71, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret);
+            EXM_THROW(83, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret);
     }
 
     strm.next_in = 0;
@@ -963,11 +965,11 @@ FIO_compressLzmaFrame(cRess_t* ress,
         ret = lzma_code(&strm, action);
 
         if (ret != LZMA_OK && ret != LZMA_STREAM_END)
-            EXM_THROW(72, "zstd: %s: lzma_code encoding error %d", srcFileName, ret);
+            EXM_THROW(84, "zstd: %s: lzma_code encoding error %d", srcFileName, ret);
         {   size_t const compBytes = ress->dstBufferSize - strm.avail_out;
             if (compBytes) {
                 if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes)
-                    EXM_THROW(73, "Write error : %s", strerror(errno));
+                    EXM_THROW(85, "Write error : %s", strerror(errno));
                 outFileSize += compBytes;
                 strm.next_out = (BYTE*)ress->dstBuffer;
                 strm.avail_out = ress->dstBufferSize;
@@ -1657,7 +1659,7 @@ FIO_fwriteSparse(const FIO_prefs_t* const prefs,
     if (storedSkips > 1 GB) {
         int const seekResult = LONG_SEEK(file, 1 GB, SEEK_CUR);
         if (seekResult != 0)
-            EXM_THROW(71, "1 GB skip error (sparse file support)");
+            EXM_THROW(91, "1 GB skip error (sparse file support)");
         storedSkips -= 1 GB;
     }
 
@@ -1673,13 +1675,13 @@ FIO_fwriteSparse(const FIO_prefs_t* const prefs,
 
         if (nb0T != seg0SizeT) {   /* not all 0s */
             int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
-            if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse");
+            if (seekResult) EXM_THROW(92, "Sparse skip error ; try --no-sparse");
             storedSkips = 0;
             seg0SizeT -= nb0T;
             ptrT += nb0T;
             {   size_t const sizeCheck = fwrite(ptrT, sizeof(size_t), seg0SizeT, file);
                 if (sizeCheck != seg0SizeT)
-                    EXM_THROW(73, "Write error : cannot write decoded block : %s",
+                    EXM_THROW(93, "Write error : cannot write decoded block : %s",
                             strerror(errno));
         }   }
         ptrT += seg0SizeT;
@@ -1697,11 +1699,11 @@ FIO_fwriteSparse(const FIO_prefs_t* const prefs,
             if (restPtr != restEnd) {
                 int seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
                 if (seekResult)
-                    EXM_THROW(74, "Sparse skip error ; try --no-sparse");
+                    EXM_THROW(94, "Sparse skip error ; try --no-sparse");
                 storedSkips = 0;
                 {   size_t const sizeCheck = fwrite(restPtr, 1, (size_t)(restEnd - restPtr), file);
                     if (sizeCheck != (size_t)(restEnd - restPtr))
-                        EXM_THROW(75, "Write error : cannot write decoded end of block : %s",
+                        EXM_THROW(95, "Write error : cannot write decoded end of block : %s",
                             strerror(errno));
     }   }   }   }
 
@@ -2383,11 +2385,13 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
 
     if (outFileName) {
         unsigned u;
-        ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName);
-        if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName);
+        if (!prefs->testMode) {
+            ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName);
+            if (ress.dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName);
+        }
         for (u=0; utestMode) && (fclose(ress.dstFile)))
             EXM_THROW(72, "Write error : %s : cannot properly close output file",
                         strerror(errno));
     } else {

From ad86a5d0bc8b47bbd77a70c757eef5e008850c15 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 18 Oct 2019 11:15:10 -0700
Subject: [PATCH 43/59] rewrite FIO_createFilename_fromOutDir()

---
 programs/fileio.c | 62 +++++++++++++++++++++--------------------------
 tests/Makefile    |  3 ++-
 2 files changed, 30 insertions(+), 35 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 948310e5..a9075db1 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -686,56 +686,50 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
     return 0;
 }
 
+static const char*
+extractFilename(const char* path, char separator)
+{
+    const char* search = strrchr(path, separator);
+    if (search == NULL) return path;
+    return search+1;
+}
+
 /* FIO_createFilename_fromOutDir() :
  * Takes a source file name and specified output directory, and
  * allocates memory for and returns a pointer to final path.
  * This function never returns an error (it may abort() in case of pb)
  */
 static char*
-FIO_createFilename_fromOutDir(const char* srcFilename, const char* outDirName, const size_t suffixLen)
+FIO_createFilename_fromOutDir(const char* path, const char* outDirName, const size_t suffixLen)
 {
-    const char* c, *filenameBegin;
-    char* filename, *result;
-    size_t finalPathLen;
+    const char* filenameStart;
+    char separator;
+    char* result;
 
-    #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
-    c = "\\";
-    #else
-    c = "/";
-    #endif
+#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
+    separator = '\\';
+#else
+    separator = '/';
+#endif
 
-    finalPathLen = strlen(outDirName);
-    filenameBegin = strrchr(srcFilename, c[0]);
-    if (filenameBegin == NULL) {
-        filename = (char*) malloc((strlen(srcFilename)+1) * sizeof(char));
-        if (!filename) {
-            EXM_THROW(30, "zstd: %s", strerror(errno));
-        }
-        strcpy(filename, srcFilename);
-    } else {
-        filename = (char*) malloc((strlen(filenameBegin+1)+1) * sizeof(char));
-        if (!filename) {
-            EXM_THROW(30, "zstd: %s", strerror(errno));
-        }
-        strcpy(filename, filenameBegin+1);
-    }
+    filenameStart = extractFilename(path, separator);
+#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
+    filenameStart = extractFilename(filenameStart, '/');  /* sometimes, '/' separator is also used on Windows (mingw+msys2) */
+#endif
 
-    finalPathLen += strlen(filename);
-    result = (char*) malloc((finalPathLen+suffixLen+30) * sizeof(char));
+    result = (char*) calloc(1, strlen(outDirName) + 1 + strlen(filenameStart) + suffixLen + 1);
     if (!result) {
-        free(filename);
-        EXM_THROW(30, "zstd: %s", strerror(errno));
+        EXM_THROW(30, "zstd: FIO_createFilename_fromOutDir: %s", strerror(errno));
     }
 
-    strcpy(result, outDirName);
-    if (outDirName[strlen(outDirName)-1] == c[0]) {
-        strcat(result, filename);
+    memcpy(result, outDirName, strlen(outDirName));
+    if (outDirName[strlen(outDirName)-1] == separator) {
+        memcpy(result + strlen(outDirName), filenameStart, strlen(filenameStart));
     } else {
-        strcat(result, c);
-        strcat(result, filename);
+        memcpy(result + strlen(outDirName), &separator, 1);
+        memcpy(result + strlen(outDirName) + 1, filenameStart, strlen(filenameStart));
     }
 
-    free(filename);
     return result;
 }
 
diff --git a/tests/Makefile b/tests/Makefile
index bd2f9097..04d0b185 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -250,7 +250,8 @@ clean:
 	$(MAKE) -C $(ZSTDDIR) clean
 	$(MAKE) -C $(PRGDIR) clean
 	@$(RM) -fR $(TESTARTEFACT)
-	@$(RM) -f core *.o tmp* *.tmp result* *.gcda dictionary *.zst \
+	@$(RM) -rf tmp*  # some test directories are named tmp*
+	@$(RM) core *.o *.tmp result* *.gcda dictionary *.zst \
         $(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \
         fullbench$(EXT) fullbench32$(EXT) \
         fullbench-lib$(EXT) fullbench-dll$(EXT) \

From 8c11f089a1a68914f308ef83d3fbcc607d0b8262 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 18 Oct 2019 13:34:35 -0700
Subject: [PATCH 44/59] [fuzz] Increase output buffer size of stream_round_trip

Fixes OSS-Fuzz crash.
Credit to OSS-Fuzz
---
 tests/fuzz/stream_round_trip.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/fuzz/stream_round_trip.c b/tests/fuzz/stream_round_trip.c
index c534a904..703b1171 100644
--- a/tests/fuzz/stream_round_trip.c
+++ b/tests/fuzz/stream_round_trip.c
@@ -125,13 +125,14 @@ static size_t compress(uint8_t *dst, size_t capacity,
 
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 {
+    size_t neededBufSize;
+
     /* Give a random portion of src data to the producer, to use for
     parameter generation. The rest will be used for (de)compression */
     FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
     size = FUZZ_dataProducer_reserveDataPrefix(producer);
 
-    size_t neededBufSize;
-    neededBufSize = ZSTD_compressBound(size) * 5;
+    neededBufSize = ZSTD_compressBound(size) * 15;
 
     /* Allocate all buffers and contexts if not already allocated */
     if (neededBufSize > bufSize) {

From 29e46ed0bd1b206a3eb8c0f65a9c3399cfc6dfea Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 18 Oct 2019 14:28:34 -0700
Subject: [PATCH 45/59] fix test on windows

isDirectory() doesn't work on Windows
if directory name is followed by '/'
---
 programs/util.c    | 27 +++++++++++++--------------
 tests/playTests.sh |  4 ++--
 2 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/programs/util.c b/programs/util.c
index dbdd1cee..2249e451 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -328,7 +328,6 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
     unsigned i, nbFiles;
     char* buf = (char*)malloc(LIST_SIZE_INCREASE);
     char* bufend = buf + LIST_SIZE_INCREASE;
-    const char** fileTable;
 
     if (!buf) return NULL;
 
@@ -343,7 +342,7 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
                 if (!buf) return NULL;
             }
             if (buf + pos + len < bufend) {
-                memcpy(buf+pos, inputNames[i], len+1);  /* with final \0 */
+                memcpy(buf+pos, inputNames[i], len+1);  /* including final \0 */
                 pos += len + 1;
                 nbFiles++;
             }
@@ -354,20 +353,20 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
 
     if (nbFiles == 0) { free(buf); return NULL; }
 
-    fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*));
-    if (!fileTable) { free(buf); return NULL; }
+    {   const char** const fileTable = (const char**)malloc((nbFiles + 1) * sizeof(*fileTable));
+        if (!fileTable) { free(buf); return NULL; }
 
-    for (i=0, pos=0; i bufend) { free(buf); free((void*)fileTable); return NULL; }
+            pos += strlen(fileTable[i]) + 1;
+        }
+
+        *allocatedBuffer = buf;
+        *allocatedNamesNb = nbFiles;
+
+        return fileTable;
     }
-
-    if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; }
-
-    *allocatedBuffer = buf;
-    *allocatedNamesNb = nbFiles;
-
-    return fileTable;
 }
 
 
diff --git a/tests/playTests.sh b/tests/playTests.sh
index f63a7f69..80f36d52 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -240,7 +240,7 @@ rm tmp         # erase source file
 touch tmp.zst  # create destination file
 $ZSTD -f tmp && die "attempt to compress a non existing file"
 test -f tmp.zst  # destination file should still be present
-rm tmp*
+rm -rf tmp*  # may also erase tmp* directory from previous failed run
 
 println "\n===> decompression only tests "
 head -c 1048576 /dev/zero > tmp
@@ -284,7 +284,7 @@ test -f tmpOutDir/tmp1.zst
 test -f tmpOutDir/tmp2.zst
 println "test : decompress multiple files into an output directory, --output-dir-flat"
 mkdir tmpOutDirDecomp
-$ZSTD tmpOutDir/ -r -d --output-dir-flat tmpOutDirDecomp
+$ZSTD tmpOutDir -r -d --output-dir-flat tmpOutDirDecomp
 test -f tmpOutDirDecomp/tmp2
 test -f tmpOutDirDecomp/tmp1
 rm -rf tmp*

From 03ef7b73a7f1d61311d747f83e9f0daa7a350488 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 18 Oct 2019 14:52:16 -0700
Subject: [PATCH 46/59] attempt to run 'make check' tests on Appveyor

for mingw builds
---
 appveyor.yml | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/appveyor.yml b/appveyor.yml
index 35f019dd..9a3d5338 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -1,3 +1,7 @@
+# Following tests are run _only_ on master branch
+# To reproduce these tests, it's possible to push into a branch `appveyorTest`
+# or a branch `visual*`, they will intentionnally trigger `master` tests
+
 -
   version: 1.0.{build}
   branches:
@@ -176,6 +180,11 @@
       fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%
     )
 
+
+# The following tests are for regular pushes
+# into `dev` or some feature branch
+# There run less tests, for shorter feedback loop
+
 -
   version: 1.0.{build}
   environment:
@@ -249,3 +258,11 @@
       COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
       COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\
     )
+
+
+  test_script:
+  - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
+  - if [%HOST%]==[mingw] (
+      SET CC=%COMPILER%
+      make check
+    )

From 0e154fc40ea9767636ef5726806113296524df1d Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 18 Oct 2019 15:02:30 -0700
Subject: [PATCH 47/59] minor appveyor test script fix

---
 appveyor.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/appveyor.yml b/appveyor.yml
index 9a3d5338..87fa5c12 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -263,6 +263,6 @@
   test_script:
   - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
   - if [%HOST%]==[mingw] (
-      SET CC=%COMPILER%
+      set "CC=%COMPILER%" &&
       make check
     )

From 243824551f92258c148e77dd4e6e9406c9163c21 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 18 Oct 2019 12:33:45 -0700
Subject: [PATCH 48/59] [threading] Add debug utilities

---
 lib/common/pool.c      | 10 ++++++---
 lib/common/threading.c | 47 +++++++++++++++++++++++++++++++++++++++++-
 lib/common/threading.h | 33 ++++++++++++++++++++++++++++-
 tests/poolTests.c      |  2 +-
 4 files changed, 86 insertions(+), 6 deletions(-)

diff --git a/lib/common/pool.c b/lib/common/pool.c
index 7a829454..f5759350 100644
--- a/lib/common/pool.c
+++ b/lib/common/pool.c
@@ -127,9 +127,13 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
     ctx->queueTail = 0;
     ctx->numThreadsBusy = 0;
     ctx->queueEmpty = 1;
-    (void)ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL);
-    (void)ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL);
-    (void)ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL);
+    {
+        int error = 0;
+        error |= ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL);
+        error |= ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL);
+        error |= ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL);
+        if (error) { POOL_free(ctx); return NULL; }
+    }
     ctx->shutdown = 0;
     /* Allocate space for the thread handles */
     ctx->threads = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), customMem);
diff --git a/lib/common/threading.c b/lib/common/threading.c
index f3d4fa84..482664bd 100644
--- a/lib/common/threading.c
+++ b/lib/common/threading.c
@@ -14,6 +14,8 @@
  * This file will hold wrapper for systems, which do not support pthreads
  */
 
+#include "threading.h"
+
 /* create fake symbol to avoid empty translation unit warning */
 int g_ZSTD_threading_useless_symbol;
 
@@ -28,7 +30,6 @@ int g_ZSTD_threading_useless_symbol;
 /* ===  Dependencies  === */
 #include 
 #include 
-#include "threading.h"
 
 
 /* ===  Implementation  === */
@@ -73,3 +74,47 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
 }
 
 #endif   /* ZSTD_MULTITHREAD */
+
+#if defined(ZSTD_MULTITHREAD) && DEBUGLEVEL >= 1 && !defined(_WIN32)
+
+#include 
+
+int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr)
+{
+    *mutex = (pthread_mutex_t*)malloc(sizeof(pthread_mutex_t));
+    if (!*mutex)
+        return 1;
+    return pthread_mutex_init(*mutex, attr);
+}
+
+int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex)
+{
+    if (!*mutex)
+        return 0;
+    {
+        int const ret = pthread_mutex_destroy(*mutex);
+        free(*mutex);
+        return ret;
+    }
+}
+
+int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr)
+{
+    *cond = (pthread_cond_t*)malloc(sizeof(pthread_cond_t));
+    if (!*cond)
+        return 1;
+    return pthread_cond_init(*cond, attr);
+}
+
+int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond)
+{
+    if (!*cond)
+        return 0;
+    {
+        int const ret = pthread_cond_destroy(*cond);
+        free(*cond);
+        return ret;
+    }
+}
+
+#endif
diff --git a/lib/common/threading.h b/lib/common/threading.h
index d806c89d..3193ca7d 100644
--- a/lib/common/threading.h
+++ b/lib/common/threading.h
@@ -13,6 +13,8 @@
 #ifndef THREADING_H_938743
 #define THREADING_H_938743
 
+#include "debug.h"
+
 #if defined (__cplusplus)
 extern "C" {
 #endif
@@ -75,10 +77,12 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
  */
 
 
-#elif defined(ZSTD_MULTITHREAD)   /* posix assumed ; need a better detection method */
+#elif defined(ZSTD_MULTITHREAD)    /* posix assumed ; need a better detection method */
 /* ===   POSIX Systems   === */
 #  include 
 
+#if DEBUGLEVEL < 1
+
 #define ZSTD_pthread_mutex_t            pthread_mutex_t
 #define ZSTD_pthread_mutex_init(a, b)   pthread_mutex_init((a), (b))
 #define ZSTD_pthread_mutex_destroy(a)   pthread_mutex_destroy((a))
@@ -96,6 +100,33 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
 #define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
 #define ZSTD_pthread_join(a, b)         pthread_join((a),(b))
 
+#else /* DEBUGLEVEL >= 1 */
+
+/* Debug implementation of threading.
+ * In this implementation we use pointers for mutexes and condition variables.
+ * This way, if we forget to init/destroy them the program will crash or ASAN
+ * will report leaks.
+ */
+
+#define ZSTD_pthread_mutex_t            pthread_mutex_t*
+int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr);
+int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex);
+#define ZSTD_pthread_mutex_lock(a)      pthread_mutex_lock(*(a))
+#define ZSTD_pthread_mutex_unlock(a)    pthread_mutex_unlock(*(a))
+
+#define ZSTD_pthread_cond_t             pthread_cond_t*
+int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr);
+int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond);
+#define ZSTD_pthread_cond_wait(a, b)    pthread_cond_wait(*(a), *(b))
+#define ZSTD_pthread_cond_signal(a)     pthread_cond_signal(*(a))
+#define ZSTD_pthread_cond_broadcast(a)  pthread_cond_broadcast(*(a))
+
+#define ZSTD_pthread_t                  pthread_t
+#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
+#define ZSTD_pthread_join(a, b)         pthread_join((a),(b))
+
+#endif
+
 #else  /* ZSTD_MULTITHREAD not defined */
 /* No multithreading support */
 
diff --git a/tests/poolTests.c b/tests/poolTests.c
index 26d57fb5..02ec62af 100644
--- a/tests/poolTests.c
+++ b/tests/poolTests.c
@@ -46,7 +46,7 @@ static int testOrder(size_t numThreads, size_t queueSize)
   POOL_ctx* const ctx = POOL_create(numThreads, queueSize);
   ASSERT_TRUE(ctx);
   data.i = 0;
-  (void)ZSTD_pthread_mutex_init(&data.mutex, NULL);
+  ASSERT_FALSE(ZSTD_pthread_mutex_init(&data.mutex, NULL));
   { size_t i;
     for (i = 0; i < 16; ++i) {
       POOL_add(ctx, &fn, &data);

From 0bc39bc3a030d7d40e71cff0b9afe98d170d342f Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 18 Oct 2019 10:59:15 -0700
Subject: [PATCH 49/59] [zstdmt] Don't memset the jobDescription

---
 lib/compress/zstdmt_compress.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 44cbd94b..bc3062b5 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -927,12 +927,18 @@ static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx)
     unsigned jobID;
     DEBUGLOG(3, "ZSTDMT_releaseAllJobResources");
     for (jobID=0; jobID <= mtctx->jobIDMask; jobID++) {
+        /* Copy the mutex/cond out */
+        ZSTD_pthread_mutex_t const mutex = mtctx->jobs[jobID].job_mutex;
+        ZSTD_pthread_cond_t const cond = mtctx->jobs[jobID].job_cond;
+
         DEBUGLOG(4, "job%02u: release dst address %08X", jobID, (U32)(size_t)mtctx->jobs[jobID].dstBuff.start);
         ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].dstBuff);
-        mtctx->jobs[jobID].dstBuff = g_nullBuffer;
-        mtctx->jobs[jobID].cSize = 0;
+
+        /* Clear the job description, but keep the mutex/cond */
+        memset(&mtctx->jobs[jobID], 0, sizeof(mtctx->jobs[jobID]));
+        mtctx->jobs[jobID].job_mutex = mutex;
+        mtctx->jobs[jobID].job_cond = cond;
     }
-    memset(mtctx->jobs, 0, (mtctx->jobIDMask+1)*sizeof(ZSTDMT_jobDescription));
     mtctx->inBuff.buffer = g_nullBuffer;
     mtctx->inBuff.filled = 0;
     mtctx->allJobsCompleted = 1;

From 85a016ed92d7c7df1cbb2fc84db31ba4b2c36a82 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 18 Oct 2019 15:21:50 -0700
Subject: [PATCH 50/59] made `make check` faster

for shorter feedback loop
---
 tests/playTests.sh | 33 +++++++++++++++------------------
 1 file changed, 15 insertions(+), 18 deletions(-)

diff --git a/tests/playTests.sh b/tests/playTests.sh
index 80f36d52..c74d548b 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -212,8 +212,8 @@ $ZSTD tmp -c --no-compress-literals -19      | $ZSTD -t
 $ZSTD tmp -c --compress-literals    -1       | $ZSTD -t
 $ZSTD tmp -c --compress-literals    --fast=1 | $ZSTD -t
 $ZSTD tmp -c --compress-literals    -19      | $ZSTD -t
-$ZSTD -b --fast=1 -i1e1 tmp --compress-literals
-$ZSTD -b --fast=1 -i1e1 tmp --no-compress-literals
+$ZSTD -b --fast=1 -i0e1 tmp --compress-literals
+$ZSTD -b --fast=1 -i0e1 tmp --no-compress-literals
 
 println "test : file removal"
 $ZSTD -f --rm tmp
@@ -578,30 +578,27 @@ println "- Create dictionary with short dictID"
 $ZSTD --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
 println "- Create dictionaries with shrink-dict flag enabled"
-$ZSTD --train-fastcover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict
-$ZSTD --train-fastcover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1
-$ZSTD --train-fastcover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2
+$ZSTD --train-fastcover=steps=1,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict
+$ZSTD --train-fastcover=steps=1,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1
+$ZSTD --train-fastcover=steps=1,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2
 println "- Create dictionary with size limit"
-$ZSTD --train-fastcover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
-println "- Compare size of dictionary from 90% training samples with 80% training samples"
-$ZSTD --train-fastcover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
-$ZSTD --train-fastcover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
+$ZSTD --train-fastcover=steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
 println "- Create dictionary using all samples for both training and testing"
-$ZSTD --train-fastcover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
+$ZSTD --train-fastcover=k=56,d=8,split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
 println "- Create dictionary using f=16"
-$ZSTD --train-fastcover=f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
-$ZSTD --train-fastcover=accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15"
+$ZSTD --train-fastcover=k=56,d=8,f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
+$ZSTD --train-fastcover=k=56,d=8,accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15"
 println "- Create dictionary using accel=2"
-$ZSTD --train-fastcover=accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
+$ZSTD --train-fastcover=k=56,d=8,accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
 println "- Create dictionary using accel=10"
-$ZSTD --train-fastcover=accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
+$ZSTD --train-fastcover=k=56,d=8,accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
 println "- Create dictionary with multithreading"
 $ZSTD --train-fastcover -T4 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
 println "- Test -o before --train-fastcover"
 rm -f tmpDict dictionary
-$ZSTD -o tmpDict --train-fastcover "$TESTDIR"/*.c "$PRGDIR"/*.c
+$ZSTD -o tmpDict --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c
 test -f tmpDict
-$ZSTD --train-fastcover "$TESTDIR"/*.c "$PRGDIR"/*.c
+$ZSTD --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c
 test -f dictionary
 rm tmp* dictionary
 
@@ -675,10 +672,10 @@ $ZSTD -i0b0e3 tmp1
 println "bench negative level"
 $ZSTD -bi0 --fast tmp1
 println "with recursive and quiet modes"
-$ZSTD -rqi1b1e2 tmp1
+$ZSTD -rqi0b1e2 tmp1
 println "benchmark decompression only"
 $ZSTD -f tmp1
-$ZSTD -b -d -i1 tmp1.zst
+$ZSTD -b -d -i0 tmp1.zst
 
 println "\n===>  zstd compatibility tests "
 

From 2ad75bb574de8fbd857d220a56960e2e25b3d611 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 18 Oct 2019 15:37:07 -0700
Subject: [PATCH 51/59] validated 'make test' for mingw environment

---
 tests/Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/Makefile b/tests/Makefile
index 04d0b185..161c823e 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -265,7 +265,7 @@ clean:
 
 
 #----------------------------------------------------------------------------------
-#make valgrindTest is validated only for Linux, macOS, BSD, Hurd and Solaris targets
+# valgrind tests are validated only for some posix platforms
 #----------------------------------------------------------------------------------
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
 HOST_OS = POSIX
@@ -287,7 +287,7 @@ valgrindTest: zstd datagen fuzzer fullbench
 endif
 
 
-ifneq (,$(filter MSYS%,$(shell uname)))
+ifneq (,$(filter MINGW% MSYS%,$(shell uname)))
 HOST_OS = MSYS
 endif
 

From 58c59341d34891778df3d3ce7a5606b79e764563 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 18 Oct 2019 15:41:12 -0700
Subject: [PATCH 52/59] updated windows binary resources

---
 programs/windres/zstd32.res | Bin 1044 -> 1044 bytes
 programs/windres/zstd64.res | Bin 1044 -> 1044 bytes
 2 files changed, 0 insertions(+), 0 deletions(-)

diff --git a/programs/windres/zstd32.res b/programs/windres/zstd32.res
index 843499254924c1c45ed155677538d8856721f527..9984215b5430ce5ea4f91c9e12426579f5dcd0a3 100644
GIT binary patch
delta 47
tcmbQjF@(2WJ-oIC$D7A1^~*X2-W}q

delta 47
scmbQjF@(2WJ-oIC$D7A1^~*X2-W}q

delta 47
scmbQjF@
Date: Fri, 18 Oct 2019 15:45:31 -0700
Subject: [PATCH 53/59] fix function cast warning on Windows with gcc9

---
 programs/util.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/programs/util.c b/programs/util.c
index 2249e451..c82d04e3 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -399,8 +399,8 @@ int UTIL_countPhysicalCores(void)
         DWORD returnLength = 0;
         size_t byteOffset = 0;
 
-        glpi = (LPFN_GLPI)GetProcAddress(GetModuleHandle(TEXT("kernel32")),
-                                         "GetLogicalProcessorInformation");
+        glpi = (LPFN_GLPI)(void*)GetProcAddress(GetModuleHandle(TEXT("kernel32")),
+                                               "GetLogicalProcessorInformation");
 
         if (glpi == NULL) {
             goto failed;

From f3796370259059657048350d22f446f5e5e2525e Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 18 Oct 2019 17:05:42 -0700
Subject: [PATCH 54/59] removed Visual warning

for pointer casts
---
 programs/util.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/programs/util.c b/programs/util.c
index c82d04e3..de25bc23 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -399,6 +399,11 @@ int UTIL_countPhysicalCores(void)
         DWORD returnLength = 0;
         size_t byteOffset = 0;
 
+#ifdef (_MSC_VER)
+/* Visual Studio does not like the following cast */
+#   pragma warning( disable : 4054 )  /* conversion from function ptr to data ptr */
+#   pragma warning( disable : 4055 )  /* conversion from data ptr to function ptr */
+#endif
         glpi = (LPFN_GLPI)(void*)GetProcAddress(GetModuleHandle(TEXT("kernel32")),
                                                "GetLogicalProcessorInformation");
 

From 0492c570139d252eaa47cacd788317d3f66a9c6e Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 18 Oct 2019 17:08:52 -0700
Subject: [PATCH 55/59] fixed visual defined test

---
 programs/util.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/util.c b/programs/util.c
index de25bc23..e1aa447a 100644
--- a/programs/util.c
+++ b/programs/util.c
@@ -399,7 +399,7 @@ int UTIL_countPhysicalCores(void)
         DWORD returnLength = 0;
         size_t byteOffset = 0;
 
-#ifdef (_MSC_VER)
+#if defined(_MSC_VER)
 /* Visual Studio does not like the following cast */
 #   pragma warning( disable : 4054 )  /* conversion from function ptr to data ptr */
 #   pragma warning( disable : 4055 )  /* conversion from data ptr to function ptr */

From ca73c218be4793acef114ad6154cca772b3540c5 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 18 Oct 2019 17:22:45 -0700
Subject: [PATCH 56/59] added mode documentation on ZSTD_CLEVEL

providing range restriction [1-19] explicitly
partially answering #1829
---
 programs/README.md  |  9 ++++++---
 programs/zstd.1     |  5 ++++-
 programs/zstd.1.md  | 10 ++++++++++
 programs/zstdgrep.1 |  2 +-
 programs/zstdless.1 |  2 +-
 5 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/programs/README.md b/programs/README.md
index c3a5590d..7668d49a 100644
--- a/programs/README.md
+++ b/programs/README.md
@@ -173,10 +173,13 @@ Benchmark arguments :
 ```
 
 #### Restricted usage of Environment Variables
-Using environment variables to set compression/decompression parameters has security implications. Therefore,
-we intentionally restrict its usage. Currently, only `ZSTD_CLEVEL` is supported for setting compression level.
+Using environment variables to set parameters has security implications.
+Therefore, this avenue is intentionally restricted.
+Only `ZSTD_CLEVEL` is supported currently, for setting compression level.
+`ZSTD_CLEVEL` can be used to set the level between 1 and 19 (the "normal" range).
 If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a warning message.
-Note that command line options will override corresponding environment variable settings.
+`ZSTD_CLEVEL` just replaces the default compression level (`3`).
+It can be overridden by corresponding command line arguments.
 
 #### Long distance matching mode
 The long distance matching mode, enabled with `--long`, is designed to improve
diff --git a/programs/zstd.1 b/programs/zstd.1
index bb5103c6..1072c323 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -1,5 +1,5 @@
 .
-.TH "ZSTD" "1" "September 2019" "zstd 1.4.4" "User Commands"
+.TH "ZSTD" "1" "October 2019" "zstd 1.4.4" "User Commands"
 .
 .SH "NAME"
 \fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@@ -206,6 +206,9 @@ add integrity check computed from uncompressed data (default: enabled)
 \fB\-\-\fR
 All arguments after \fB\-\-\fR are treated as files
 .
+.SS "Restricted usage of Environment Variables"
+Using environment variables to set parameters has security implications\. Therefore, this avenue is intentionally restricted\. Only \fBZSTD_CLEVEL\fR is supported currently, for setting compression level\. \fBZSTD_CLEVEL\fR can be used to set the level between 1 and 19 (the "normal" range)\. If the value of \fBZSTD_CLEVEL\fR is not a valid integer, it will be ignored with a warning message\. \fBZSTD_CLEVEL\fR just replaces the default compression level (\fB3\fR)\. It can be overridden by corresponding command line arguments\.
+.
 .SH "DICTIONARY BUILDER"
 \fBzstd\fR offers \fIdictionary\fR compression, which greatly improves efficiency on small files and messages\. It\'s possible to train \fBzstd\fR with a set of samples, the result of which is saved into a file called a \fBdictionary\fR\. Then during compression and decompression, reference the same dictionary, using command \fB\-D dictionaryFileName\fR\. Compression of small files similar to the sample set will be greatly improved\.
 .
diff --git a/programs/zstd.1.md b/programs/zstd.1.md
index dff4d9ea..c1f321bc 100644
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@@ -214,6 +214,16 @@ the last one takes effect.
 * `--`:
     All arguments after `--` are treated as files
 
+### Restricted usage of Environment Variables
+
+Using environment variables to set parameters has security implications.
+Therefore, this avenue is intentionally restricted.
+Only `ZSTD_CLEVEL` is supported currently, for setting compression level.
+`ZSTD_CLEVEL` can be used to set the level between 1 and 19 (the "normal" range).
+If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a warning message.
+`ZSTD_CLEVEL` just replaces the default compression level (`3`).
+It can be overridden by corresponding command line arguments.
+
 
 DICTIONARY BUILDER
 ------------------
diff --git a/programs/zstdgrep.1 b/programs/zstdgrep.1
index 06927ab7..b97f8cab 100644
--- a/programs/zstdgrep.1
+++ b/programs/zstdgrep.1
@@ -1,5 +1,5 @@
 .
-.TH "ZSTDGREP" "1" "September 2019" "zstd 1.4.4" "User Commands"
+.TH "ZSTDGREP" "1" "October 2019" "zstd 1.4.4" "User Commands"
 .
 .SH "NAME"
 \fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files
diff --git a/programs/zstdless.1 b/programs/zstdless.1
index d4904227..1ecc8bdc 100644
--- a/programs/zstdless.1
+++ b/programs/zstdless.1
@@ -1,5 +1,5 @@
 .
-.TH "ZSTDLESS" "1" "September 2019" "zstd 1.4.4" "User Commands"
+.TH "ZSTDLESS" "1" "October 2019" "zstd 1.4.4" "User Commands"
 .
 .SH "NAME"
 \fBzstdless\fR \- view zstandard\-compressed files

From ff7bd16c0a3676e3d87063a3355eff0c9cc4f090 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 18 Oct 2019 17:48:12 -0700
Subject: [PATCH 57/59] clarifications for the FSE decoding table

requested in #1782
---
 doc/zstd_compression_format.md | 58 ++++++++++++++++++----------------
 1 file changed, 30 insertions(+), 28 deletions(-)

diff --git a/doc/zstd_compression_format.md b/doc/zstd_compression_format.md
index 111dd98a..90ac0fe9 100644
--- a/doc/zstd_compression_format.md
+++ b/doc/zstd_compression_format.md
@@ -16,7 +16,7 @@ Distribution of this document is unlimited.
 
 ### Version
 
-0.3.3 (16/08/19)
+0.3.4 (16/08/19)
 
 
 Introduction
@@ -1107,18 +1107,18 @@ It follows the following build rule :
 
 The table has a size of `Table_Size = 1 << Accuracy_Log`.
 Each cell describes the symbol decoded,
-and instructions to get the next state.
+and instructions to get the next state (`Number_of_Bits` and `Baseline`).
 
 Symbols are scanned in their natural order for "less than 1" probabilities.
 Symbols with this probability are being attributed a single cell,
 starting from the end of the table and retreating.
 These symbols define a full state reset, reading `Accuracy_Log` bits.
 
-All remaining symbols are allocated in their natural order.
-Starting from symbol `0` and table position `0`,
+Then, all remaining symbols, sorted in natural order, are allocated cells.
+Starting from symbol `0` (if it exists), and table position `0`,
 each symbol gets allocated as many cells as its probability.
 Cell allocation is spreaded, not linear :
-each successor position follow this rule :
+each successor position follows this rule :
 
 ```
 position += (tableSize>>1) + (tableSize>>3) + 3;
@@ -1130,40 +1130,41 @@ A position is skipped if already occupied by a "less than 1" probability symbol.
 each position in the table, switching to the next symbol when enough
 states have been allocated to the current one.
 
-The result is a list of state values.
-Each state will decode the current symbol.
+The process guarantees that the table is entirely filled.
+Each cell corresponds to a state value, which contains the symbol being decoded.
 
-To get the `Number_of_Bits` and `Baseline` required for next state,
-it's first necessary to sort all states in their natural order.
-The lower states will need 1 more bit than higher ones.
+To add the `Number_of_Bits` and `Baseline` required to retrieve next state,
+it's first necessary to sort all occurrences of each symbol in state order.
+Lower states will need 1 more bit than higher ones.
 The process is repeated for each symbol.
 
 __Example__ :
-Presuming a symbol has a probability of 5.
-It receives 5 state values. States are sorted in natural order.
+Presuming a symbol has a probability of 5,
+it receives 5 cells, corresponding to 5 state values.
+These state values are then sorted in natural order.
 
-Next power of 2 is 8.
-Space of probabilities is divided into 8 equal parts.
-Presuming the `Accuracy_Log` is 7, it defines 128 states.
+Next power of 2 after 5 is 8.
+Space of probabilities must be divided into 8 equal parts.
+Presuming the `Accuracy_Log` is 7, it defines a space of 128 states.
 Divided by 8, each share is 16 large.
 
-In order to reach 8, 8-5=3 lowest states will count "double",
-doubling the number of shares (32 in width),
-requiring one more bit in the process.
+In order to reach 8 shares, 8-5=3 lowest states will count "double",
+doubling their shares (32 in width), hence requiring one more bit.
 
 Baseline is assigned starting from the higher states using fewer bits,
-and proceeding naturally, then resuming at the first state,
-each takes its allocated width from Baseline.
+increasing at each state, then resuming at the first state,
+each state takes its allocated width from Baseline.
 
-| state order      |   0   |   1   |    2   |   3  |   4   |
-| ---------------- | ----- | ----- | ------ | ---- | ----- |
-| width            |  32   |  32   |   32   |  16  |  16   |
-| `Number_of_Bits` |   5   |   5   |    5   |   4  |   4   |
-| range number     |   2   |   4   |    6   |   0  |   1   |
-| `Baseline`       |  32   |  64   |   96   |   0  |  16   |
-| range            | 32-63 | 64-95 | 96-127 | 0-15 | 16-31 |
+| state value      |   1   |  39   |   77   |  84  |  122   |
+| state order      |   0   |   1   |    2   |   3  |    4   |
+| ---------------- | ----- | ----- | ------ | ---- | ------ |
+| width            |  32   |  32   |   32   |  16  |   16   |
+| `Number_of_Bits` |   5   |   5   |    5   |   4  |    4   |
+| range number     |   2   |   4   |    6   |   0  |    1   |
+| `Baseline`       |  32   |  64   |   96   |   0  |   16   |
+| range            | 32-63 | 64-95 | 96-127 | 0-15 | 16-31  |
 
-The next state is determined from current state
+During decoding, the next state value is determined from current state value,
 by reading the required `Number_of_Bits`, and adding the specified `Baseline`.
 
 See [Appendix A] for the results of this process applied to the default distributions.
@@ -1657,6 +1658,7 @@ or at least provide a meaningful error code explaining for which reason it canno
 
 Version changes
 ---------------
+- 0.3.4 : clarifications for FSE decoding table
 - 0.3.3 : clarifications for field Block_Size
 - 0.3.2 : remove additional block size restriction on compressed blocks
 - 0.3.1 : minor clarification regarding offset history update rules

From 632e07747c1a4ebf419f1b84b7b2358e5987ce26 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 21 Oct 2019 12:14:59 -0700
Subject: [PATCH 58/59] minor refactor of FIO_compressGzFrame()

mostly narrowing variables lifetime and mutability.
---
 programs/fileio.c | 54 +++++++++++++++++++++++------------------------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index a9075db1..f4384484 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -834,13 +834,12 @@ static void FIO_freeCResources(cRess_t ress)
 
 #ifdef ZSTD_GZCOMPRESS
 static unsigned long long
-FIO_compressGzFrame(cRess_t* ress,
+FIO_compressGzFrame(const cRess_t* ress,  /* buffers & handlers are used, but not changed */
                     const char* srcFileName, U64 const srcFileSize,
                     int compressionLevel, U64* readsize)
 {
     unsigned long long inFileSize = 0, outFileSize = 0;
     z_stream strm;
-    int ret;
 
     if (compressionLevel > Z_BEST_COMPRESSION)
         compressionLevel = Z_BEST_COMPRESSION;
@@ -849,11 +848,12 @@ FIO_compressGzFrame(cRess_t* ress,
     strm.zfree = Z_NULL;
     strm.opaque = Z_NULL;
 
-    ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED,
+    {   int const ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED,
                         15 /* maxWindowLogSize */ + 16 /* gzip only */,
                         8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */
-    if (ret != Z_OK)
-        EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret);
+        if (ret != Z_OK) {
+            EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret);
+    }   }
 
     strm.next_in = 0;
     strm.avail_in = 0;
@@ -861,6 +861,7 @@ FIO_compressGzFrame(cRess_t* ress,
     strm.avail_out = (uInt)ress->dstBufferSize;
 
     while (1) {
+        int ret;
         if (strm.avail_in == 0) {
             size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile);
             if (inSize == 0) break;
@@ -871,32 +872,31 @@ FIO_compressGzFrame(cRess_t* ress,
         ret = deflate(&strm, Z_NO_FLUSH);
         if (ret != Z_OK)
             EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret);
-        {   size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
-            if (decompBytes) {
-                if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes)
-                    EXM_THROW(73, "Write error : cannot write to output file : %s", strerror(errno));
-                outFileSize += decompBytes;
+        {   size_t const cSize = ress->dstBufferSize - strm.avail_out;
+            if (cSize) {
+                if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize)
+                    EXM_THROW(73, "Write error : cannot write to output file : %s ", strerror(errno));
+                outFileSize += cSize;
                 strm.next_out = (Bytef*)ress->dstBuffer;
                 strm.avail_out = (uInt)ress->dstBufferSize;
-            }
-        }
-        if (srcFileSize == UTIL_FILESIZE_UNKNOWN)
-            DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%",
+        }   }
+        if (srcFileSize == UTIL_FILESIZE_UNKNOWN) {
+            DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ",
                             (unsigned)(inFileSize>>20),
                             (double)outFileSize/inFileSize*100)
-        else
-            DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%",
+        } else {
+            DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%% ",
                             (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20),
                             (double)outFileSize/inFileSize*100);
-    }
+    }   }
 
     while (1) {
-        ret = deflate(&strm, Z_FINISH);
-        {   size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
-            if (decompBytes) {
-                if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes)
-                    EXM_THROW(75, "Write error : %s", strerror(errno));
-                outFileSize += decompBytes;
+        int const ret = deflate(&strm, Z_FINISH);
+        {   size_t const cSize = ress->dstBufferSize - strm.avail_out;
+            if (cSize) {
+                if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize)
+                    EXM_THROW(75, "Write error : %s ", strerror(errno));
+                outFileSize += cSize;
                 strm.next_out = (Bytef*)ress->dstBuffer;
                 strm.avail_out = (uInt)ress->dstBufferSize;
         }   }
@@ -905,11 +905,11 @@ FIO_compressGzFrame(cRess_t* ress,
             EXM_THROW(77, "zstd: %s: deflate error %d \n", srcFileName, ret);
     }
 
-    ret = deflateEnd(&strm);
-    if (ret != Z_OK)
-        EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret);
+    {   int const ret = deflateEnd(&strm);
+        if (ret != Z_OK) {
+            EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret);
+    }   }
     *readsize = inFileSize;
-
     return outFileSize;
 }
 #endif

From bad35bd307fd9dafa1c675e56ed3abafb06af522 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 21 Oct 2019 12:55:39 -0700
Subject: [PATCH 59/59] turned off zlib tests during msan

since the local zlib library is not msan-instrumented
---
 .travis.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 20fa9bee..a6e1a99e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -54,7 +54,10 @@ matrix:
     - name: Trusty (clang-3.8 + MSan + Test Zstd)
       script:
         - make clang38install
-        - CC=clang-3.8 make clean msan-test-zstd
+        # External libraries must be turned off when using MSAN tests,
+        # because they are not msan-instrumented,
+        # so any data coming from these libraries is always considered "uninitialized"
+        - CC=clang-3.8 make clean msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0
 
     - name: Trusty (Minimal Decompressor Macros)
       script: