Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev

2017-03-18 11:16:21 +01:00 · 2017-03-18 11:16:21 +01:00 · beaeccc294
commit beaeccc294
parent 5c1c80cbb6 8a7f416871
56 changed files with 1822 additions and 779 deletions
--- a/.buckconfig
+++ b/.buckconfig
@ -1,8 +1,8 @@
 [cxx]
-  cppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=1
+  cppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4
  cflags = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wpointer-arith
-  cxxppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=1
-  cxxflags = -std=c++11 -Wno-format-security -Wno-deprecated-declarations
+  cxxppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4
+  cxxflags = -std=c++11 -Wno-deprecated-declarations
  gtest_dep = //contrib/pzstd:gtest

 [httpserver]
--- a/.travis.yml
+++ b/.travis.yml
@ -1,39 +1,39 @@
+# Medium Tests: Run on all commits/PRs to dev branch
+
 language: c
 sudo: required
 dist: trusty
 matrix:
-  fast_finish: true
  include:
    # Ubuntu 14.04
-    - env: Cmd="make libc6install && make -C tests test32"
-    - env: Cmd='make valgrindinstall arminstall ppcinstall arm-ppc-compilation && make clean lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest'
+    - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-test-zstd'
+    - env: Cmd='make gcc6install libc6install && CC=gcc-6 make clean uasan-test-zstd32'
+    - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd'

-    - env: Cmd='CC=gcc-6 make gcc6install uasan-test'
-    - env: Cmd='CC=gcc-6 make gcc6install uasan-test32'
-    - env: Cmd="make arminstall armtest && make clean && make aarch64test"
-    - env: Cmd='make ppcinstall ppctest && make clean && make ppc64test'
-    - env: Cmd='make gpp6install zlibwrapper && make -C tests clean test-zstd-nolegacy && make -C tests versionsTest && make clean && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan'
-      install:
-        - export CXX="g++-6" CC="gcc-6"
+    - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest'
+    - env: Cmd='make gcc6install libc6install && CC=gcc-6 CFLAGS=-m32 make clean uasan-fuzztest'
+    - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-fuzztest'
+    - env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream'

-    # OS X Mavericks
-    - env: Cmd="make gnu90test && make clean && make test && make clean && make travis-install"
-      os: osx
+    - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest'
+
+    - env: Cmd='make arminstall && make armfuzz'
+    - env: Cmd='make arminstall && make aarch64fuzz'
+    - env: Cmd='make ppcinstall && make ppcfuzz'
+    - env: Cmd='make ppcinstall && make ppc64fuzz'
+
+git:
+  depth: 1
+
+branches:
+  only:
+  - dev
+  - master

 script:
  - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:')
-  #  cron & master          => full tests, as this is the final step towards a Release
-  #  pull requests          => normal tests (job numbers 1-3)
-  #  other feature branches => short tests (job numbers 1-2)
  - echo JOB_NUMBER=$JOB_NUMBER TRAVIS_BRANCH=$TRAVIS_BRANCH TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST
-  - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "master" ]; then
-        FUZZERTEST=-T7mn sh -c "$Cmd" || travis_terminate 1;
-    else
-        if [ "$TRAVIS_EVENT_TYPE" = "pull_request" ] && [ $JOB_NUMBER -lt 4 ]; then
-            sh -c "$Cmd" || travis_terminate 1;
-        else
-            if [ $JOB_NUMBER -lt 3 ]; then
-                sh -c "$Cmd" || travis_terminate 1;
-            fi
-        fi
-    fi
+  - export FUZZERTEST=-T2mn;
+    export ZSTREAM_TESTTIME=-T2mn;
+    export DECODECORPUS_TESTTIME=-T1mn;
+    sh -c "$Cmd" || travis_terminate 1;
--- a/118
+++ b/118
@ -23,7 +23,7 @@ EXT =
 endif

 .PHONY: default
-default: lib zstd-release
+default: lib-release zstd-release

 .PHONY: all
 all: | allmost examples manual
@ -35,6 +35,13 @@ allmost:
 	$(MAKE) -C $(TESTDIR) all
 	$(MAKE) -C $(ZWRAPDIR) all

+#skip zwrapper, can't build that on alternate architectures without the proper zlib installed
+.PHONY: allarch
+allarch:
+	$(MAKE) -C $(ZSTDDIR) all
+	$(MAKE) -C $(PRGDIR) all
+	$(MAKE) -C $(TESTDIR) all
+
 .PHONY: all32
 all32:
 	$(MAKE) -C $(PRGDIR) zstd32
@ -42,6 +49,10 @@ all32:

 .PHONY: lib
 lib:
+	@$(MAKE) -C $(ZSTDDIR) $@
+
+.PHONY: lib-release
+lib-release:
 	@$(MAKE) -C $(ZSTDDIR)

 .PHONY: zstd
@ -63,6 +74,10 @@ zstdmt:
 zlibwrapper:
 	$(MAKE) -C $(ZWRAPDIR) test

+.PHONY: shortest
+shortest:
+	$(MAKE) -C $(TESTDIR) $@
+
 .PHONY: test
 test:
 	$(MAKE) -C $(TESTDIR) $@
@ -86,7 +101,6 @@ clean:
 	@$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
 	@echo Cleaning completed

-
 #------------------------------------------------------------------------------
 # make install is validated only for Linux, OSX, Hurd and some BSD targets
 #------------------------------------------------------------------------------
@ -105,9 +119,53 @@ uninstall:
 travis-install:
 	$(MAKE) install PREFIX=~/install_test_dir

-gpptest: clean
+gppbuild: clean
+	g++ -v
 	CC=g++ $(MAKE) -C programs all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"

+gcc5build: clean
+	gcc-5 -v
+	CC=gcc-5 $(MAKE) all MOREFLAGS="-Werror"
+
+gcc6build: clean
+	gcc-6 -v
+	CC=gcc-6 $(MAKE) all MOREFLAGS="-Werror"
+
+clangbuild: clean
+	clang -v
+	CXX=clang++ CC=clang $(MAKE) all MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
+
+m32build: clean
+	gcc -v
+	$(MAKE) all32
+
+armbuild: clean
+	CC=arm-linux-gnueabi-gcc CFLAGS="-Werror" $(MAKE) allarch
+
+aarch64build: clean
+	CC=aarch64-linux-gnu-gcc CFLAGS="-Werror" $(MAKE) allarch
+
+ppcbuild: clean
+	CC=powerpc-linux-gnu-gcc CLAGS="-m32 -Wno-attributes -Werror" $(MAKE) allarch
+
+ppc64build: clean
+	CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allarch
+
+armfuzz: clean
+	CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest
+
+aarch64fuzz: clean
+	CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest
+
+ppcfuzz: clean
+	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest
+
+ppc64fuzz: clean
+	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" $(MAKE) -C $(TESTDIR) fuzztest
+
+gpptest: clean
+	CC=g++ $(MAKE) -C $(PRGDIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
+
 gcc5test: clean
 	gcc-5 -v
 	$(MAKE) all CC=gcc-5 MOREFLAGS="-Werror"
@ -118,7 +176,7 @@ gcc6test: clean

 clangtest: clean
 	clang -v
-	$(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
+	$(MAKE) all CXX=clang-++ CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"

 armtest: clean
 	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
@ -142,24 +200,36 @@ arm-ppc-compilation:
 	$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static"
 	$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static"

+# run UBsan with -fsanitize-recover=signed-integer-overflow
+# due to a bug in UBsan when doing pointer subtraction
+# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63303
+
 usan: clean
-	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"
+	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=undefined"

 asan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address"

+asan-%: clean
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address" $(MAKE) -C $(TESTDIR) $*
+
 msan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer"   # datagen.c fails this test for no obvious reason

+msan-%: clean
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer" $(MAKE) -C $(TESTDIR) $*
+
 asan32: clean
 	$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"

 uasan: clean
-	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined"
+	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined"

 uasan-%: clean
-	LDFLAGS=-fuse-ld=gold CFLAGS="-Og -fsanitize=address -fsanitize=undefined" $(MAKE) -C $(TESTDIR) $*
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-Og -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined" $(MAKE) -C $(TESTDIR) $*

+tsan-%: clean
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread" $(MAKE) -C $(TESTDIR) $*
 apt-install:
 	sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES)

@ -171,9 +241,9 @@ ppcinstall:
 	APT_PACKAGES="qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu" $(MAKE) apt-install

 arminstall:
-	APT_PACKAGES="qemu-system-arm qemu-user-static gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross" $(MAKE) apt-install
+	APT_PACKAGES="qemu-system-arm qemu-user-static gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross" $(MAKE) apt-install

-valgrindinstall: 
+valgrindinstall:
 	APT_PACKAGES="valgrind" $(MAKE) apt-install

 libc6install:
@ -185,6 +255,9 @@ gcc6install: apt-add-repo
 gpp6install: apt-add-repo
 	APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install

+clang38install:
+	APT_PACKAGES="clang-3.8" $(MAKE) apt-install
+
 endif


@ -198,36 +271,45 @@ endif
 #make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
 #------------------------------------------------------------------------
 ifneq (,$(filter $(HOST_OS),MSYS POSIX))
-cmaketest:
+cmakebuild:
 	cmake --version
 	$(RM) -r $(BUILDIR)/cmake/build
 	mkdir $(BUILDIR)/cmake/build
 	cd $(BUILDIR)/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall

-c90test: clean
+c90build: clean
+	gcc -v
 	CFLAGS="-std=c90" $(MAKE) allmost  # will fail, due to missing support for `long long`

-gnu90test: clean
+gnu90build: clean
+	gcc -v
 	CFLAGS="-std=gnu90" $(MAKE) allmost

-c99test: clean
+c99build: clean
+	gcc -v
 	CFLAGS="-std=c99" $(MAKE) allmost

-gnu99test: clean
+gnu99build: clean
+	gcc -v
 	CFLAGS="-std=gnu99" $(MAKE) allmost

-c11test: clean
+c11build: clean
+	gcc -v
 	CFLAGS="-std=c11" $(MAKE) allmost

-bmix64test: clean
+bmix64build: clean
+	gcc -v
 	CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(TESTDIR) test

-bmix32test: clean
+bmix32build: clean
+	gcc -v
 	CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(TESTDIR) test

-bmi32test: clean
+bmi32build: clean
+	gcc -v
 	CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test

 staticAnalyze: clean
+	gcc -v
 	CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all
 endif
--- a/12
+++ b/12
@ -3,9 +3,15 @@ cli : new : can compress in *.gz format, using --format=gzip command, by Przemys
 cli : new : advanced benchmark command --priority=rt
 cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77
 cli : fix : --rm remains silent when input is stdin
-API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize(), by Sean Purcell
-API : change : ZSTD_compress*() with srcSize==0 create an empty-frame of known size
-build:new : meson build system in contrib/meson, by Dima Krasner
+cli : experimental : xzstd, with support for xz/lzma decoding, by Przemyslaw Skibinski
+speed : improved decompression speed in streaming mode for single shot scenarios (+5%)
+memory : DDict (decompression dictionary) memory usage down from 150 KB to 20 KB
+arch : 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell
+API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize()
+API : changed : dropped support of legacy versions <= v0.3 (can be changed by modifying ZSTD_LEGACY_SUPPORT value)
+build: new: meson build system in contrib/meson, by Dima Krasner
+build: improved cmake script, by @Majlen
+build: added -Wformat-security flag, as recommended by Padraig Brady
 doc : new : educational decoder, by Sean Purcell

 v1.1.3
--- a/README.md
+++ b/README.md
@ -11,23 +11,26 @@ you can consult a list of known ports on [Zstandard homepage](http://www.zstd.ne
 |master      | [![Build Status](https://travis-ci.org/facebook/zstd.svg?branch=master)](https://travis-ci.org/facebook/zstd) |
 |dev         | [![Build Status](https://travis-ci.org/facebook/zstd.svg?branch=dev)](https://travis-ci.org/facebook/zstd) |

-As a reference, several fast compression algorithms were tested and compared on a Core i7-3930K CPU @ 4.5GHz, using [lzbench], an open-source in-memory benchmark by @inikep compiled with GCC 5.4.0, with the [Silesia compression corpus].
+As a reference, several fast compression algorithms were tested and compared
+on a server running Linux Debian (`Linux version 4.8.0-1-amd64`),
+with a Core i7-6700K CPU @ 4.0GHz,
+using [lzbench], an open-source in-memory benchmark by @inikep
+compiled with GCC 6.3.0,
+on the [Silesia compression corpus].

 [lzbench]: https://github.com/inikep/lzbench
 [Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia

-
-| Name                    | Ratio | C.speed | D.speed |
-|-------------------------|-------|--------:|--------:|
-|                         |       |   MB/s  |  MB/s   |
-| **zstd 0.8.2 -1**     |**2.877**| **330** | **940** |
-| [zlib] 1.2.8 deflate -1 | 2.730 |    95   |   360   |
-| brotli 0.4 -0           | 2.708 |   320   |   375   |
-| QuickLZ 1.5             | 2.237 |   510   |   605   |
-| LZO 2.09                | 2.106 |   610   |   870   |
-| [LZ4] r131              | 2.101 |   620   |  3100   |
-| Snappy 1.1.3            | 2.091 |   480   |  1600   |
-| LZF 3.6                 | 2.077 |   375   |   790   |
+| Compressor name         | Ratio | Compression| Decompress.|
+| ---------------         | ------| -----------| ---------- |
+| **zstd 1.1.3 -1**       | 2.877 |   430 MB/s |  1110 MB/s |
+| zlib 1.2.8 -1           | 2.743 |   110 MB/s |   400 MB/s |
+| brotli 0.5.2 -0         | 2.708 |   400 MB/s |   430 MB/s |
+| quicklz 1.5.0 -1        | 2.238 |   550 MB/s |   710 MB/s |
+| lzo1x 2.09 -1           | 2.108 |   650 MB/s |   830 MB/s |
+| lz4 1.7.5               | 2.101 |   720 MB/s |  3600 MB/s |
+| snappy 1.1.3            | 2.091 |   500 MB/s |  1650 MB/s |
+| lzf 3.6 -1              | 2.077 |   400 MB/s |   860 MB/s |

 [zlib]:http://www.zlib.net/
 [LZ4]: http://www.lz4.org/
@ -35,7 +38,12 @@ As a reference, several fast compression algorithms were tested and compared on
 Zstd can also offer stronger compression ratios at the cost of compression speed.
 Speed vs Compression trade-off is configurable by small increments. Decompression speed is preserved and remains roughly the same at all settings, a property shared by most LZ compression algorithms, such as [zlib] or lzma.

-The following tests were run on a Core i7-3930K CPU @ 4.5GHz, using [lzbench], an open-source in-memory benchmark by @inikep compiled with GCC 5.2.1, on the [Silesia compression corpus].
+The following tests were run
+on a server running Linux Debian (`Linux version 4.8.0-1-amd64`)
+with a Core i7-6700K CPU @ 4.0GHz,
+using [lzbench], an open-source in-memory benchmark by @inikep
+compiled with GCC 6.3.0,
+on the [Silesia compression corpus].

 Compression Speed vs Ratio | Decompression Speed
 ---------------------------|--------------------
--- a/TESTING.md
+++ b/TESTING.md
@ -0,0 +1,44 @@
+Testing
+=======
+
+Zstandard CI testing is split up into three sections:
+short, medium, and long tests.
+
+Short Tests
+-----------
+Short tests run on CircleCI for new commits on every branch and pull request.
+They consist of the following tests:
+- Compilation on all supported targets (x86, x86_64, ARM, AArch64, PowerPC, and PowerPC64)
+- Compilation on various versions of gcc, clang, and g++
+- `tests/playTests.sh` on x86_64, without the tests on long data (CLI tests)
+- Small tests (`tests/legacy.c`, `tests/longmatch.c`, `tests/symbols.c`) on x64_64
+
+Medium Tests
+------------
+Medium tests run on every commit and pull request to `dev` branch, on TravisCI.
+They consist of the following tests:
+- The following tests run with UBsan and Asan on x86_64 and x86, as well as with
+  Msan on x86_64
+  - `tests/playTests.sh --test-long-data`
+  - Fuzzer tests: `tests/fuzzer.c`, `tests/zstreamtest.c`, and `tests/decodecorpus.c`
+- `tests/zstreamtest.c` under Tsan (streaming mode, including multithreaded mode)
+- Valgrind Test (`make -C tests valgrindTest`) (testing CLI and fuzzer under valgrind)
+- Fuzzer tests (see above) on ARM, AArch64, PowerPC, and PowerPC64
+
+Long Tests
+----------
+Long tests run on all commits to `master` branch,
+and once a day on the current version of `dev` branch,
+on TravisCI.
+They consist of the following tests:
+- Entire test suite (including fuzzers and some other specialized tests) on:
+  - x86_64 and x86 with UBsan and Asan
+  - x86_64 with Msan
+  - ARM, AArch64, PowerPC, and PowerPC64
+- Streaming mode fuzzer with Tsan (for the `zstdmt` testing)
+- ZlibWrapper tests, including under valgrind
+- Versions test (ensuring `zstd` can decode files from all previous versions)
+- `pzstd` with asan and tsan, as well as in 32-bits mode
+- Testing `zstd` with legacy mode off
+- Testing `zbuff` (old streaming API)
+- Entire test suite and make install on OS X
--- a/appveyor.yml
+++ b/appveyor.yml
@ -146,6 +146,11 @@ test_script:
      fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%
    )

+branches:
+  only:
+  - dev
+  - master
+
 artifacts:
  - path: bin\zstd.exe
  - path: bin\zstd32.exe
--- a/build/VS2005/zstd/zstd.vcproj
+++ b/build/VS2005/zstd/zstd.vcproj
@ -44,7 +44,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@ -121,7 +121,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@ -196,7 +196,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@ -274,7 +274,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
--- a/build/VS2005/zstdlib/zstdlib.vcproj
+++ b/build/VS2005/zstdlib/zstdlib.vcproj
@ -44,7 +44,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@ -120,7 +120,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@ -194,7 +194,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@ -271,7 +271,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
--- a/build/VS2008/zstd/zstd.vcproj
+++ b/build/VS2008/zstd/zstd.vcproj
@ -45,7 +45,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@ -122,7 +122,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@ -197,7 +197,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@ -275,7 +275,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
--- a/build/VS2008/zstdlib/zstdlib.vcproj
+++ b/build/VS2008/zstdlib/zstdlib.vcproj
@ -45,7 +45,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@ -121,7 +121,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@ -195,7 +195,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@ -272,7 +272,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
--- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
+++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
@ -149,7 +149,7 @@
      </PrecompiledHeader>
      <WarningLevel>Level4</WarningLevel>
      <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
      <MinimalRebuild>true</MinimalRebuild>
      <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@ -169,7 +169,7 @@
      </PrecompiledHeader>
      <WarningLevel>Level4</WarningLevel>
      <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
      <TreatWarningAsError>true</TreatWarningAsError>
      <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@ -189,7 +189,7 @@
      <Optimization>MaxSpeed</Optimization>
      <FunctionLevelLinking>true</FunctionLevelLinking>
      <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
      <EnablePREfast>false</EnablePREfast>
      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
@ -211,7 +211,7 @@
      <Optimization>MaxSpeed</Optimization>
      <FunctionLevelLinking>true</FunctionLevelLinking>
      <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
      <TreatWarningAsError>false</TreatWarningAsError>
      <EnablePREfast>false</EnablePREfast>
      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
--- a/build/VS2010/libzstd/libzstd.vcxproj
+++ b/build/VS2010/libzstd/libzstd.vcxproj
@ -146,7 +146,7 @@
      </PrecompiledHeader>
      <WarningLevel>Level4</WarningLevel>
      <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
      <MinimalRebuild>true</MinimalRebuild>
      <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@ -166,7 +166,7 @@
      </PrecompiledHeader>
      <WarningLevel>Level4</WarningLevel>
      <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
      <TreatWarningAsError>true</TreatWarningAsError>
      <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@ -186,7 +186,7 @@
      <Optimization>MaxSpeed</Optimization>
      <FunctionLevelLinking>true</FunctionLevelLinking>
      <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
      <EnablePREfast>false</EnablePREfast>
      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
@ -208,7 +208,7 @@
      <Optimization>MaxSpeed</Optimization>
      <FunctionLevelLinking>true</FunctionLevelLinking>
      <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
      <TreatWarningAsError>false</TreatWarningAsError>
      <EnablePREfast>false</EnablePREfast>
      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
--- a/build/VS2010/zstd/zstd.vcxproj
+++ b/build/VS2010/zstd/zstd.vcxproj
@ -155,7 +155,7 @@
      </PrecompiledHeader>
      <WarningLevel>Level4</WarningLevel>
      <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
      <TreatWarningAsError>true</TreatWarningAsError>
      <EnablePREfast>false</EnablePREfast>
    </ClCompile>
@ -171,7 +171,7 @@
      </PrecompiledHeader>
      <WarningLevel>Level4</WarningLevel>
      <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
      <TreatWarningAsError>true</TreatWarningAsError>
      <EnablePREfast>false</EnablePREfast>
    </ClCompile>
@ -189,7 +189,7 @@
      <Optimization>MaxSpeed</Optimization>
      <FunctionLevelLinking>true</FunctionLevelLinking>
      <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
      <EnablePREfast>false</EnablePREfast>
      <TreatWarningAsError>false</TreatWarningAsError>
      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
@ -210,7 +210,7 @@
      <Optimization>MaxSpeed</Optimization>
      <FunctionLevelLinking>true</FunctionLevelLinking>
      <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
      <TreatWarningAsError>false</TreatWarningAsError>
      <EnablePREfast>false</EnablePREfast>
      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
--- a/build/cmake/CMakeLists.txt
+++ b/build/cmake/CMakeLists.txt
@ -11,10 +11,12 @@ PROJECT(zstd)
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8.7)

 OPTION(ZSTD_LEGACY_SUPPORT "LEGACY SUPPORT" OFF)
+OPTION(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" ON)
+OPTION(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF)

 IF (ZSTD_LEGACY_SUPPORT)
    MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT defined!")
-    ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=1)
+    ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=4)
 ELSE (ZSTD_LEGACY_SUPPORT)
    MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT not defined!")
    ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=0)
@ -23,6 +25,9 @@ ENDIF (ZSTD_LEGACY_SUPPORT)
 ADD_SUBDIRECTORY(lib)
 ADD_SUBDIRECTORY(programs)
 ADD_SUBDIRECTORY(tests)
+IF (ZSTD_BUILD_CONTRIB)
+    ADD_SUBDIRECTORY(contrib)
+ENDIF (ZSTD_BUILD_CONTRIB)

 #-----------------------------------------------------------------------------
 # Add extra compilation flags
--- a/build/cmake/CMakeModules/AddExtraCompilationFlags.cmake
+++ b/build/cmake/CMakeModules/AddExtraCompilationFlags.cmake
@ -132,10 +132,8 @@ MACRO(ADD_EXTRA_COMPILATION_FLAGS)
        endif (ACTIVATE_WARNING_CAST_ALIGN)

        if (ACTIVATE_WARNING_STRICT_PROTOTYPES)
-            list(APPEND CMAKE_CXX_FLAGS ${WARNING_STRICT_PROTOTYPES})
            list(APPEND CMAKE_C_FLAGS ${WARNING_STRICT_PROTOTYPES})
        else ()
-            string(REPLACE ${WARNING_STRICT_PROTOTYPES} "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
            string(REPLACE ${WARNING_STRICT_PROTOTYPES} "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
        endif (ACTIVATE_WARNING_STRICT_PROTOTYPES)

--- a/build/cmake/contrib/CMakeLists.txt
+++ b/build/cmake/contrib/CMakeLists.txt
@ -0,0 +1,16 @@
+# ################################################################
+# * Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
+# * All rights reserved.
+# *
+# * This source code is licensed under the BSD-style license found in the
+# * LICENSE file in the root directory of this source tree. An additional grant
+# * of patent rights can be found in the PATENTS file in the same directory.
+#
+# You can contact the author at :
+#  - zstd homepage : http://www.zstd.net/
+# ################################################################
+
+PROJECT(contrib)
+
+ADD_SUBDIRECTORY(pzstd)
+
--- a/build/cmake/contrib/pzstd/CMakeLists.txt
+++ b/build/cmake/contrib/pzstd/CMakeLists.txt
@ -0,0 +1,30 @@
+# ################################################################
+# * Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
+# * All rights reserved.
+# *
+# * This source code is licensed under the BSD-style license found in the
+# * LICENSE file in the root directory of this source tree. An additional grant
+# * of patent rights can be found in the PATENTS file in the same directory.
+#
+# You can contact the author at :
+#  - zstd homepage : http://www.zstd.net/
+# ################################################################
+
+PROJECT(pzstd)
+
+SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
+
+# Define project root directory
+SET(ROOT_DIR ../../../..)
+
+# Define programs directory, where sources and header files are located
+SET(LIBRARY_DIR ${ROOT_DIR}/lib)
+SET(PROGRAMS_DIR ${ROOT_DIR}/programs)
+SET(PZSTD_DIR ${ROOT_DIR}/contrib/pzstd)
+INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD_DIR})
+
+ADD_EXECUTABLE(pzstd ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp)
+TARGET_LINK_LIBRARIES(pzstd libzstd_static pthread)
+SET_TARGET_PROPERTIES(pzstd PROPERTIES COMPILE_DEFINITIONS "NDEBUG")
+SET_TARGET_PROPERTIES(pzstd PROPERTIES COMPILE_OPTIONS "-Wno-shadow")
+
--- a/build/cmake/lib/CMakeLists.txt
+++ b/build/cmake/lib/CMakeLists.txt
@ -166,23 +166,17 @@ IF (UNIX)
    SET(SHARED_LIBRARY_SYMLINK1_PATH ${CMAKE_CURRENT_BINARY_DIR}/${SHARED_LIBRARY_SYMLINK1})
    SET(SHARED_LIBRARY_SYMLINK2_PATH ${CMAKE_CURRENT_BINARY_DIR}/${SHARED_LIBRARY_SYMLINK2})

-    if (EXISTS ${SHARED_LIBRARY_SYMLINK1_PATH})
-        FILE(REMOVE ${SHARED_LIBRARY_SYMLINK1_PATH})
-    endif (EXISTS ${SHARED_LIBRARY_SYMLINK1_PATH})
-
-    if (EXISTS ${SHARED_LIBRARY_SYMLINK2_PATH})
-        FILE(REMOVE ${SHARED_LIBRARY_SYMLINK2_PATH})
-    endif (EXISTS ${SHARED_LIBRARY_SYMLINK2_PATH})
+    ADD_CUSTOM_COMMAND(TARGET libzstd_shared POST_BUILD
+            COMMAND ${CMAKE_COMMAND} -E create_symlink ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK1}
+            DEPENDS  ${SHARED_LIBRARY_LINK_PATH}
+            COMMENT "Generating symbolic link ${SHARED_LIBRARY_LINK} -> ${SHARED_LIBRARY_SYMLINK1}")

    ADD_CUSTOM_COMMAND(TARGET libzstd_shared POST_BUILD
-            COMMAND ln -s ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK1}
-            DEPENDS ${SHARED_LIBRARY_LINK_PATH}
-            COMMENT "Generating symbolic link")
+            COMMAND ${CMAKE_COMMAND} -E create_symlink ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK2}
+            DEPENDS  ${SHARED_LIBRARY_LINK_PATH}
+            COMMENT "Generating symbolic link ${SHARED_LIBRARY_LINK} -> ${SHARED_LIBRARY_SYMLINK2}")

-    ADD_CUSTOM_COMMAND(TARGET libzstd_shared POST_BUILD
-            COMMAND ln -s ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK2}
-            DEPENDS ${SHARED_LIBRARY_LINK_PATH}
-            COMMENT "Generating symbolic link")
+    SET_DIRECTORY_PROPERTIES(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${SHARED_LIBRARY_SYMLINK1};${SHARED_LIBRARY_SYMLINK2}")

    INSTALL(FILES ${SHARED_LIBRARY_SYMLINK1_PATH} DESTINATION ${INSTALL_LIBRARY_DIR})
    INSTALL(FILES ${SHARED_LIBRARY_SYMLINK2_PATH} DESTINATION ${INSTALL_LIBRARY_DIR})
--- a/build/cmake/programs/CMakeLists.txt
+++ b/build/cmake/programs/CMakeLists.txt
@ -34,9 +34,17 @@ ENDIF (MSVC)

 ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
 TARGET_LINK_LIBRARIES(zstd libzstd_static)
-
 IF (UNIX)
    ADD_EXECUTABLE(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c)
    TARGET_LINK_LIBRARIES(zstd-frugal libzstd_static)
    SET_TARGET_PROPERTIES(zstd-frugal PROPERTIES COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
 ENDIF (UNIX)
+
+IF (ZSTD_MULTITHREAD_SUPPORT)
+    ADD_EXECUTABLE(zstdmt ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
+    SET_TARGET_PROPERTIES(zstdmt PROPERTIES COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
+    TARGET_LINK_LIBRARIES(zstdmt libzstd_static)
+    IF (UNIX)
+        TARGET_LINK_LIBRARIES(zstdmt pthread)
+    ENDIF (UNIX)
+ENDIF (ZSTD_MULTITHREAD_SUPPORT)
--- a/circle.yml
+++ b/circle.yml
@ -1,42 +1,60 @@
 dependencies:
  override:
+    - sudo dpkg --add-architecture i386
    - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update
-    #- sudo apt-get -y install qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu valgrind
-    #- sudo apt-get -y install qemu-system-arm gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
-    - sudo apt-get -y install libc6-dev-i386 clang gcc-5 gcc-6
-
-  # use default "parallel: true" for commands in the machine, checkout, dependencies and database build phase
-  post:
-    - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make cmaketest           && make clean; fi
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-invalidDictionaries && make clean; fi
-    - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then g++ -v; make gpptest     && make clean; fi
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-decodecorpus && make clean; fi
-    - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make gnu90test   && make clean; fi
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-symbols             && make clean; fi
-    - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make c99test     && make clean; fi
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-longmatch           && make clean; fi
-    - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make gnu99test   && make clean; fi
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests dll                      && make clean; fi
-    - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then clang -v; make clangtest && make clean; fi
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C programs zstd-small zstd-decompress zstd-compress zstd32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean lib && make clean; fi
-    - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make travis-install      && make clean; fi
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-fullbench           && make clean; fi
-    - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc-5 -v; make gcc5test  && gcc-6 -v && make gcc6test && make clean; fi
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-zstream             && make clean; fi
-    - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make -C tests test-zstd  && make clean; fi
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-fuzzer FUZZERTEST=-T4mn && make clean; fi
+    - sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
+    - sudo apt-get -y install libstdc++-6-dev clang gcc g++ gcc-5 gcc-6
+    - sudo apt-get -y install linux-libc-dev:i386 libc6-dev-i386

 test:
  override:
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then cc -v; make all   && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu90build   && make clean; fi
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make c99build     && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu99build   && make clean; fi
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make c11build     && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make cmakebuild   && make clean; fi
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make gppbuild     && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc5build    && make clean; fi
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make gcc6build    && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make clangbuild   && make clean; fi
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make m32build     && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make armbuild     && make clean; fi
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make aarch64build && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppcbuild     && make clean; fi
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make ppc64build   && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then true              && make clean; fi #could add another test here
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make shortest     && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-longmatch test-symbols && make clean; fi
+      :
+        parallel: true
+
+  post:
    - echo Circle CI tests finished

  # Longer tests
--- a/contrib/gen_html/gen_html.cpp
+++ b/contrib/gen_html/gen_html.cpp
@ -19,7 +19,7 @@ void trim(string& s, string characters)
 {
    size_t p = s.find_first_not_of(characters);
    s.erase(0, p);
- 
+
    p = s.find_last_not_of(characters);
    if (string::npos != p)
       s.erase(p+1);
@ -48,7 +48,7 @@ vector<string> get_lines(vector<string>& input, int& linenum, string terminator)
        line = input[linenum];

        if (terminator.empty() && line.empty()) { linenum--; break; }
-        
+
        epos = line.find(terminator);
        if (!terminator.empty() && epos!=string::npos) {
            out.push_back(line);
@ -168,7 +168,11 @@ int main(int argc, char *argv[]) {
            sout << "<pre><b>";
            for (l=0; l<lines.size(); l++) {
              //  fprintf(stderr, "line[%d]=%s\n", l, lines[l].c_str());
-                print_line(sout, lines[l]);
+                string fline = lines[l];
+                if (fline.substr(0, 12) == "ZSTDLIB_API " ||
+                    fline.substr(0, 12) == string(12, ' '))
+                  fline = fline.substr(12);
+                print_line(sout, fline);
            }
            sout << "</b><p>";
            for (l=0; l<comments.size(); l++) {
@ -217,4 +221,4 @@ int main(int argc, char *argv[]) {
    ostream << "</html>" << endl << "</body>" << endl;

    return 0;
-}
+}
--- a/contrib/meson/meson.build
+++ b/contrib/meson/meson.build
@ -15,7 +15,7 @@ libzstd_includes = [include_directories(common_dir, dictbuilder_dir, compress_di

 if get_option('legacy_support')
    message('Enabling legacy support')
-    libzstd_cflags = ['-DZSTD_LEGACY_SUPPORT=1']
+    libzstd_cflags = ['-DZSTD_LEGACY_SUPPORT=4']

    legacy_dir = join_paths(lib_dir, 'legacy')
    libzstd_includes += [include_directories(legacy_dir)]
--- a/contrib/pzstd/Pzstd.cpp
+++ b/contrib/pzstd/Pzstd.cpp
@ -410,7 +410,7 @@ std::uint64_t asyncCompressChunks(
    });
    // Pass the output queue to the writer thread.
    chunks.push(std::move(out));
-    state.log(VERBOSE, "Starting a new frame\n");
+    state.log(VERBOSE, "%s\n", "Starting a new frame");
    // Fill the input queue for the compression job we just started
    status = readData(*in, ZSTD_CStreamInSize(), step, fd, &bytesRead);
  }
@ -547,8 +547,8 @@ std::uint64_t asyncDecompressFrames(
    if (frameSize == 0) {
      // We hit a non SkippableFrame ==> not compressed by pzstd or corrupted
      // Pass the rest of the source to this decompression task
-      state.log(VERBOSE,
-          "Input not in pzstd format, falling back to serial decompression\n");
+      state.log(VERBOSE, "%s\n",
+          "Input not in pzstd format, falling back to serial decompression");
      while (status == FileStatus::Continue && !state.errorHolder.hasError()) {
        status = readData(*in, chunkSize, chunkSize, fd, &totalBytesRead);
      }
--- a/contrib/pzstd/Pzstd.h
+++ b/contrib/pzstd/Pzstd.h
@ -41,7 +41,7 @@ class SharedState {
      auto parameters = options.determineParameters();
      cStreamPool.reset(new ResourcePool<ZSTD_CStream>{
          [this, parameters]() -> ZSTD_CStream* {
-            this->log(VERBOSE, "Creating new ZSTD_CStream\n");
+            this->log(VERBOSE, "%s\n", "Creating new ZSTD_CStream");
            auto zcs = ZSTD_createCStream();
            if (zcs) {
              auto err = ZSTD_initCStream_advanced(
@ -59,7 +59,7 @@ class SharedState {
    } else {
      dStreamPool.reset(new ResourcePool<ZSTD_DStream>{
          [this]() -> ZSTD_DStream* {
-            this->log(VERBOSE, "Creating new ZSTD_DStream\n");
+            this->log(VERBOSE, "%s\n", "Creating new ZSTD_DStream");
            auto zds = ZSTD_createDStream();
            if (zds) {
              auto err = ZSTD_initDStream(zds);
--- a/doc/images/Cspeed4.png
+++ b/doc/images/Cspeed4.png
--- a/doc/images/Dspeed4.png
+++ b/doc/images/Dspeed4.png
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@ -55,8 +55,8 @@
 <a name="Chapter3"></a><h2>Simple API</h2><pre></pre>

 <pre><b>size_t ZSTD_compress( void* dst, size_t dstCapacity,
-                            const void* src, size_t srcSize,
-                                  int compressionLevel);
+                const void* src, size_t srcSize,
+                      int compressionLevel);
 </b><p>    Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
    Hint : compression runs faster if `dstCapacity` >=  `ZSTD_compressBound(srcSize)`.
    @return : compressed size written into `dst` (<= `dstCapacity),
@ -64,7 +64,7 @@
 </p></pre><BR>

 <pre><b>size_t ZSTD_decompress( void* dst, size_t dstCapacity,
-                              const void* src, size_t compressedSize);
+                  const void* src, size_t compressedSize);
 </b><p>    `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
    `dstCapacity` is an upper bound of originalSize.
    If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
@ -118,7 +118,11 @@ size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
 </b><p>    Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). 
 </p></pre><BR>

-<h3>Decompression context</h3><pre></pre><b><pre>typedef struct ZSTD_DCtx_s ZSTD_DCtx;
+<h3>Decompression context</h3><pre>   When decompressing many times,
+   it is recommended to allocate a context just once, and re-use it for each successive compression operation.
+   This will make workload friendlier for system's memory.
+   Use one context per thread for parallel execution in multi-threaded environments. 
+</pre><b><pre>typedef struct ZSTD_DCtx_s ZSTD_DCtx;
 ZSTD_DCtx* ZSTD_createDCtx(void);
 size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 </pre></b><BR>
@ -129,19 +133,19 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 <a name="Chapter5"></a><h2>Simple dictionary API</h2><pre></pre>

 <pre><b>size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
-                                           void* dst, size_t dstCapacity,
-                                     const void* src, size_t srcSize,
-                                     const void* dict,size_t dictSize,
-                                           int compressionLevel);
+                               void* dst, size_t dstCapacity,
+                         const void* src, size_t srcSize,
+                         const void* dict,size_t dictSize,
+                               int compressionLevel);
 </b><p>   Compression using a predefined Dictionary (see dictBuilder/zdict.h).
   Note : This function loads the dictionary, resulting in significant startup delay.
   Note : When `dict == NULL || dictSize < 8` no dictionary is used. 
 </p></pre><BR>

 <pre><b>size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
-                                             void* dst, size_t dstCapacity,
-                                       const void* src, size_t srcSize,
-                                       const void* dict,size_t dictSize);
+                                 void* dst, size_t dstCapacity,
+                           const void* src, size_t srcSize,
+                           const void* dict,size_t dictSize);
 </b><p>   Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
   Dictionary must be identical to the one used during compression.
   Note : This function loads the dictionary, resulting in significant startup delay.
@ -162,9 +166,9 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 </p></pre><BR>

 <pre><b>size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
-                                            void* dst, size_t dstCapacity,
-                                      const void* src, size_t srcSize,
-                                      const ZSTD_CDict* cdict);
+                                void* dst, size_t dstCapacity,
+                          const void* src, size_t srcSize,
+                          const ZSTD_CDict* cdict);
 </b><p>   Compression using a digested Dictionary.
   Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
   Note that compression level is decided during dictionary creation. 
@ -180,9 +184,9 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 </p></pre><BR>

 <pre><b>size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
-                                              void* dst, size_t dstCapacity,
-                                        const void* src, size_t srcSize,
-                                        const ZSTD_DDict* ddict);
+                                  void* dst, size_t dstCapacity,
+                            const void* src, size_t srcSize,
+                            const ZSTD_DDict* ddict);
 </b><p>   Decompression using a digested Dictionary.
   Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. 
 </p></pre><BR>
@ -239,6 +243,14 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 
 <BR></pre>

+<h3>ZSTD_CStream management functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream(void);
+size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
+</pre></b><BR>
+<h3>Streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
+size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+</pre></b><BR>
 <pre><b>size_t ZSTD_CStreamInSize(void);    </b>/**< recommended size for input buffer */<b>
 </b></pre><BR>
 <pre><b>size_t ZSTD_CStreamOutSize(void);   </b>/**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */<b>
@ -264,6 +276,12 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 
 <BR></pre>

+<h3>ZSTD_DStream management functions</h3><pre></pre><b><pre>ZSTD_DStream* ZSTD_createDStream(void);
+size_t ZSTD_freeDStream(ZSTD_DStream* zds);
+</pre></b><BR>
+<h3>Streaming decompression functions</h3><pre></pre><b><pre>size_t ZSTD_initDStream(ZSTD_DStream* zds);
+size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+</pre></b><BR>
 <pre><b>size_t ZSTD_DStreamInSize(void);    </b>/*!< recommended size for input buffer */<b>
 </b></pre><BR>
 <pre><b>size_t ZSTD_DStreamOutSize(void);   </b>/*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */<b>
@ -381,7 +399,7 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 </p></pre><BR>

 <pre><b>ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
-                                                  ZSTD_parameters params, ZSTD_customMem customMem);
+                                      ZSTD_parameters params, ZSTD_customMem customMem);
 </b><p>  Create a ZSTD_CDict using external alloc and free, and customized compression parameters 
 </p></pre><BR>

@ -409,10 +427,10 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 </p></pre><BR>

 <pre><b>size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
-                                           void* dst, size_t dstCapacity,
-                                     const void* src, size_t srcSize,
-                                     const void* dict,size_t dictSize,
-                                           ZSTD_parameters params);
+                               void* dst, size_t dstCapacity,
+                         const void* src, size_t srcSize,
+                         const void* dict,size_t dictSize,
+                               ZSTD_parameters params);
 </b><p>   Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter 
 </p></pre><BR>

@ -443,6 +461,11 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
  It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict 
 </p></pre><BR>

+<pre><b>ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
+                                      unsigned byReference, ZSTD_customMem customMem);
+</b><p>  Create a ZSTD_DDict using external alloc and free, optionally by reference 
+</p></pre><BR>
+
 <pre><b>size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 </b><p>  Gives the amount of memory used by a given ZSTD_DDict 
 </p></pre><BR>
--- a/lib/Makefile
+++ b/lib/Makefile
@ -24,19 +24,22 @@ CPPFLAGS+= -I. -I./common -DXXH_NAMESPACE=ZSTD_
 CFLAGS  ?= -O3
 DEBUGFLAGS = -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
           -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-           -Wstrict-prototypes -Wundef -Wpointer-arith
+           -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security
 CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
 FLAGS    = $(CPPFLAGS) $(CFLAGS)


 ZSTD_FILES := $(wildcard common/*.c compress/*.c decompress/*.c dictBuilder/*.c deprecated/*.c)

-ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
-CPPFLAGS  += -DZSTD_LEGACY_SUPPORT=0
-else
-CPPFLAGS  += -I./legacy -DZSTD_LEGACY_SUPPORT=1
-ZSTD_FILES+= $(wildcard legacy/*.c)
+ZSTD_LEGACY_SUPPORT ?= 4
+
+ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
+ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
+	ZSTD_FILES += $(shell ls legacy/*.c | grep 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
 endif
+	CPPFLAGS += -I./legacy
+endif
+CPPFLAGS  += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)

 ZSTD_OBJ   := $(patsubst %.c,%.o,$(ZSTD_FILES))

--- a/lib/common/bitstream.h
+++ b/lib/common/bitstream.h
@ -60,6 +60,9 @@ extern "C" {
 #  include <immintrin.h>   /* support for bextr (experimental) */
 #endif

+#define STREAM_ACCUMULATOR_MIN_32  25
+#define STREAM_ACCUMULATOR_MIN_64  57
+#define STREAM_ACCUMULATOR_MIN    ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64))

 /*-******************************************
 *  bitStream encoding API (write forward)
--- a/lib/common/entropy_common.c
+++ b/lib/common/entropy_common.c
@ -43,25 +43,15 @@
 #include "huf.h"


-/*-****************************************
-*  Version
-******************************************/
+/*===   Version   ===*/
 unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; }


-/*-****************************************
-*  FSE Error Management
-******************************************/
+/*===   Error Management   ===*/
 unsigned FSE_isError(size_t code) { return ERR_isError(code); }
-
 const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); }

-
-/* **************************************************************
-*  HUF Error Management
-****************************************************************/
 unsigned HUF_isError(size_t code) { return ERR_isError(code); }
-
 const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }


--- a/lib/common/huf.h
+++ b/lib/common/huf.h
@ -91,7 +91,7 @@ size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize

 /** HUF_compress4X_wksp() :
 *   Same as HUF_compress2(), but uses externally allocated `workSpace`, which must be a table of >= 1024 unsigned */
-size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);  /**< `workSpace` must be a table of at least 1024 unsigned */
+size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);  /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */



@ -102,10 +102,11 @@ size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t s


 /* *** Constants *** */
-#define HUF_TABLELOG_ABSOLUTEMAX  15   /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
-#define HUF_TABLELOG_MAX  12           /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
+#define HUF_TABLELOG_MAX      12       /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
 #define HUF_TABLELOG_DEFAULT  11       /* tableLog by default, when not specified */
-#define HUF_SYMBOLVALUE_MAX 255
+#define HUF_SYMBOLVALUE_MAX  255
+
+#define HUF_TABLELOG_ABSOLUTEMAX  15   /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
 #if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
 #  error "HUF_TABLELOG_MAX is too large !"
 #endif
@ -133,6 +134,10 @@ typedef U32 HUF_DTable;
 #define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
        HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }

+/* The workspace must have alignment at least 4 and be at least this large */
+#define HUF_WORKSPACE_SIZE (6 << 10)
+#define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32))
+

 /* ****************************************
 *  Advanced decompression functions
@ -168,6 +173,17 @@ size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSym
 size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
 size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);

+typedef enum {
+   HUF_repeat_none,  /**< Cannot use the previous table */
+   HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */
+   HUF_repeat_valid  /**< Can use the previous table and it is asumed to be valid */
+ } HUF_repeat;
+/** HUF_compress4X_repeat() :
+*   Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
+*   If it uses hufTable it does not modify hufTable or repeat.
+*   If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
+*   If preferRepeat then the old table will always be used if valid. */
+size_t HUF_compress4X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat);  /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */

 /** HUF_buildCTable_wksp() :
 *  Same as HUF_buildCTable(), but using externally allocated scratch buffer.
@ -214,8 +230,14 @@ size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* c
 /* single stream variants */

 size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
-size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);  /**< `workSpace` must be a table of at least 1024 unsigned */
+size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);  /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
 size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
+/** HUF_compress1X_repeat() :
+*   Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
+*   If it uses hufTable it does not modify hufTable or repeat.
+*   If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
+*   If preferRepeat then the old table will always be used if valid. */
+size_t HUF_compress1X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat);  /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */

 size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* single-symbol decoder */
 size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* double-symbol decoder */
--- a/lib/common/mem.h
+++ b/lib/common/mem.h
@ -48,14 +48,15 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size
 *****************************************************************/
 #if  !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
 # include <stdint.h>
-  typedef  uint8_t BYTE;
-  typedef uint16_t U16;
-  typedef  int16_t S16;
-  typedef uint32_t U32;
-  typedef  int32_t S32;
-  typedef uint64_t U64;
-  typedef  int64_t S64;
-  typedef intptr_t iPtrDiff;
+  typedef   uint8_t BYTE;
+  typedef  uint16_t U16;
+  typedef   int16_t S16;
+  typedef  uint32_t U32;
+  typedef   int32_t S32;
+  typedef  uint64_t U64;
+  typedef   int64_t S64;
+  typedef  intptr_t iPtrDiff;
+  typedef uintptr_t uPtrDiff;
 #else
  typedef unsigned char      BYTE;
  typedef unsigned short      U16;
@ -65,6 +66,7 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size
  typedef unsigned long long  U64;
  typedef   signed long long  S64;
  typedef ptrdiff_t      iPtrDiff;
+  typedef size_t         uPtrDiff;
 #endif


--- a/lib/common/xxhash.c
+++ b/lib/common/xxhash.c
@ -104,7 +104,9 @@ static void  XXH_free  (void* p)  { free(p); }
 #include <string.h>
 static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }

-#define XXH_STATIC_LINKING_ONLY
+#ifndef XXH_STATIC_LINKING_ONLY
+#  define XXH_STATIC_LINKING_ONLY
+#endif
 #include "xxhash.h"


--- a/lib/common/xxhash.h
+++ b/lib/common/xxhash.h
@ -64,16 +64,12 @@ XXH64       13.8 GB/s            1.9 GB/s
 XXH32        6.8 GB/s            6.0 GB/s
 */

-#ifndef XXHASH_H_5627135585666179
-#define XXHASH_H_5627135585666179 1
-
 #if defined (__cplusplus)
 extern "C" {
 #endif

-#ifndef XXH_NAMESPACE
-#  define XXH_NAMESPACE ZSTD_  /* Zstandard specific */
-#endif
+#ifndef XXHASH_H_5627135585666179
+#define XXHASH_H_5627135585666179 1


 /* ****************************
@ -242,6 +238,11 @@ XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH
 /* **************************
 *  Canonical representation
 ****************************/
+/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
+*  The canonical representation uses human-readable write convention, aka big-endian (large digits first).
+*  These functions allow transformation of hash result into and from its canonical format.
+*  This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
+*/
 typedef struct { unsigned char digest[4]; } XXH32_canonical_t;
 typedef struct { unsigned char digest[8]; } XXH64_canonical_t;

@ -251,14 +252,9 @@ XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t
 XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
 XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);

-/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
-*  The canonical representation uses human-readable write convention, aka big-endian (large digits first).
-*  These functions allow transformation of hash result into and from its canonical format.
-*  This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
-*/
+#endif /* XXHASH_H_5627135585666179 */


-#ifdef XXH_STATIC_LINKING_ONLY

 /* ================================================================================================
   This section contains definitions which are not guaranteed to remain stable.
@ -266,6 +262,8 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src
   They shall only be used with static linking.
   Never use these definitions in association with dynamic linking !
 =================================================================================================== */
+#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXH_STATIC_H_3543687687345)
+#define XXH_STATIC_H_3543687687345

 /* These definitions are only meant to allow allocation of XXH state
   statically, on stack, or in a struct for example.
@ -299,11 +297,9 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src
 #    include "xxhash.c"   /* include xxhash functions as `static`, for inlining */
 #  endif

-#endif /* XXH_STATIC_LINKING_ONLY */
+#endif /* XXH_STATIC_LINKING_ONLY && XXH_STATIC_H_3543687687345 */


 #if defined (__cplusplus)
 }
 #endif
-
-#endif /* XXHASH_H_5627135585666179 */
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@ -49,6 +49,10 @@
 #include "error_private.h"
 #define ZSTD_STATIC_LINKING_ONLY
 #include "zstd.h"
+#ifndef XXH_STATIC_LINKING_ONLY
+#  define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
+#endif
+#include "xxhash.h"               /* XXH_reset, update, digest */


 /*-*************************************
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@ -409,6 +409,25 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
    return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, nodeTable, sizeof(nodeTable));
 }

+static size_t HUF_estimateCompressedSize(HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue)
+{
+    size_t nbBits = 0;
+    int s;
+    for (s = 0; s <= (int)maxSymbolValue; ++s) {
+        nbBits += CTable[s].nbBits * count[s];
+    }
+    return nbBits >> 3;
+}
+
+static int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) {
+  int bad = 0;
+  int s;
+  for (s = 0; s <= (int)maxSymbolValue; ++s) {
+    bad |= (count[s] != 0) & (CTable[s].nbBits == 0);
+  }
+  return !bad;
+}
+
 static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable)
 {
    BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits);
@ -510,25 +529,43 @@ size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, si
 }


+static size_t HUF_compressCTable_internal(
+                BYTE* const ostart, BYTE* op, BYTE* const oend,
+                const void* src, size_t srcSize,
+                unsigned singleStream, const HUF_CElt* CTable)
+{
+    size_t const cSize = singleStream ?
+                         HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) :
+                         HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
+    if (HUF_isError(cSize)) { return cSize; }
+    if (cSize==0) { return 0; }   /* uncompressible */
+    op += cSize;
+    /* check compressibility */
+    if ((size_t)(op-ostart) >= srcSize-1) { return 0; }
+    return op-ostart;
+}
+
+
 /* `workSpace` must a table of at least 1024 unsigned */
 static size_t HUF_compress_internal (
                void* dst, size_t dstSize,
                const void* src, size_t srcSize,
                unsigned maxSymbolValue, unsigned huffLog,
                unsigned singleStream,
-                void* workSpace, size_t wkspSize)
+                void* workSpace, size_t wkspSize,
+                HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat)
 {
    BYTE* const ostart = (BYTE*)dst;
    BYTE* const oend = ostart + dstSize;
    BYTE* op = ostart;

-    union {
-        U32 count[HUF_SYMBOLVALUE_MAX+1];
-        HUF_CElt CTable[HUF_SYMBOLVALUE_MAX+1];
-    } table;   /* `count` can overlap with `CTable`; saves 1 KB */
+    U32* count;
+    size_t const countSize = sizeof(U32) * (HUF_SYMBOLVALUE_MAX + 1);
+    HUF_CElt* CTable;
+    size_t const CTableSize = sizeof(HUF_CElt) * (HUF_SYMBOLVALUE_MAX + 1);

    /* checks & inits */
-    if (wkspSize < sizeof(huffNodeTable)) return ERROR(GENERIC);
+    if (wkspSize < sizeof(huffNodeTable) + countSize + CTableSize) return ERROR(GENERIC);
    if (!srcSize) return 0;  /* Uncompressed (note : 1 means rle, so first byte must be correct) */
    if (!dstSize) return 0;  /* cannot fit within dst budget */
    if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong);   /* current block size limit */
@ -536,38 +573,58 @@ static size_t HUF_compress_internal (
    if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX;
    if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT;

+    count = (U32*)workSpace;
+    workSpace = (BYTE*)workSpace + countSize;
+    wkspSize -= countSize;
+    CTable = (HUF_CElt*)workSpace;
+    workSpace = (BYTE*)workSpace + CTableSize;
+    wkspSize -= CTableSize;
+
+    /* Heuristic : If we don't need to check the validity of the old table use the old table for small inputs */
+    if (preferRepeat && repeat && *repeat == HUF_repeat_valid) {
+        return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
+    }
+
    /* Scan input and build symbol stats */
-    {   CHECK_V_F(largest, FSE_count_wksp (table.count, &maxSymbolValue, (const BYTE*)src, srcSize, (U32*)workSpace) );
+    {   CHECK_V_F(largest, FSE_count_wksp (count, &maxSymbolValue, (const BYTE*)src, srcSize, (U32*)workSpace) );
        if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; }   /* single symbol, rle */
        if (largest <= (srcSize >> 7)+1) return 0;   /* Fast heuristic : not compressible enough */
    }

+    /* Check validity of previous table */
+    if (repeat && *repeat == HUF_repeat_check && !HUF_validateCTable(oldHufTable, count, maxSymbolValue)) {
+        *repeat = HUF_repeat_none;
+    }
+    /* Heuristic : use existing table for small inputs */
+    if (preferRepeat && repeat && *repeat != HUF_repeat_none) {
+        return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
+    }
+
    /* Build Huffman Tree */
    huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
-    {   CHECK_V_F(maxBits, HUF_buildCTable_wksp (table.CTable, table.count, maxSymbolValue, huffLog, workSpace, wkspSize) );
+    {   CHECK_V_F(maxBits, HUF_buildCTable_wksp (CTable, count, maxSymbolValue, huffLog, workSpace, wkspSize) );
        huffLog = (U32)maxBits;
+        /* Zero the unused symbols so we can check it for validity */
+        memset(CTable + maxSymbolValue + 1, 0, CTableSize - (maxSymbolValue + 1) * sizeof(HUF_CElt));
    }

    /* Write table description header */
-    {   CHECK_V_F(hSize, HUF_writeCTable (op, dstSize, table.CTable, maxSymbolValue, huffLog) );
-        if (hSize + 12 >= srcSize) return 0;   /* not useful to try compression */
+    {   CHECK_V_F(hSize, HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog) );
+        /* Check if using the previous table will be beneficial */
+        if (repeat && *repeat != HUF_repeat_none) {
+            size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, count, maxSymbolValue);
+            size_t const newSize = HUF_estimateCompressedSize(CTable, count, maxSymbolValue);
+            if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) {
+                return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
+            }
+        }
+        /* Use the new table */
+        if (hSize + 12ul >= srcSize) { return 0; }
        op += hSize;
+        if (repeat) { *repeat = HUF_repeat_none; }
+        if (oldHufTable) { memcpy(oldHufTable, CTable, CTableSize); } /* Save the new table */
    }
-
-    /* Compress */
-    {   size_t const cSize = (singleStream) ?
-                            HUF_compress1X_usingCTable(op, oend - op, src, srcSize, table.CTable) :   /* single segment */
-                            HUF_compress4X_usingCTable(op, oend - op, src, srcSize, table.CTable);
-        if (HUF_isError(cSize)) return cSize;
-        if (cSize==0) return 0;   /* uncompressible */
-        op += cSize;
-    }
-
-    /* check compressibility */
-    if ((size_t)(op-ostart) >= srcSize-1)
-        return 0;
-
-    return op-ostart;
+    return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, CTable);
 }


@ -576,7 +633,16 @@ size_t HUF_compress1X_wksp (void* dst, size_t dstSize,
                      unsigned maxSymbolValue, unsigned huffLog,
                      void* workSpace, size_t wkspSize)
 {
-    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize);
+    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, NULL, NULL, 0);
+}
+
+size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
+                      const void* src, size_t srcSize,
+                      unsigned maxSymbolValue, unsigned huffLog,
+                      void* workSpace, size_t wkspSize,
+                      HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat)
+{
+    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, hufTable, repeat, preferRepeat);
 }

 size_t HUF_compress1X (void* dst, size_t dstSize,
@ -592,7 +658,16 @@ size_t HUF_compress4X_wksp (void* dst, size_t dstSize,
                      unsigned maxSymbolValue, unsigned huffLog,
                      void* workSpace, size_t wkspSize)
 {
-    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize);
+    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, NULL, NULL, 0);
+}
+
+size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
+                      const void* src, size_t srcSize,
+                      unsigned maxSymbolValue, unsigned huffLog,
+                      void* workSpace, size_t wkspSize,
+                      HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat)
+{
+    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, hufTable, repeat, preferRepeat);
 }

 size_t HUF_compress2 (void* dst, size_t dstSize,
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@ -13,8 +13,6 @@
 ***************************************/
 #include <string.h>         /* memset */
 #include "mem.h"
-#define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
-#include "xxhash.h"               /* XXH_reset, update, digest */
 #define FSE_STATIC_LINKING_ONLY   /* FSE_encodeSymbol */
 #include "fse.h"
 #define HUF_STATIC_LINKING_ONLY
@ -81,10 +79,11 @@ struct ZSTD_CCtx_s {
    U32* chainTable;
    HUF_CElt* hufTable;
    U32 flagStaticTables;
+    HUF_repeat flagStaticHufTable;
    FSE_CTable offcodeCTable  [FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
    FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
    FSE_CTable litlengthCTable  [FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
-    unsigned tmpCounters[1024];
+    unsigned tmpCounters[HUF_WORKSPACE_SIZE_U32];
 };

 ZSTD_CCtx* ZSTD_createCCtx(void)
@ -248,14 +247,17 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 fra
 typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_compResetPolicy_e;

 /*! ZSTD_resetCCtx_advanced() :
-    note : @params must be validated */
+    note : `params` must be validated */
 static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
                                       ZSTD_parameters params, U64 frameContentSize,
                                       ZSTD_compResetPolicy_e const crp)
 {
    if (crp == ZSTDcrp_continue)
-        if (ZSTD_equivalentParams(params, zc->params))
+        if (ZSTD_equivalentParams(params, zc->params)) {
+            zc->flagStaticTables = 0;
+            zc->flagStaticHufTable = HUF_repeat_none;
            return ZSTD_continueCCtx(zc, params, frameContentSize);
+        }

    {   size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog);
        U32    const divider = (params.cParams.searchLength==3) ? 3 : 4;
@ -289,6 +291,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
        ptr = zc->hashTable3 + h3Size;
        zc->hufTable = (HUF_CElt*)ptr;
        zc->flagStaticTables = 0;
+        zc->flagStaticHufTable = HUF_repeat_none;
        ptr = ((U32*)ptr) + 256;  /* note : HUF_CElt* is incomplete type, size is simulated using U32 */

        zc->nextToUpdate = 1;
@ -374,12 +377,15 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long

    /* copy entropy tables */
    dstCCtx->flagStaticTables = srcCCtx->flagStaticTables;
+    dstCCtx->flagStaticHufTable = srcCCtx->flagStaticHufTable;
    if (srcCCtx->flagStaticTables) {
-        memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256*4);
        memcpy(dstCCtx->litlengthCTable, srcCCtx->litlengthCTable, sizeof(dstCCtx->litlengthCTable));
        memcpy(dstCCtx->matchlengthCTable, srcCCtx->matchlengthCTable, sizeof(dstCCtx->matchlengthCTable));
        memcpy(dstCCtx->offcodeCTable, srcCCtx->offcodeCTable, sizeof(dstCCtx->offcodeCTable));
    }
+    if (srcCCtx->flagStaticHufTable) {
+        memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256*4);
+    }

    return 0;
 }
@ -493,24 +499,28 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc,

    /* small ? don't even attempt compression (speed opt) */
 #   define LITERAL_NOENTROPY 63
-    {   size_t const minLitSize = zc->flagStaticTables ? 6 : LITERAL_NOENTROPY;
+    {   size_t const minLitSize = zc->flagStaticHufTable == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY;
        if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
    }

    if (dstCapacity < lhSize+1) return ERROR(dstSize_tooSmall);   /* not enough space for compression */
-    if (zc->flagStaticTables && (lhSize==3)) {
-        hType = set_repeat;
-        singleStream = 1;
-        cLitSize = HUF_compress1X_usingCTable(ostart+lhSize, dstCapacity-lhSize, src, srcSize, zc->hufTable);
-    } else {
-        cLitSize = singleStream ? HUF_compress1X_wksp(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters))
-                                : HUF_compress4X_wksp(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters));
+    {   HUF_repeat repeat = zc->flagStaticHufTable;
+        int const preferRepeat = zc->params.cParams.strategy < ZSTD_lazy ? srcSize <= 1024 : 0;
+        if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
+        cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat)
+                                : HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat);
+        if (repeat != HUF_repeat_none) { hType = set_repeat; }    /* reused the existing table */
+        else { zc->flagStaticHufTable = HUF_repeat_check; }       /* now have a table to reuse */
    }

-    if ((cLitSize==0) | (cLitSize >= srcSize - minGain))
+    if ((cLitSize==0) | (cLitSize >= srcSize - minGain)) {
+        zc->flagStaticHufTable = HUF_repeat_none;
        return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
-    if (cLitSize==1)
+    }
+    if (cLitSize==1) {
+        zc->flagStaticHufTable = HUF_repeat_none;
        return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
+    }

    /* Build header */
    switch(lhSize)
@ -578,11 +588,11 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
        mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
 }

-
-size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
+MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
                              void* dst, size_t dstCapacity,
                              size_t srcSize)
 {
+    const int longOffsets = zc->params.cParams.windowLog > STREAM_ACCUMULATOR_MIN;
    const seqStore_t* seqStorePtr = &(zc->seqStore);
    U32 count[MaxSeq+1];
    S16 norm[MaxSeq+1];
@ -716,7 +726,18 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
        if (MEM_32bits()) BIT_flushBits(&blockStream);
        BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]);
        if (MEM_32bits()) BIT_flushBits(&blockStream);
-        BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]);
+        if (longOffsets) {
+            U32 const ofBits = ofCodeTable[nbSeq-1];
+            int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
+            if (extraBits) {
+                BIT_addBits(&blockStream, sequences[nbSeq-1].offset, extraBits);
+                BIT_flushBits(&blockStream);
+            }
+            BIT_addBits(&blockStream, sequences[nbSeq-1].offset >> extraBits,
+                        ofBits - extraBits);
+        } else {
+            BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]);
+        }
        BIT_flushBits(&blockStream);

        {   size_t n;
@ -738,7 +759,17 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
                if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream);
                BIT_addBits(&blockStream, sequences[n].matchLength, mlBits);
                if (MEM_32bits()) BIT_flushBits(&blockStream);                  /* (7)*/
-                BIT_addBits(&blockStream, sequences[n].offset, ofBits);         /* 31 */
+                if (longOffsets) {
+                    int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
+                    if (extraBits) {
+                        BIT_addBits(&blockStream, sequences[n].offset, extraBits);
+                        BIT_flushBits(&blockStream);                            /* (7)*/
+                    }
+                    BIT_addBits(&blockStream, sequences[n].offset >> extraBits,
+                                ofBits - extraBits);                            /* 31 */
+                } else {
+                    BIT_addBits(&blockStream, sequences[n].offset, ofBits);     /* 31 */
+                }
                BIT_flushBits(&blockStream);                                    /* (7)*/
        }   }

@ -753,9 +784,12 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,

    /* check compressibility */
 _check_compressibility:
-    { size_t const minGain = ZSTD_minGain(srcSize);
-      size_t const maxCSize = srcSize - minGain;
-      if ((size_t)(op-ostart) >= maxCSize) return 0; }
+    {   size_t const minGain = ZSTD_minGain(srcSize);
+        size_t const maxCSize = srcSize - minGain;
+        if ((size_t)(op-ostart) >= maxCSize) {
+            zc->flagStaticHufTable = HUF_repeat_none;
+            return 0;
+    }   }

    /* confirm repcodes */
    { int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = zc->repToConfirm[i]; }
@ -763,7 +797,6 @@ _check_compressibility:
    return op - ostart;
 }

-
 #if 0 /* for debug */
 #  define STORESEQ_DEBUG
 #include <stdio.h>   /* fprintf */
@ -2314,7 +2347,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
        if (remaining < blockSize) blockSize = remaining;

        /* preemptive overflow correction */
-        if (cctx->lowLimit > (2U<<30)) {
+        if (cctx->lowLimit > (3U<<29)) {
            U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->params.cParams.hashLog, cctx->params.cParams.strategy)) - 1;
            U32 const current = (U32)(ip - cctx->base);
            U32 const newCurrent = (current & cycleMask) + (1 << cctx->params.cParams.windowLog);
@ -2606,6 +2639,7 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_
    }

    cctx->flagStaticTables = 1;
+    cctx->flagStaticHufTable = HUF_repeat_valid;
    return dictPtr - (const BYTE*)dict;
 }

@ -3041,7 +3075,7 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
 size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
 {
    if (zcs==NULL) return 0;   /* support sizeof on NULL */
-    return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdictLocal) + zcs->outBuffSize + zcs->inBuffSize;
+    return sizeof(*zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdictLocal) + zcs->outBuffSize + zcs->inBuffSize;
 }

 /*======   Compression   ======*/
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@ -25,8 +25,6 @@
 #include "threading.h"  /* mutex */
 #include "zstd_internal.h"   /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
 #include "zstdmt_compress.h"
-#define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
-#include "xxhash.h"


 /* ======   Debug   ====== */
--- a/lib/decompress/huf_decompress.c
+++ b/lib/decompress/huf_decompress.c
@ -35,16 +35,19 @@
 /* **************************************************************
 *  Compiler specifics
 ****************************************************************/
-#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-/* inline is defined */
-#elif defined(_MSC_VER) || defined(__GNUC__)
-#  define inline __inline
-#else
-#  define inline /* disable inline */
-#endif
-
 #ifdef _MSC_VER    /* Visual Studio */
+#  define FORCE_INLINE static __forceinline
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
+#else
+#  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#    ifdef __GNUC__
+#      define FORCE_INLINE static inline __attribute__((always_inline))
+#    else
+#      define FORCE_INLINE static inline
+#    endif
+#  else
+#    define FORCE_INLINE static
+#  endif /* __STDC_VERSION__ */
 #endif


@ -108,10 +111,10 @@ size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize)
        memcpy(DTable, &dtd, sizeof(dtd));
    }

-    /* Prepare ranks */
+    /* Calculate starting value for each rank */
    {   U32 n, nextRankStart = 0;
        for (n=1; n<tableLog+1; n++) {
-            U32 current = nextRankStart;
+            U32 const current = nextRankStart;
            nextRankStart += (rankVal[n] << (n-1));
            rankVal[n] = current;
    }   }
@ -121,11 +124,11 @@ size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize)
        for (n=0; n<nbSymbols; n++) {
            U32 const w = huffWeight[n];
            U32 const length = (1 << w) >> 1;
-            U32 i;
+            U32 u;
            HUF_DEltX2 D;
            D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w);
-            for (i = rankVal[w]; i < rankVal[w] + length; i++)
-                dt[i] = D;
+            for (u = rankVal[w]; u < rankVal[w] + length; u++)
+                dt[u] = D;
            rankVal[w] += length;
    }   }

@ -152,7 +155,7 @@ static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, con
    if (MEM_64bits()) \
        HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr)

-static inline size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX2* const dt, const U32 dtLog)
+FORCE_INLINE size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX2* const dt, const U32 dtLog)
 {
    BYTE* const pStart = p;

@ -559,7 +562,7 @@ static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DE
    if (MEM_64bits()) \
        ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)

-static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUF_DEltX4* const dt, const U32 dtLog)
+FORCE_INLINE size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUF_DEltX4* const dt, const U32 dtLog)
 {
    BYTE* const pStart = p;

--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@ -43,8 +43,6 @@
 *********************************************************/
 #include <string.h>      /* memcpy, memmove, memset */
 #include "mem.h"         /* low level memory routines */
-#define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
-#include "xxhash.h"      /* XXH64_* */
 #define FSE_STATIC_LINKING_ONLY
 #include "fse.h"
 #define HUF_STATIC_LINKING_ONLY
@ -322,7 +320,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
 *             - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */
 unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
 {
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
    if (ZSTD_isLegacy(src, srcSize)) {
        unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize);
        return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret;
@ -878,7 +876,7 @@ typedef struct {
    size_t prevOffset[ZSTD_REP_NUM];
    const BYTE* base;
    size_t pos;
-    iPtrDiff gotoDict;
+    uPtrDiff gotoDict;
 } seqState_t;


@ -1035,7 +1033,7 @@ size_t ZSTD_execSequence(BYTE* op,
    if (sequence.offset > (size_t)(oLitEnd - base)) {
        /* offset beyond prefix */
        if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected);
-        match += (dictEnd-base);
+        match = dictEnd + (match - base);
        if (match + sequence.matchLength <= dictEnd) {
            memmove(oLitEnd, match, sequence.matchLength);
            return sequenceLength;
@ -1144,7 +1142,7 @@ static size_t ZSTD_decompressSequences(
 }


-static seq_t ZSTD_decodeSequenceLong(seqState_t* seqState)
+FORCE_INLINE seq_t ZSTD_decodeSequenceLong_generic(seqState_t* seqState, int const longOffsets)
 {
    seq_t seq;

@ -1179,8 +1177,15 @@ static seq_t ZSTD_decodeSequenceLong(seqState_t* seqState)
        if (!ofCode)
            offset = 0;
        else {
-            offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits);   /* <=  (ZSTD_WINDOWLOG_MAX-1) bits */
-            if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);
+            if (longOffsets) {
+                int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN);
+                offset = OF_base[ofCode] + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits);
+                if (MEM_32bits() || extraBits) BIT_reloadDStream(&seqState->DStream);
+                if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits);
+            } else {
+                offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits);   /* <=  (ZSTD_WINDOWLOG_MAX-1) bits */
+                if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);
+            }
        }

        if (ofCode <= 1) {
@ -1224,6 +1229,14 @@ static seq_t ZSTD_decodeSequenceLong(seqState_t* seqState)
    return seq;
 }

+static seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, unsigned const windowSize) {
+    if (ZSTD_highbit32(windowSize) > STREAM_ACCUMULATOR_MIN) {
+        return ZSTD_decodeSequenceLong_generic(seqState, 1);
+    } else {
+        return ZSTD_decodeSequenceLong_generic(seqState, 0);
+    }
+}
+
 FORCE_INLINE
 size_t ZSTD_execSequenceLong(BYTE* op,
                                BYTE* const oend, seq_t sequence,
@ -1321,6 +1334,7 @@ static size_t ZSTD_decompressSequencesLong(
    const BYTE* const base = (const BYTE*) (dctx->base);
    const BYTE* const vBase = (const BYTE*) (dctx->vBase);
    const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
+    unsigned const windowSize = dctx->fParams.windowSize;
    int nbSeq;

    /* Build Decoding Tables */
@ -1342,7 +1356,7 @@ static size_t ZSTD_decompressSequencesLong(
        { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
        seqState.base = base;
        seqState.pos = (size_t)(op-base);
-        seqState.gotoDict = (iPtrDiff)(dictEnd - base);
+        seqState.gotoDict = (uPtrDiff)dictEnd - (uPtrDiff)base; /* cast to avoid undefined behaviour */
        CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected);
        FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
        FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
@ -1350,13 +1364,13 @@ static size_t ZSTD_decompressSequencesLong(

        /* prepare in advance */
        for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && seqNb<seqAdvance; seqNb++) {
-            sequences[seqNb] = ZSTD_decodeSequenceLong(&seqState);
+            sequences[seqNb] = ZSTD_decodeSequenceLong(&seqState, windowSize);
        }
        if (seqNb<seqAdvance) return ERROR(corruption_detected);

        /* decode and decompress */
        for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && seqNb<nbSeq ; seqNb++) {
-            seq_t const sequence = ZSTD_decodeSequenceLong(&seqState);
+            seq_t const sequence = ZSTD_decodeSequenceLong(&seqState, windowSize);
            size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[(seqNb-ADVANCED_SEQS) & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd);
            if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
            ZSTD_PREFETCH(sequence.match);
@ -1396,13 +1410,18 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,

    if (srcSize >= ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(srcSize_wrong);

-    /* Decode literals sub-block */
+    /* Decode literals section */
    {   size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
        if (ZSTD_isError(litCSize)) return litCSize;
        ip += litCSize;
        srcSize -= litCSize;
    }
-    if (dctx->fParams.windowSize > (1<<23)) return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize);
+    if (sizeof(size_t) > 4)  /* do not enable prefetching on 32-bits x86, as it's performance detrimental */
+                             /* likely because of register pressure */
+                             /* if that's the correct cause, then 32-bits ARM should be affected differently */
+                             /* it would be good to test this on ARM real hardware, to see if prefetch version improves speed */
+        if (dctx->fParams.windowSize > (1<<23))
+            return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize);
    return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize);
 }

@ -1453,7 +1472,7 @@ size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t len
 *  @return : the compressed size of the frame starting at `src` */
 size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
 {
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
    if (ZSTD_isLegacy(src, srcSize)) return ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
 #endif
    if (srcSize >= ZSTD_skippableHeaderSize &&
@ -1469,8 +1488,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
        if (ZSTD_isError(headerSize)) return headerSize;

        /* Frame Header */
-        {
-            size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize);
+        {   size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize);
            if (ZSTD_isError(ret)) return ret;
            if (ret > 0) return ERROR(srcSize_wrong);
        }
@ -1503,7 +1521,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
 }

 /*! ZSTD_decompressFrame() :
-*   `dctx` must be properly initialized */
+*   @dctx must be properly initialized */
 static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
                                 void* dst, size_t dstCapacity,
                                 const void** srcPtr, size_t *srcSizePtr)
@ -1570,12 +1588,15 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
        remainingSize -= 4;
    }

-    // Allow caller to get size read
+    /* Allow caller to get size read */
    *srcPtr = ip;
    *srcSizePtr = remainingSize;
    return op-ostart;
 }

+static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict);
+static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict);
+
 static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
                                        void* dst, size_t dstCapacity,
                                  const void* src, size_t srcSize,
@ -1583,6 +1604,17 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
                                  const ZSTD_DDict* ddict)
 {
    void* const dststart = dst;
+
+    if (ddict) {
+        if (dict) {
+            /* programmer error, these two cases should be mutually exclusive */
+            return ERROR(GENERIC);
+        }
+
+        dict = ZSTD_DDictDictContent(ddict);
+        dictSize = ZSTD_DDictDictSize(ddict);
+    }
+
    while (srcSize >= ZSTD_frameHeaderSize_prefix) {
        U32 magicNumber;

@ -1938,6 +1970,16 @@ struct ZSTD_DDict_s {
    ZSTD_customMem cMem;
 };  /* typedef'd to ZSTD_DDict within "zstd.h" */

+static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict)
+{
+    return ddict->dictContent;
+}
+
+static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict)
+{
+    return ddict->dictSize;
+}
+
 static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict)
 {
    ZSTD_decompressBegin(dstDCtx);  /* init */
@ -1966,6 +2008,7 @@ static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict)

 static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict)
 {
+    ddict->dictID = 0;
    ddict->entropyPresent = 0;
    if (ddict->dictSize < 8) return 0;
    {   U32 const magic = MEM_readLE32(ddict->dictContent);
@ -2100,7 +2143,7 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
 {
    /* pass content and size in case legacy frames are encountered */
    return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize,
-                                     ddict->dictContent, ddict->dictSize,
+                                     NULL, 0,
                                     ddict);
 }

@ -2301,6 +2344,21 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                    break;
            }   }

+            /* check for single-pass mode opportunity */
+            if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */
+                && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) {
+                size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart);
+                if (cSize <= (size_t)(iend-istart)) {
+                    size_t const decompressedSize = ZSTD_decompress_usingDDict(zds->dctx, op, oend-op, istart, cSize, zds->ddict);
+                    if (ZSTD_isError(decompressedSize)) return decompressedSize;
+                    ip = istart + cSize;
+                    op += decompressedSize;
+                    zds->dctx->expected = 0;
+                    zds->stage = zdss_init;
+                    someMoreWork = 0;
+                    break;
+            }   }
+
            /* Consume header */
            ZSTD_refDDict(zds->dctx, zds->ddict);
            {   size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx);  /* == ZSTD_frameHeaderSize_prefix */
--- a/lib/dictBuilder/cover.c
+++ b/lib/dictBuilder/cover.c
@ -966,6 +966,7 @@ ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void *dictBuffer,
    if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d)) {
      LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
      COVER_best_destroy(&best);
+      POOL_free(pool);
      return ERROR(GENERIC);
    }
    /* Loop through k reusing the same context */
@ -978,6 +979,7 @@ ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void *dictBuffer,
        LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n");
        COVER_best_destroy(&best);
        COVER_ctx_destroy(&ctx);
+        POOL_free(pool);
        return ERROR(GENERIC);
      }
      data->ctx = &ctx;
@ -990,6 +992,7 @@ ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void *dictBuffer,
      /* Check the parameters */
      if (!COVER_checkParameters(data->parameters)) {
        DISPLAYLEVEL(1, "Cover parameters incorrect\n");
+        free(data);
        continue;
      }
      /* Call the function and pass ownership of data to it */
@ -1012,8 +1015,10 @@ ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void *dictBuffer,
  {
    const size_t dictSize = best.dictSize;
    if (ZSTD_isError(best.compressedSize)) {
+      const size_t compressedSize = best.compressedSize;
      COVER_best_destroy(&best);
-      return best.compressedSize;
+      POOL_free(pool);
+      return compressedSize;
    }
    *parameters = best.parameters;
    memcpy(dictBuffer, best.dict, dictSize);
--- a/lib/legacy/zstd_legacy.h
+++ b/lib/legacy/zstd_legacy.h
@ -20,14 +20,33 @@ extern "C" {
 #include "mem.h"            /* MEM_STATIC */
 #include "error_private.h"  /* ERROR */
 #include "zstd.h"           /* ZSTD_inBuffer, ZSTD_outBuffer */
-#include "zstd_v01.h"
-#include "zstd_v02.h"
-#include "zstd_v03.h"
-#include "zstd_v04.h"
-#include "zstd_v05.h"
-#include "zstd_v06.h"
-#include "zstd_v07.h"

+#if !defined (ZSTD_LEGACY_SUPPORT) || (ZSTD_LEGACY_SUPPORT == 0)
+#  undef ZSTD_LEGACY_SUPPORT
+#  define ZSTD_LEGACY_SUPPORT 8
+#endif
+
+#if (ZSTD_LEGACY_SUPPORT <= 1)
+#  include "zstd_v01.h"
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 2)
+#  include "zstd_v02.h"
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 3)
+#  include "zstd_v03.h"
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 4)
+#  include "zstd_v04.h"
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 5)
+#  include "zstd_v05.h"
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
+#  include "zstd_v06.h"
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
+#  include "zstd_v07.h"
+#endif

 /** ZSTD_isLegacy() :
    @return : > 0 if supported by legacy decoder. 0 otherwise.
@ -40,13 +59,27 @@ MEM_STATIC unsigned ZSTD_isLegacy(const void* src, size_t srcSize)
    magicNumberLE = MEM_readLE32(src);
    switch(magicNumberLE)
    {
+#if (ZSTD_LEGACY_SUPPORT <= 1)
        case ZSTDv01_magicNumberLE:return 1;
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 2)
        case ZSTDv02_magicNumber : return 2;
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 3)
        case ZSTDv03_magicNumber : return 3;
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 4)
        case ZSTDv04_magicNumber : return 4;
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 5)
        case ZSTDv05_MAGICNUMBER : return 5;
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
        case ZSTDv06_MAGICNUMBER : return 6;
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
        case ZSTDv07_MAGICNUMBER : return 7;
+#endif
        default : return 0;
    }
 }
@ -56,24 +89,30 @@ MEM_STATIC unsigned long long ZSTD_getDecompressedSize_legacy(const void* src, s
 {
    U32 const version = ZSTD_isLegacy(src, srcSize);
    if (version < 5) return 0;  /* no decompressed size in frame header, or not a legacy format */
+#if (ZSTD_LEGACY_SUPPORT <= 5)
    if (version==5) {
        ZSTDv05_parameters fParams;
        size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize);
        if (frResult != 0) return 0;
        return fParams.srcSize;
    }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
    if (version==6) {
        ZSTDv06_frameParams fParams;
        size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize);
        if (frResult != 0) return 0;
        return fParams.frameContentSize;
    }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
    if (version==7) {
        ZSTDv07_frameParams fParams;
        size_t const frResult = ZSTDv07_getFrameParams(&fParams, src, srcSize);
        if (frResult != 0) return 0;
        return fParams.frameContentSize;
    }
+#endif
    return 0;   /* should not be possible */
 }

@ -86,14 +125,23 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
    U32 const version = ZSTD_isLegacy(src, compressedSize);
    switch(version)
    {
+#if (ZSTD_LEGACY_SUPPORT <= 1)
        case 1 :
            return ZSTDv01_decompress(dst, dstCapacity, src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 2)
        case 2 :
            return ZSTDv02_decompress(dst, dstCapacity, src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 3)
        case 3 :
            return ZSTDv03_decompress(dst, dstCapacity, src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 4)
        case 4 :
            return ZSTDv04_decompress(dst, dstCapacity, src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 5)
        case 5 :
            {   size_t result;
                ZSTDv05_DCtx* const zd = ZSTDv05_createDCtx();
@ -102,6 +150,8 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
                ZSTDv05_freeDCtx(zd);
                return result;
            }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
        case 6 :
            {   size_t result;
                ZSTDv06_DCtx* const zd = ZSTDv06_createDCtx();
@ -110,6 +160,8 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
                ZSTDv06_freeDCtx(zd);
                return result;
            }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
        case 7 :
            {   size_t result;
                ZSTDv07_DCtx* const zd = ZSTDv07_createDCtx();
@ -118,6 +170,7 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
                ZSTDv07_freeDCtx(zd);
                return result;
            }
+#endif
        default :
            return ERROR(prefix_unknown);
    }
@ -129,20 +182,34 @@ MEM_STATIC size_t ZSTD_findFrameCompressedSizeLegacy(const void *src,
    U32 const version = ZSTD_isLegacy(src, compressedSize);
    switch(version)
    {
+#if (ZSTD_LEGACY_SUPPORT <= 1)
        case 1 :
            return ZSTDv01_findFrameCompressedSize(src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 2)
        case 2 :
            return ZSTDv02_findFrameCompressedSize(src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 3)
        case 3 :
            return ZSTDv03_findFrameCompressedSize(src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 4)
        case 4 :
            return ZSTDv04_findFrameCompressedSize(src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 5)
        case 5 :
            return ZSTDv05_findFrameCompressedSize(src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
        case 6 :
            return ZSTDv06_findFrameCompressedSize(src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
        case 7 :
            return ZSTDv07_findFrameCompressedSize(src, compressedSize);
+#endif
        default :
            return ERROR(prefix_unknown);
    }
@ -157,10 +224,18 @@ MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version)
        case 2 :
        case 3 :
            return ERROR(version_unsupported);
+#if (ZSTD_LEGACY_SUPPORT <= 4)
        case 4 : return ZBUFFv04_freeDCtx((ZBUFFv04_DCtx*)legacyContext);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 5)
        case 5 : return ZBUFFv05_freeDCtx((ZBUFFv05_DCtx*)legacyContext);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
        case 6 : return ZBUFFv06_freeDCtx((ZBUFFv06_DCtx*)legacyContext);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
        case 7 : return ZBUFFv07_freeDCtx((ZBUFFv07_DCtx*)legacyContext);
+#endif
    }
 }

@ -176,6 +251,7 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
        case 2 :
        case 3 :
            return 0;
+#if (ZSTD_LEGACY_SUPPORT <= 4)
        case 4 :
        {
            ZBUFFv04_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv04_createDCtx() : (ZBUFFv04_DCtx*)*legacyContext;
@ -185,6 +261,8 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
            *legacyContext = dctx;
            return 0;
        }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 5)
        case 5 :
        {
            ZBUFFv05_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv05_createDCtx() : (ZBUFFv05_DCtx*)*legacyContext;
@ -193,6 +271,8 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
            *legacyContext = dctx;
            return 0;
        }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
        case 6 :
        {
            ZBUFFv06_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv06_createDCtx() : (ZBUFFv06_DCtx*)*legacyContext;
@ -201,6 +281,8 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
            *legacyContext = dctx;
            return 0;
        }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
        case 7 :
        {
            ZBUFFv07_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv07_createDCtx() : (ZBUFFv07_DCtx*)*legacyContext;
@ -209,6 +291,7 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
            *legacyContext = dctx;
            return 0;
        }
+#endif
    }
 }

@ -224,6 +307,7 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
        case 2 :
        case 3 :
            return ERROR(version_unsupported);
+#if (ZSTD_LEGACY_SUPPORT <= 4)
        case 4 :
            {
                ZBUFFv04_DCtx* dctx = (ZBUFFv04_DCtx*) legacyContext;
@ -236,6 +320,8 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
                input->pos += readSize;
                return hintSize;
            }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 5)
        case 5 :
            {
                ZBUFFv05_DCtx* dctx = (ZBUFFv05_DCtx*) legacyContext;
@ -248,6 +334,8 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
                input->pos += readSize;
                return hintSize;
            }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
        case 6 :
            {
                ZBUFFv06_DCtx* dctx = (ZBUFFv06_DCtx*) legacyContext;
@ -260,6 +348,8 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
                input->pos += readSize;
                return hintSize;
            }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
        case 7 :
            {
                ZBUFFv07_DCtx* dctx = (ZBUFFv07_DCtx*) legacyContext;
@ -272,6 +362,7 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
                input->pos += readSize;
                return hintSize;
            }
+#endif
    }
 }

--- a/lib/legacy/zstd_v07.c
+++ b/lib/legacy/zstd_v07.c
@ -13,12 +13,14 @@
 #include <string.h>     /* memcpy */
 #include <stdlib.h>     /* malloc, free, qsort */

-#define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
-#include "xxhash.h"      /* XXH64_* */
+#ifndef XXH_STATIC_LINKING_ONLY
+#  define XXH_STATIC_LINKING_ONLY    /* XXH64_state_t */
+#endif
+#include "xxhash.h"                  /* XXH64_* */
 #include "zstd_v07.h"

-#define FSEv07_STATIC_LINKING_ONLY  /* FSEv07_MIN_TABLELOG */
-#define HUFv07_STATIC_LINKING_ONLY  /* HUFv07_TABLELOG_ABSOLUTEMAX */
+#define FSEv07_STATIC_LINKING_ONLY   /* FSEv07_MIN_TABLELOG */
+#define HUFv07_STATIC_LINKING_ONLY   /* HUFv07_TABLELOG_ABSOLUTEMAX */
 #define ZSTDv07_STATIC_LINKING_ONLY

 #include "error_private.h"
@ -4536,7 +4538,8 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd,
                    if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; }   /* this was just a header */
                    zbd->outEnd = zbd->outStart +  decodedSize;
                    zbd->stage = ZBUFFds_flush;
-                    // break; /* ZBUFFds_flush follows */
+                    /* break; */
+                    /* pass-through */
            }   }

        case ZBUFFds_flush:
--- a/lib/zstd.h
+++ b/lib/zstd.h
@ -139,7 +139,11 @@ ZSTDLIB_API size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
    Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). */
 ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);

-/*= Decompression context */
+/*= Decompression context
+*   When decompressing many times,
+*   it is recommended to allocate a context just once, and re-use it for each successive compression operation.
+*   This will make workload friendlier for system's memory.
+*   Use one context per thread for parallel execution in multi-threaded environments. */
 typedef struct ZSTD_DCtx_s ZSTD_DCtx;
 ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
 ZSTDLIB_API size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
@ -277,9 +281,11 @@ typedef struct ZSTD_outBuffer_s {
 * *******************************************************************/

 typedef struct ZSTD_CStream_s ZSTD_CStream;
+/*===== ZSTD_CStream management functions =====*/
 ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
 ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);

+/*===== Streaming compression functions =====*/
 ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
 ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
 ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
@ -313,9 +319,11 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output
 * *******************************************************************************/

 typedef struct ZSTD_DStream_s ZSTD_DStream;
+/*===== ZSTD_DStream management functions =====*/
 ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
 ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);

+/*===== Streaming decompression functions =====*/
 ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
 ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);

@ -343,9 +351,9 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output
 #define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
 #define ZSTD_CONTENTSIZE_ERROR   (0ULL - 2)

-#define ZSTD_WINDOWLOG_MAX_32  25
+#define ZSTD_WINDOWLOG_MAX_32  27
 #define ZSTD_WINDOWLOG_MAX_64  27
-#define ZSTD_WINDOWLOG_MAX    ((U32)(MEM_32bits() ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
+#define ZSTD_WINDOWLOG_MAX    ((unsigned)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
 #define ZSTD_WINDOWLOG_MIN     10
 #define ZSTD_HASHLOG_MAX       ZSTD_WINDOWLOG_MAX
 #define ZSTD_HASHLOG_MIN        6
@ -540,6 +548,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
 *  It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict */
 ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);

+/*! ZSTD_createDDict_advanced() :
+ *  Create a ZSTD_DDict using external alloc and free, optionally by reference */
 ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
                                                  unsigned byReference, ZSTD_customMem customMem);

--- a/programs/Makefile
+++ b/programs/Makefile
@ -18,17 +18,32 @@

 ZSTDDIR = ../lib

+# Version numbers
+LIBVER_SRC := $(ZSTDDIR)/zstd.h
+LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
+LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
+LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LIBVER_SRC)`
+LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
+LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT))
+LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
+LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
+LIBVER  := $(shell echo $(LIBVER_SCRIPT))
+
+ZSTD_VERSION=$(LIBVER)
+
 ifeq ($(shell $(CC) -v 2>&1 | grep -c "gcc version "), 1)
 ALIGN_LOOP = -falign-loops=32
 else
 ALIGN_LOOP =
 endif

-CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress -I$(ZSTDDIR)/dictBuilder
+CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
+           -I$(ZSTDDIR)/dictBuilder \
+           -DXXH_NAMESPACE=ZSTD_   # because xxhash.o already compiled with this macro from library
 CFLAGS  ?= -O3
 DEBUGFLAGS = -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
          -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-          -Wstrict-prototypes -Wundef -Wpointer-arith
+          -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security
 CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
 FLAGS    = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)

@ -40,12 +55,14 @@ ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
 ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
 ZSTDDECOMP_O = $(ZSTDDIR)/decompress/zstd_decompress.o

-ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
+ZSTD_LEGACY_SUPPORT ?= 4
 ZSTDLEGACY_FILES:=
+ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
+ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
+	ZSTDLEGACY_FILES += $(shell ls $(ZSTDDIR)/legacy/*.c | grep 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
+endif
+	CPPFLAGS += -I$(ZSTDDIR)/legacy
 else
-ZSTD_LEGACY_SUPPORT:=1
-CPPFLAGS  += -I$(ZSTDDIR)/legacy
-ZSTDLEGACY_FILES:= $(ZSTDDIR)/legacy/*.c
 endif

 ZSTDLIB_FILES := $(wildcard $(ZSTD_FILES)) $(wildcard $(ZSTDLEGACY_FILES)) $(wildcard $(ZDICT_FILES))
@ -66,12 +83,29 @@ EXT =
 endif

 # zlib detection
+NO_ZLIB_MSG := ==> no zlib, building zstd without .gz support
 VOID = /dev/null
 HAVE_ZLIB := $(shell printf '\#include <zlib.h>\nint main(){}' | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0)
 ifeq ($(HAVE_ZLIB), 1)
+ZLIB_MSG := ==> building zstd with .gz compression support
 ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS
 ZLIBLD = -lz
+else
+ZLIB_MSG := $(NO_ZLIB_MSG)
 endif
+# lzma detection
+NO_LZMA_MSG := ==> no liblzma, building zstd without .xz/.lzma support
+HAVE_LZMA := $(shell printf '\#include <lzma.h>\nint main(){}' | $(CC) -o have_lzma -x c - -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0)
+ifeq ($(HAVE_LZMA), 1)
+LZMA_MSG := ==> building zstd with .xz/.lzma compression support
+LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS
+LZMALD = -llzma
+else
+LZMA_MSG := $(NO_LZMA_MSG)
+endif
+
+MD2ROFF  =ronn
+MD2ROFF_FLAGS  = --roff --warnings --manual="User Commands" --organization="zstd $(ZSTD_VERSION)"

 .PHONY: default all clean clean_decomp_o install uninstall generate_res

@ -81,25 +115,22 @@ all: zstd

 $(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)

-zstd-internal : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
-zstd-internal : $(ZSTDLIB_OBJ) zstdcli.o fileio.o bench.o datagen.o dibio.o
-ifeq ($(HAVE_ZLIB), 1)
-	@echo "==> building zstd with .gz decompression support "
-else
-	@echo "==> no zlib, building zstd with .zst support only (no .gz support) "
-endif
+zstd : CPPFLAGS += $(ZLIBCPP)
+zstd : LDFLAGS += $(ZLIBLD)
+zstd : LZMA_MSG := $(NO_LZMA_MSG)
+zstd-nogz : ZLIB_MSG := $(NO_ZLIB_MSG)
+zstd-nogz : LZMA_MSG := $(NO_LZMA_MSG)
+xzstd : CPPFLAGS += $(ZLIBCPP) $(LZMACPP)
+xzstd : LDFLAGS += $(ZLIBLD) $(LZMALD)
+zstd zstd-nogz xzstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
+zstd zstd-nogz xzstd : $(ZSTDLIB_OBJ) zstdcli.o fileio.o bench.o datagen.o dibio.o
+	@echo "$(ZLIB_MSG)"
+	@echo "$(LZMA_MSG)"
 ifneq (,$(filter Windows%,$(OS)))
 	windres/generate_res.bat
 endif
 	$(CC) $(FLAGS) $^ $(RES_FILE) -o zstd$(EXT) $(LDFLAGS)

-zstd-nogz : HAVE_ZLIB=0
-zstd-nogz : zstd-internal
-
-zstd : CPPFLAGS += $(ZLIBCPP)
-zstd : LDFLAGS += $(ZLIBLD)
-zstd : zstd-internal
-
 zstd-release: DEBUGFLAGS :=
 zstd-release: zstd

@ -158,6 +189,16 @@ clean:
 clean_decomp_o:
 	@$(RM) $(ZSTDDECOMP_O)

+zstd.1: zstd.1.md
+	cat $^ | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
+
+man: zstd.1
+
+clean-man:
+	rm zstd.1
+
+preview-man: clean-man man
+	man ./zstd.1

 #-----------------------------------------------------------------------------
 # make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
--- a/programs/bench.c
+++ b/programs/bench.c
@ -581,6 +581,7 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* di
 {
    double const compressibility = (double)g_compressibilityDefault / 100;

+    if (cLevel < 1) cLevel = 1;   /* minimum compression level */
    if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
    if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
    if (cLevelLast < cLevel) cLevelLast = cLevel;
--- a/programs/fileio.c
+++ b/programs/fileio.c
@ -44,6 +44,9 @@
 #    define z_const
 #  endif
 #endif
+#if defined(ZSTD_LZMACOMPRESS) || defined(ZSTD_LZMADECOMPRESS)
+#  include <lzma.h>
+#endif


 /*-*************************************
@ -71,7 +74,6 @@
 #define MAX_DICT_SIZE (8 MB)   /* protection against large input (attack scenario) */

 #define FNSPACE 30
-#define GZ_EXTENSION ".gz"


 /*-*************************************
@ -434,6 +436,65 @@ static unsigned long long FIO_compressGzFrame(cRess_t* ress, const char* srcFile
 #endif


+#ifdef ZSTD_LZMACOMPRESS
+static unsigned long long FIO_compressLzmaFrame(cRess_t* ress, const char* srcFileName, U64 const srcFileSize, int compressionLevel, U64* readsize, int plain_lzma)
+{
+    unsigned long long inFileSize = 0, outFileSize = 0;
+    lzma_stream strm = LZMA_STREAM_INIT;
+    lzma_action action = LZMA_RUN;
+    lzma_ret ret;
+
+    if (compressionLevel < 0) compressionLevel = 0;
+    if (compressionLevel > 9) compressionLevel = 9;
+
+    if (plain_lzma) {
+        lzma_options_lzma opt_lzma;
+        if (lzma_lzma_preset(&opt_lzma, compressionLevel)) EXM_THROW(71, "zstd: %s: lzma_lzma_preset error", srcFileName);
+        ret = lzma_alone_encoder(&strm, &opt_lzma); /* LZMA */
+        if (ret != LZMA_OK) EXM_THROW(71, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret);
+    } else {
+        ret = lzma_easy_encoder(&strm, compressionLevel, LZMA_CHECK_CRC64); /* XZ */
+        if (ret != LZMA_OK) EXM_THROW(71, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret);
+    }
+
+    strm.next_in = 0;
+    strm.avail_in = 0;
+    strm.next_out = ress->dstBuffer;
+    strm.avail_out = ress->dstBufferSize;
+
+    while (1) {
+        if (strm.avail_in == 0) {
+            size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile);
+            if (inSize == 0) action = LZMA_FINISH;
+            inFileSize += inSize;
+            strm.next_in = ress->srcBuffer;
+            strm.avail_in = inSize;
+        }
+
+        ret = lzma_code(&strm, action);
+
+        if (ret != LZMA_OK && ret != LZMA_STREAM_END) EXM_THROW(72, "zstd: %s: lzma_code encoding error %d", srcFileName, ret);
+        {   size_t const compBytes = ress->dstBufferSize - strm.avail_out;
+            if (compBytes) {
+                if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes) EXM_THROW(73, "Write error : cannot write to output file");
+                outFileSize += compBytes;
+                strm.next_out = ress->dstBuffer;
+                strm.avail_out = ress->dstBufferSize;
+            }
+        }
+        if (!srcFileSize) DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", (U32)(inFileSize>>20), (double)outFileSize/inFileSize*100)
+        else DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", (U32)(inFileSize>>20), (U32)(srcFileSize>>20), (double)outFileSize/inFileSize*100);
+        if (ret == LZMA_STREAM_END) break;
+    }
+
+    lzma_end(&strm);
+    *readsize = inFileSize;
+
+    return outFileSize;
+}
+#endif
+
+
 /*! FIO_compressFilename_internal() :
 *  same as FIO_compressFilename_extRess(), with `ress.desFile` already opened.
 *  @return : 0 : compression completed correctly,
@ -448,14 +509,26 @@ static int FIO_compressFilename_internal(cRess_t ress,
    U64 compressedfilesize = 0;
    U64 const fileSize = UTIL_getFileSize(srcFileName);

-    if (g_compressionType) {
+    switch (g_compressionType) {
+        case FIO_zstdCompression:
+            break;
+        case FIO_gzipCompression:
 #ifdef ZSTD_GZCOMPRESS
-        compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize);
+            compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize);
 #else
-        (void)compressionLevel;
-        EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", srcFileName);
+            (void)compressionLevel;
+            EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", srcFileName);
 #endif
-        goto finish;
+            goto finish;
+        case FIO_xzCompression:
+        case FIO_lzmaCompression:
+#ifdef ZSTD_LZMACOMPRESS
+            compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, g_compressionType==FIO_lzmaCompression);
+#else
+            (void)compressionLevel;
+            EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n", srcFileName);
+#endif
+            goto finish;
    }

    /* init */
@ -763,10 +836,10 @@ static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
 {
    if (storedSkips-->0) {   /* implies g_sparseFileSupport>0 */
        int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
-        if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n");
+        if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)");
        {   const char lastZeroByte[1] = { 0 };
            size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file);
-            if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n");
+            if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero");
    }   }
 }

@ -849,6 +922,7 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, co
 {
    unsigned long long outFileSize = 0;
    z_stream strm;
+    int flush = Z_NO_FLUSH;
    int ret;

    strm.zalloc = Z_NULL;
@ -866,11 +940,12 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, co
    for ( ; ; ) {
        if (strm.avail_in == 0) {
            ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile);
-            if (ress->srcBufferLoaded == 0) break;
+            if (ress->srcBufferLoaded == 0) flush = Z_FINISH;
            strm.next_in = (z_const unsigned char*)ress->srcBuffer;
            strm.avail_in = (uInt)ress->srcBufferLoaded;
        }
-        ret = inflate(&strm, Z_NO_FLUSH);
+        ret = inflate(&strm, flush);
+        if (ret == Z_BUF_ERROR) EXM_THROW(39, "zstd: %s: premature end", srcFileName);
        if (ret != Z_OK && ret != Z_STREAM_END) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; }
        {   size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
            if (decompBytes) {
@ -886,7 +961,60 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, co
    if (strm.avail_in > 0) memmove(ress->srcBuffer, strm.next_in, strm.avail_in);
    ress->srcBufferLoaded = strm.avail_in;
    ret = inflateEnd(&strm);
-    if (ret != Z_OK) EXM_THROW(32, "zstd: %s: inflateEnd error %d \n", srcFileName, ret);
+    if (ret != Z_OK) EXM_THROW(32, "zstd: %s: inflateEnd error %d", srcFileName, ret);
+    return outFileSize;
+}
+#endif
+
+
+#ifdef ZSTD_LZMADECOMPRESS
+static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName, int plain_lzma)
+{
+    unsigned long long outFileSize = 0;
+    lzma_stream strm = LZMA_STREAM_INIT;
+    lzma_action action = LZMA_RUN;
+    lzma_ret ret;
+
+    strm.next_in = 0;
+    strm.avail_in = 0;
+    if (plain_lzma) {
+        ret = lzma_alone_decoder(&strm, UINT64_MAX); /* LZMA */
+    } else {
+        ret = lzma_stream_decoder(&strm, UINT64_MAX, 0); /* XZ */
+    }
+
+    if (ret != LZMA_OK) EXM_THROW(71, "zstd: %s: lzma_alone_decoder/lzma_stream_decoder error %d", srcFileName, ret);
+
+    strm.next_out = ress->dstBuffer;
+    strm.avail_out = ress->dstBufferSize;
+    strm.avail_in = ress->srcBufferLoaded;
+    strm.next_in = ress->srcBuffer;
+
+    for ( ; ; ) {
+        if (strm.avail_in == 0) {
+            ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile);
+            if (ress->srcBufferLoaded == 0) action = LZMA_FINISH;
+            strm.next_in = ress->srcBuffer;
+            strm.avail_in = ress->srcBufferLoaded;
+        }
+        ret = lzma_code(&strm, action);
+
+        if (ret == LZMA_BUF_ERROR) EXM_THROW(39, "zstd: %s: premature end", srcFileName);
+        if (ret != LZMA_OK && ret != LZMA_STREAM_END) { DISPLAY("zstd: %s: lzma_code decoding error %d \n", srcFileName, ret); return 0; }
+        {   size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
+            if (decompBytes) {
+                if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(31, "Write error : cannot write to output file");
+                outFileSize += decompBytes;
+                strm.next_out = ress->dstBuffer;
+                strm.avail_out = ress->dstBufferSize;
+            }
+        }
+        if (ret == LZMA_STREAM_END) break;
+    }
+
+    if (strm.avail_in > 0) memmove(ress->srcBuffer, strm.next_in, strm.avail_in);
+    ress->srcBufferLoaded = strm.avail_in;
+    lzma_end(&strm);
    return outFileSize;
 }
 #endif
@ -924,7 +1052,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
        }
        readSomething = 1;   /* there is at least >= 4 bytes in srcFile */
        if (ress.srcBufferLoaded < toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; }  /* srcFileName is empty */
-        if (buf[0] == 31 && buf[1] == 139) { /* gz header */
+        if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */
 #ifdef ZSTD_GZDECOMPRESS
            unsigned long long const result = FIO_decompressGzFrame(&ress, srcFile, srcFileName);
            if (result == 0) return 1;
@ -932,6 +1060,16 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
 #else
            DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without ZSTD_GZDECOMPRESS) -- ignored \n", srcFileName);
            return 1;
+#endif
+        } else if ((buf[0] == 0xFD && buf[1] == 0x37)  /* xz magic number */
+                || (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */
+#ifdef ZSTD_LZMADECOMPRESS
+            unsigned long long const result = FIO_decompressLzmaFrame(&ress, srcFile, srcFileName, buf[0] != 0xFD);
+            if (result == 0) return 1;
+            filesize += result;
+#else
+            DISPLAYLEVEL(1, "zstd: %s: xz/lzma file cannot be uncompressed (zstd compiled without ZSTD_LZMADECOMPRESS) -- ignored \n", srcFileName);
+            return 1;
 #endif
        } else {
            if (!ZSTD_isFrame(ress.srcBuffer, toRead)) {
@ -1020,32 +1158,31 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
            missingFiles += FIO_decompressSrcFile(ress, suffix, srcNamesTable[u]);
        if (fclose(ress.dstFile)) EXM_THROW(72, "Write error : cannot properly close stdout");
    } else {
-        size_t const suffixSize = strlen(suffix);
-        size_t const gzipSuffixSize = strlen(GZ_EXTENSION);
+        size_t suffixSize;
        size_t dfnSize = FNSPACE;
        unsigned u;
        char* dstFileName = (char*)malloc(FNSPACE);
        if (dstFileName==NULL) EXM_THROW(73, "not enough memory for dstFileName");
        for (u=0; u<nbFiles; u++) {   /* create dstFileName */
            const char* const srcFileName = srcNamesTable[u];
+            const char* const suffixPtr = strrchr(srcFileName, '.');
            size_t const sfnSize = strlen(srcFileName);
-            const char* const suffixPtr = srcFileName + sfnSize - suffixSize;
-            const char* const gzipSuffixPtr = srcFileName + sfnSize - gzipSuffixSize;
+            if (!suffixPtr) {
+                DISPLAYLEVEL(1, "zstd: %s: unknown suffix -- ignored \n", srcFileName);
+                skippedFiles++;
+                continue;
+            }
+            suffixSize = strlen(suffixPtr);
            if (dfnSize+suffixSize <= sfnSize+1) {
                free(dstFileName);
                dfnSize = sfnSize + 20;
                dstFileName = (char*)malloc(dfnSize);
                if (dstFileName==NULL) EXM_THROW(74, "not enough memory for dstFileName");
            }
-            if (sfnSize <= suffixSize || strcmp(suffixPtr, suffix) != 0) {
-                if (sfnSize <= gzipSuffixSize || strcmp(gzipSuffixPtr, GZ_EXTENSION) != 0) {
-                    DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s/%s expected) -- ignored \n", srcFileName, suffix, GZ_EXTENSION);
-                    skippedFiles++;
-                    continue;
-                } else {
-                    memcpy(dstFileName, srcFileName, sfnSize - gzipSuffixSize);
-                    dstFileName[sfnSize-gzipSuffixSize] = '\0';
-                }
+            if (sfnSize <= suffixSize || (strcmp(suffixPtr, GZ_EXTENSION) && strcmp(suffixPtr, XZ_EXTENSION) && strcmp(suffixPtr, ZSTD_EXTENSION) && strcmp(suffixPtr, LZMA_EXTENSION))) {
+                DISPLAYLEVEL(1, "zstd: %s: unknown suffix (%s/%s/%s/%s expected) -- ignored \n", srcFileName, GZ_EXTENSION, XZ_EXTENSION, ZSTD_EXTENSION, LZMA_EXTENSION);
+                skippedFiles++;
+                continue;
            } else {
                memcpy(dstFileName, srcFileName, sfnSize - suffixSize);
                dstFileName[sfnSize-suffixSize] = '\0';
--- a/programs/fileio.h
+++ b/programs/fileio.h
@ -29,12 +29,16 @@ extern "C" {
 #else
 #  define nulmark "/dev/null"
 #endif
+#define LZMA_EXTENSION  ".lzma"
+#define XZ_EXTENSION    ".xz"
+#define GZ_EXTENSION    ".gz"
+#define ZSTD_EXTENSION  ".zst"


 /*-*************************************
 *  Types
 ***************************************/
-typedef enum { FIO_zstdCompression, FIO_gzipCompression } FIO_compressionType_t;
+typedef enum { FIO_zstdCompression, FIO_gzipCompression, FIO_xzCompression, FIO_lzmaCompression } FIO_compressionType_t;


 /*-*************************************
--- a/programs/zstd.1
+++ b/programs/zstd.1
@ -1,408 +1,306 @@
-\"
-\" zstd.1: This is a manual page for 'zstd' program. This file is part of the
-\" zstd <http://www.zstd.net/> project.
-\" Author: Yann Collet
-\"
-
-\" No hyphenation
-.hy 0
-.nr HY 0
-
-.TH zstd "1" "2015-08-22" "zstd" "User Commands"
-.SH NAME
-\fBzstd, unzstd, zstdcat\fR - Compress or decompress .zst files
-
-.SH SYNOPSIS
-.TP 5
-\fBzstd\fR [\fBOPTIONS\fR] [-|INPUT-FILE] [-o <OUTPUT-FILE>]
-.PP
-.B unzstd
-is equivalent to
-.BR "zstd \-d"
-.br
-.B zstdcat
-is equivalent to
-.BR "zstd \-dcf"
-.br
-
-.SH DESCRIPTION
-.PP
-\fBzstd\fR is a fast lossless compression algorithm
-and data compression tool,
-with command line syntax similar to \fB gzip (1) \fR and \fB xz (1) \fR .
-It is based on the \fBLZ77\fR family, with further FSE & huff0 entropy stages.
-\fBzstd\fR offers highly configurable compression speed,
-with fast modes at > 200 MB/s per core,
-and strong modes nearing lzma compression ratios.
-It also features a very fast decoder, with speeds > 500 MB/s per core.
-
-\fBzstd\fR command line syntax is generally similar to gzip,
-but features the following differences :
- - Source files are preserved by default.
-   It's possible to remove them automatically by using \fB--rm\fR command.
- - When compressing a single file, \fBzstd\fR displays progress notifications and result summary by default.
-   Use \fB-q\fR to turn them off
-
-.PP
-.B zstd
-compresses or decompresses each
-.I file
-according to the selected operation mode.
-If no
-.I files
-are given or
-.I file
-is
-.BR \- ,
-.B zstd
-reads from standard input and writes the processed data
-to standard output.
-.B zstd
-will refuse (display an error and skip the
-.IR file )
-to write compressed data to standard output if it is a terminal.
-Similarly,
-.B zstd
-will refuse to read compressed data
-from standard input if it is a terminal.
-
-.PP
-Unless
-.B \-\-stdout
-or
-.B \-o
-is specified,
-.I files
-are written to a new file whose name is derived from the source
-.I file
-name:
-.IP \(bu 3
-When compressing, the suffix
-.B .zst
-is appended to the source filename to get the target filename.
-.IP \(bu 3
-When decompressing, the
-.B .zst
-suffix is removed from the filename to get the target filename.
-
-.SS "Concatenation with .zst files"
-It is possible to concatenate
-.B .zst
-files as is.
-.B zstd
-will decompress such files as if they were a single
-.B .zst
-file.
-
-
-
-.SH OPTIONS
-
+.
+.TH "ZSTD" "1" "March 2017" "zstd 1.1.4" "User Commands"
+.
+.SH "NAME"
+\fBzstd\fR \- zstd, unzstd, zstdcat \- Compress or decompress \.zst files
+.
+.SH "SYNOPSIS"
+\fBzstd\fR [\fIOPTIONS\fR] [\-|<INPUT\-FILE>] [\-o <OUTPUT\-FILE>]
+.
+.P
+\fBunzstd\fR is equivalent to \fBzstd \-d\fR
+.
+.P
+\fBzstdcat\fR is equivalent to \fBzstd \-dcf\fR
+.
+.SH "DESCRIPTION"
+\fBzstd\fR is a fast lossless compression algorithm and data compression tool, with command line syntax similar to \fBgzip (1)\fR and \fBxz (1)\fR\. It is based on the \fBLZ77\fR family, with further FSE & huff0 entropy stages\. \fBzstd\fR offers highly configurable compression speed, with fast modes at > 200 MB/s per code, and strong modes nearing lzma compression ratios\. It also features a very fast decoder, with speeds > 500 MB/s per core\.
+.
+.P
+\fBzstd\fR command line syntax is generally similar to gzip, but features the following differences :
+.
+.IP "\(bu" 4
+Source files are preserved by default\. It\'s possible to remove them automatically by using the \fB\-\-rm\fR command\.
+.
+.IP "\(bu" 4
+When compressing a single file, \fBzstd\fR displays progress notifications and result summary by default\. Use \fB\-q\fR to turn them off\.
+.
+.IP "\(bu" 4
+\fBzstd\fR does not accept input from console, but it properly accepts \fBstdin\fR when it\'s not the console\.
+.
+.IP "\(bu" 4
+\fBzstd\fR displays a short help page when command line is an error\. Use \fB\-q\fR to turn it off\.
+.
+.IP "" 0
+.
+.P
+\fBzstd\fR compresses or decompresses each \fIfile\fR according to the selected operation mode\. If no \fIfiles\fR are given or \fIfile\fR is \fB\-\fR, \fBzstd\fR reads from standard input and writes the processed data to standard output\. \fBzstd\fR will refuse to write compressed data to standard output if it is a terminal : it will display an error message and skip the \fIfile\fR\. Similarly, \fBzstd\fR will refuse to read compressed data from standard input if it is a terminal\.
+.
+.P
+Unless \fB\-\-stdout\fR or \fB\-o\fR is specified, \fIfiles\fR are written to a new file whose name is derived from the source \fIfile\fR name:
+.
+.IP "\(bu" 4
+When compressing, the suffix \fB\.zst\fR is appended to the source filename to get the target filename\.
+.
+.IP "\(bu" 4
+When decompressing, the \fB\.zst\fR suffix is removed from the source filename to get the target filename
+.
+.IP "" 0
+.
+.SS "Concatenation with \.zst files"
+It is possible to concatenate \fB\.zst\fR files as is\. \fBzstd\fR will decompress such files as if they were a single \fB\.zst\fR file\.
+.
+.SH "OPTIONS"
 .
 .SS "Integer suffixes and special values"
-In most places where an integer argument is expected,
-an optional suffix is supported to easily indicate large integers.
-There must be no space between the integer and the suffix.
+In most places where an integer argument is expected, an optional suffix is supported to easily indicate large integers\. There must be no space between the integer and the suffix\.
+.
 .TP
-.B KiB
-Multiply the integer by 1,024 (2^10).
-.BR Ki ,
-.BR K ,
-and
-.B KB
-are accepted as synonyms for
-.BR KiB .
+\fBKiB\fR
+Multiply the integer by 1,024 (2^10)\. \fBKi\fR, \fBK\fR, and \fBKB\fR are accepted as synonyms for \fBKiB\fR\.
+.
 .TP
-.B MiB
-Multiply the integer by 1,048,576 (2^20).
-.BR Mi ,
-.BR M ,
-and
-.B MB
-are accepted as synonyms for
-.BR MiB .
-
+\fBMiB\fR
+Multiply the integer by 1,048,576 (2^20)\. \fBMi\fR, \fBM\fR, and \fBMB\fR are accepted as synonyms for \fBMiB\fR\.
 .
 .SS "Operation mode"
-If multiple operation mode options are given,
-the last one takes effect.
+If multiple operation mode options are given, the last one takes effect\.
+.
 .TP
-.BR \-z ", " \-\-compress
-Compress.
-This is the default operation mode when no operation mode option
-is specified and no other operation mode is implied from
-the command name (for example,
-.B unzstd
-implies
-.BR \-\-decompress ).
+\fB\-z\fR, \fB\-\-compress\fR
+Compress\. This is the default operation mode when no operation mode option is specified and no other operation mode is implied from the command name (for example, \fBunzstd\fR implies \fB\-\-decompress\fR)\.
+.
 .TP
-.BR \-d ", " \-\-decompress ", " \-\-uncompress
-Decompress.
+\fB\-d\fR, \fB\-\-decompress\fR, \fB\-\-uncompress\fR
+Decompress\.
+.
 .TP
-.BR \-t ", " \-\-test
-Test the integrity of compressed
-.IR files .
-This option is equivalent to
-.B "\-\-decompress \-\-stdout"
-except that the decompressed data is discarded instead of being
-written to standard output.
-No files are created or removed.
+\fB\-t\fR, \fB\-\-test\fR
+Test the integrity of compressed \fIfiles\fR\. This option is equivalent to \fB\-\-decompress \-\-stdout\fR except that the decompressed data is discarded instead of being written to standard output\. No files are created or removed\.
+.
 .TP
-.B \-b#
- benchmark file(s) using compression level #
+\fB\-b#\fR
+Benchmark file(s) using compression level #
+.
 .TP
-.B \--train FILEs
- use FILEs as training set to create a dictionary. The training set should contain a lot of small files (> 100).
-
+\fB\-\-train FILEs\fR
+Use FILEs as a training set to create a dictionary\. The training set should contain a lot of small files (> 100)\.
 .
 .SS "Operation modifiers"
+.
 .TP
-.B \-#
- # compression level [1-19] (default:3)
+\fB\-#\fR
+\fB#\fR compression level [1\-19] (default: 3)
+.
 .TP
-.BR \--ultra
- unlocks high compression levels 20+ (maximum 22), using a lot more memory.
-Note that decompression will also require more memory when using these levels.
+\fB\-\-ultra\fR
+unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\.
+.
 .TP
-.B \-D file
- use `file` as Dictionary to compress or decompress FILE(s)
+\fB\-D file\fR
+use \fBfile\fR as Dictionary to compress or decompress FILE(s)
+.
 .TP
-.BR \--no-dictID
- do not store dictionary ID within frame header (dictionary compression).
- The decoder will have to rely on implicit knowledge about which dictionary to use,
-it won't be able to check if it's correct.
+\fB\-\-nodictID\fR
+do not store dictionary ID within frame header (dictionary compression)\. The decoder will have to rely on implicit knowledge about which dictionary to use, it won\'t be able to check if it\'s correct\.
+.
 .TP
-.B \-o file
- save result into `file` (only possible with a single INPUT-FILE)
+\fB\-o file\fR
+save result into \fBfile\fR (only possible with a single INPUT\-FILE)
+.
 .TP
-.BR \-f ", " --force
- overwrite output without prompting
+\fB\-f\fR, \fB\-\-force\fR
+overwrite output without prompting
+.
 .TP
-.BR \-c ", " --stdout
- force write to standard output, even if it is the console
+\fB\-c\fR, \fB\-\-stdout\fR
+force write to standard output, even if it is the console
+.
 .TP
-.BR \--[no-]sparse
- enable / disable sparse FS support, to make files with many zeroes smaller on disk.
- Creating sparse files may save disk space and speed up the decompression
-by reducing the amount of disk I/O.
- default : enabled when output is into a file, and disabled when output is stdout.
- This setting overrides default and can force sparse mode over stdout.
+\fB\-\-[no\-]sparse\fR
+enable / disable sparse FS support, to make files with many zeroes smaller on disk\. Creating sparse files may save disk space and speed up decompression by reducing the amount of disk I/O\. default : enabled when output is into a file, and disabled when output is stdout\. This setting overrides default and can force sparse mode over stdout\.
+.
 .TP
-.BR \--rm
- remove source file(s) after successful compression or decompression
+\fB\-\-rm\fR
+remove source file(s) after successful compression or decompression
+.
 .TP
-.BR \-k ", " --keep
- keep source file(s) after successful compression or decompression.
- This is the default behavior.
+\fB\-k\fR, \fB\-\-keep\fR
+keep source file(s) after successful compression or decompression\. This is the default behaviour\.
+.
 .TP
-.BR \-r
- operate recursively on directories
+\fB\-r\fR
+operate recursively on dictionaries
+.
 .TP
-.BR \-h/\-H ", " --help
- display help/long help and exit
+\fB\-h\fR/\fB\-H\fR, \fB\-\-help\fR
+display help/long help and exit
+.
 .TP
-.BR \-V ", " --version
- display Version number and exit
+\fB\-V\fR, \fB\-\-version\fR
+display version number and exit
+.
 .TP
-.BR \-v ", " --verbose
- verbose mode
+\fB\-v\fR
+verbose mode
+.
 .TP
-.BR \-q ", " --quiet
- suppress warnings, interactivity and notifications.
- specify twice to suppress errors too.
+\fB\-q\fR, \fB\-\-quiet\fR
+suppress warnings, interactivity, and notifications\. specify twice to suppress errors too\.
+.
 .TP
-.BR \-C ", " --[no-]check
- add integrity check computed from uncompressed data (default : enabled)
+\fB\-C\fR, \fB\-\-[no\-]check\fR
+add integrety check computed from uncompressed data (default : enabled)
+.
 .TP
-.BR \-t ", " --test
- Test the integrity of compressed files. This option is equivalent to \fB--decompress --stdout > /dev/null\fR.
- No files are created or removed.
+\fB\-\-\fR
+All arguments after \fB\-\-\fR are treated as files
+.
+.SH "DICTIONARY BUILDER"
+\fBzstd\fR offers \fIdictionary\fR compression, useful for very small files and messages\. It\'s possible to train \fBzstd\fR with some samples, the result of which is saved into a file called a \fBdictionary\fR\. Then during compression and decompression, reference the same dictionary\. It will improve compression ratio of small files\. Typical gains range from 10% (at 64KB) to x5 better (at <1KB)\.
+.
 .TP
-.BR --
- All arguments after -- are treated as files
-
-
-.SH DICTIONARY BUILDER
-.PP
-\fBzstd\fR offers \fIdictionary\fR compression, useful for very small files and messages.
-It's possible to train \fBzstd\fR with some samples, the result of which is saved into a file called `dictionary`.
-Then during compression and decompression, make reference to the same dictionary.
-It will improve compression ratio of small files.
-Typical gains range from ~10% (at 64KB) to x5 better (at <1KB).
+\fB\-\-train FILEs\fR
+use FILEs as training set to create a dictionary\. The training set should contain a lot of small files (> 100), and weight typically 100x the target dictionary size (for example, 10 MB for a 100 KB dictionary)\.
+.
 .TP
-.B \--train FILEs
- use FILEs as training set to create a dictionary. The training set should contain a lot of small files (> 100),
-and weight typically 100x the target dictionary size (for example, 10 MB for a 100 KB dictionary)
+\fB\-o file\fR
+dictionary saved into \fBfile\fR (default: dictionary)
+.
 .TP
-.B \-o file
- dictionary saved into `file` (default: dictionary)
+\fB\-\-maxdict #\fR
+limit dictionary to specified size (default : (112640)
+.
 .TP
-.B \--maxdict #
- limit dictionary to specified size (default : 112640)
+\fB\-\-dictID #\fR
+A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary\. By default, zstd will create a 4\-bytes random number ID\. It\'s possible to give a precise number instead\. Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header, and an ID < 65536 will only need 2 bytes\. This compares favorably to 4 bytes default\. However, it\'s up to the dictionary manager to not assign twice the same ID to 2 different dictionaries\.
+.
 .TP
-.B \--dictID #
- A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary.
- By default, zstd will create a 4-bytes random number ID.
- It's possible to give a precise number instead.
- Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header,
- and an ID < 65536 will only need 2 bytes. This compares favorably to 4 bytes default.
- However, it's up to the dictionary manager to not assign twice the same ID to 2 different dictionaries.
+\fB\-s#\fR
+dictionary selectivity level (default: 9) the smaller the value, the denser the dictionary, improving its efficiency but reducing its possible maximum size\.
+.
 .TP
-.B \-s#
- dictionary selectivity level (default: 9)
- the smaller the value, the denser the dictionary, improving its efficiency but reducing its possible maximum size.
-.TP
-.B \--cover=k=#,d=#
- Use alternate dictionary builder algorithm named cover with parameters \fIk\fR and \fId\fR with \fId\fR <= \fIk\fR.
- Selects segments of size \fIk\fR with the highest score to put in the dictionary.
- The score of a segment is computed by the sum of the frequencies of all the subsegments of of size \fId\fR.
- Generally \fId\fR should be in the range [6, 24].
- Good values for \fIk\fR vary widely based on the input data, but a safe range is [32, 2048].
- Example: \fB--train --cover=k=64,d=8 FILEs\fR.
-.TP
-.B \--optimize-cover[=steps=#,k=#,d=#]
- If \fIsteps\fR is not specified, the default value of 32 is used.
- If \fIk\fR is not specified, \fIsteps\fR values in [16, 2048] are checked for each value of \fId\fR.
- If \fId\fR is not specified, the values checked are [6, 8, ..., 16].
-
- Runs the cover dictionary builder for each parameter set saves the optimal parameters and dictionary.
- Prints the optimal parameters and writes the optimal dictionary to the output file.
- Supports multithreading if \fBzstd\fR is compiled with threading support.
-
- The parameter \fIk\fR is more sensitve than \fId\fR, and is faster to optimize over.
- Suggested use is to run with a \fIsteps\fR <= 32 with neither \fIk\fR nor \fId\fR set.
- Once it completes, use the value of \fId\fR it selects with a higher \fIsteps\fR (in the range [256, 1024]).
- \fBzstd --train --optimize-cover FILEs
- \fBzstd --train --optimize-cover=d=d,steps=512 FILEs
-.TP
-
-.SH BENCHMARK
-.TP
-.B \-b#
- benchmark file(s) using compression level #
-.TP
-.B \-e#
- benchmark file(s) using multiple compression levels, from -b# to -e# (included).
-.TP
-.B \-i#
- minimum evaluation time, in seconds (default : 3s), benchmark mode only
-.TP
-.B \-B#
- cut file into independent blocks of size # (default: no block)
-.B \--priority=rt
- set process priority to real-time
-
-.SH ADVANCED COMPRESSION OPTIONS
-.TP
-.B \--zstd[=\fIoptions\fR]
-.PD
-\fBzstd\fR provides 22 predefined compression levels. The selected or default predefined compression level can be changed with advanced compression options.
-The \fIoptions\fR are provided as a comma-separated list. You may specify only the \fIoptions\fR you want to change and the rest will be taken from the selected or default compression level.
-The list of available \fIoptions\fR:
-.RS
-
-.TP
-.BI strategy= strat
-.PD 0
-.TP
-.BI strat= strat
-.PD
-Specify a strategy used by a match finder.
-.IP ""
-There are 8 strategies numbered from 0 to 7, from faster to stronger:
-0=ZSTD_fast, 1=ZSTD_dfast, 2=ZSTD_greedy, 3=ZSTD_lazy, 4=ZSTD_lazy2, 5=ZSTD_btlazy2, 6=ZSTD_btopt, 7=ZSTD_btopt2.
-.IP ""
-
-.TP
-.BI windowLog= wlog
-.PD 0
-.TP
-.BI wlog= wlog
-.PD
-Specify the maximum number of bits for a match distance.
-.IP ""
-The higher number of bits increases the chance to find a match what usually improves compression ratio.
-It also increases memory requirements for compressor and decompressor.
-.IP ""
-The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 25 (32 MiB) for 32-bit compilation and 27 (128 MiB) for 64-bit compilation.
-.IP ""
-
-.TP
-.BI hashLog= hlog
-.PD 0
-.TP
-.BI hlog= hlog
-.PD
-Specify the maximum number of bits for a hash table.
-.IP ""
-The bigger hash table causes less collisions what usually make compression faster but requires more memory during compression.
-.IP ""
-The minimum \fIhlog\fR is 6 (64 B) and the maximum is 25 (32 MiB) for 32-bit compilation and 27 (128 MiB) for 64-bit compilation.
-
-.TP
-.BI chainLog= clog
-.PD 0
-.TP
-.BI clog= clog
-.PD
-Specify the maximum number of bits for a hash chain or a binary tree.
-.IP ""
-The higher number of bits increases the chance to find a match what usually improves compression ratio.
-It also slows down compression speed and increases memory requirements for compression.
-This option is ignored for the ZSTD_fast strategy.
-.IP ""
-The minimum \fIclog\fR is 6 (64 B) and the maximum is 26 (64 MiB) for 32-bit compilation and 28 (256 MiB) for 64-bit compilation.
-.IP ""
-
-.TP
-.BI searchLog= slog
-.PD 0
-.TP
-.BI slog= slog
-.PD
-Specify the maximum number of searches in a hash chain or a binary tree using logarithmic scale.
-.IP ""
-The bigger number of searches increases the chance to find a match what usually improves compression ratio but decreases compression speed.
-.IP ""
-The minimum \fIslog\fR is 1 and the maximum is 24 for 32-bit compilation and 26 for 64-bit compilation.
-.IP ""
-
-.TP
-.BI searchLength= slen
-.PD 0
-.TP
-.BI slen= slen
-.PD
-Specify the minimum searched length of a match in a hash table.
-.IP ""
-The bigger search length usually decreases compression ratio but improves decompression speed.
-.IP ""
-The minimum \fIslen\fR is 3 and the maximum is 7.
-.IP ""
-
-.TP
-.BI targetLength= tlen
-.PD 0
-.TP
-.BI tlen= tlen
-.PD
-Specify the minimum match length that causes a match finder to interrupt searching of better matches.
-.IP ""
-The bigger minimum match length usually improves compression ratio but decreases compression speed.
-This option is used only with ZSTD_btopt and ZSTD_btopt2 strategies.
-.IP ""
-The minimum \fItlen\fR is 4 and the maximum is 999.
-.IP ""
-
-.PP
-.B An example
+\fB\-\-cover=k#,d=#\fR
+Use alternate dictionary builder algorithm named cover with parameters \fIk\fR and \fId\fR with \fId\fR <= \fIk\fR\. Selects segments of size \fIk\fR with the highest score to put in the dictionary\. The score of a segment is computed by the sum of the frequencies of all the subsegments of of size \fId\fR\. Generally \fId\fR should be in the range [6, 24]\. Good values for \fIk\fR vary widely based on the input data, but a safe range is [32, 2048]\.
+.
 .br
-The following parameters sets advanced compression options to predefined level 19 for files bigger than 256 KB:
-.IP ""
-\fB--zstd=\fRwindowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6
-
-.SH BUGS
-Report bugs at:- https://github.com/facebook/zstd/issues
-
-.SH AUTHOR
+Example: \fB\-\-train \-\-cover=k=64,d=8 FILEs\fR\.
+.
+.TP
+\fB\-\-optimize\-cover[=steps=#,k=#,d=#]\fR
+If \fIsteps\fR is not specified, the default value of 32 is used\. If \fIk\fR is not specified, the \fIk\fR values in [16, 2048] are checked for each value of \fId\fR\. If \fId\fR is not specified, the values checked are [6, 8, \.\.\., 16]\.
+.
+.IP
+Runs the cover dictionary builder for each parameter set and saves the optimal parameters and dictionary\. Prints the optimal parameters and writes the optimal dictionary to the output file\. Supports multithreading if \fBzstd\fR is compiled with threading support\.
+.
+.IP
+The parameter \fIk\fR is more sensitive than \fId\fR, and is faster to optimize over\. Suggested use is to run with a \fIsteps\fR <= 32 with neither \fIk\fR nor \fId\fR set\. Once it completes, use the value of \fId\fR it selects with a higher \fIsteps\fR (in the range [256, 1024])\.
+.
+.IP
+\fBzstd \-\-train \-\-optimize\-cover FILEs\fR
+.
+.br
+\fBzstd \-\-train \-\-optimize\-cover=d=d,steps=512 FILEs\fR
+.
+.SH "BENCHMARK"
+.
+.TP
+\fB\-b#\fR
+benchmark file(s) using compression level #
+.
+.TP
+\fB\-e#\fR
+benchmark file(s) using multiple compression levels, from \fB\-b#\fR to \fB\-e#\fR (inclusive)
+.
+.TP
+\fB\-i#\fR
+minimum evaluation time, in seconds (default : 3s), benchmark mode only
+.
+.TP
+\fB\-B#\fR
+cut file into independent blocks of size # (default: no block)
+.
+.TP
+\fB\-\-priority=rt\fR
+set process priority to real\-time
+.
+.SH "ADVANCED COMPRESSION OPTIONS"
+.
+.SS "\-\-zstd[=options]:"
+\fBzstd\fR provides 22 predefined compression levels\. The selected or default predefined compression level can be changed with advanced compression options\. The \fIoptions\fR are provided as a comma\-separated list\. You may specify only the options you want to change and the rest will be taken from the selected or default compression level\. The list of available \fIoptions\fR:
+.
+.TP
+\fBstrategy\fR=\fIstrat\fR, \fBstrat\fR=\fIstrat\fR
+Specify a strategy used by a match finder\.
+.
+.IP
+There are 8 strategies numbered from 0 to 7, from faster to stronger: 0=ZSTD_fast, 1=ZSTD_dfast, 2=ZSTD_greedy, 3=ZSTD_lazy, 4=ZSTD_lazy2, 5=ZSTD_btlazy2, 6=ZSTD_btopt, 7=ZSTD_btopt2\.
+.
+.TP
+\fBwindowLog\fR=\fIwlog\fR, \fBwlog\fR=\fIwlog\fR
+Specify the maximum number of bits for a match distance\.
+.
+.IP
+The higher number of increases the chance to find a match which usually improves compression ratio\. It also increases memory requirements for the compressor and decompressor\. The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 27 (128 MiB)\.
+.
+.TP
+\fBhashLog\fR=\fIhlog\fR, \fBhlog\fR=\fIhlog\fR
+Specify the maximum number of bits for a hash table\.
+.
+.IP
+Bigger hash tables cause less collisions which usually makes compression faster, but requires more memory during compression\.
+.
+.IP
+The minimum \fIhlog\fR is 6 (64 B) and the maximum is 26 (128 MiB)\.
+.
+.TP
+\fBchainLog\fR=\fIclog\fR, \fBclog\fR=\fIclog\fR
+Specify the maximum number of bits for a hash chain or a binary tree\.
+.
+.IP
+Higher numbers of bits increases the chance to find a match which usually improves compression ratio\. It also slows down compression speed and increases memory requirements for compression\. This option is ignored for the ZSTD_fast strategy\.
+.
+.IP
+The minimum \fIclog\fR is 6 (64 B) and the maximum is 28 (256 MiB)\.
+.
+.TP
+\fBsearchLog\fR=\fIslog\fR, \fBslog\fR=\fIslog\fR
+Specify the maximum number of searches in a hash chain or a binary tree using logarithmic scale\.
+.
+.IP
+More searches increases the chance to find a match which usually increases compression ratio but decreases compression speed\.
+.
+.IP
+The minimum \fIslog\fR is 1 and the maximum is 26\.
+.
+.TP
+\fBsearchLength\fR=\fIslen\fR, \fBslen\fR=\fIslen\fR
+Specify the minimum searched length of a match in a hash table\.
+.
+.IP
+Larger search lengths usually decrease compression ratio but improve decompression speed\.
+.
+.IP
+The minimum \fIslen\fR is 3 and the maximum is 7\.
+.
+.TP
+\fBtargetLen\fR=\fItlen\fR, \fBtlen\fR=\fItlen\fR
+Specify the minimum match length that causes a match finder to stop searching for better matches\.
+.
+.IP
+A larger minimum match length usually improves compression ratio but decreases compression speed\. This option is only used with strategies ZSTD_btopt and ZSTD_btopt2\.
+.
+.IP
+The minimum \fItlen\fR is 4 and the maximum is 999\.
+.
+.SS "Example"
+The following parameters sets advanced compression options to those of predefined level 19 for files bigger than 256 KB:
+.
+.P
+\fB\-\-zstd\fR=windowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6
+.
+.SH "BUGS"
+Report bugs at: https://github\.com/facebook/zstd/issues
+.
+.SH "AUTHOR"
 Yann Collet
--- a/programs/zstd.1.md
+++ b/programs/zstd.1.md
@ -0,0 +1,305 @@
+zstd(1) -- zstd, unzstd, zstdcat - Compress or decompress .zst files
+====================================================================
+
+SYNOPSIS
+--------
+
+`zstd` [*OPTIONS*] [-|&lt;INPUT-FILE&gt;] [-o &lt;OUTPUT-FILE&gt;]
+
+`unzstd` is equivalent to `zstd -d`
+
+`zstdcat` is equivalent to `zstd -dcf`
+
+
+DESCRIPTION
+-----------
+`zstd` is a fast lossless compression algorithm and data compression tool,
+with command line syntax similar to `gzip (1)` and `xz (1)`.
+It is based on the **LZ77** family, with further FSE & huff0 entropy stages.
+`zstd` offers highly configurable compression speed,
+with fast modes at > 200 MB/s per code,
+and strong modes nearing lzma compression ratios.
+It also features a very fast decoder, with speeds > 500 MB/s per core.
+
+`zstd` command line syntax is generally similar to gzip,
+but features the following differences :
+
+  - Source files are preserved by default.
+    It's possible to remove them automatically by using the `--rm` command.
+  - When compressing a single file, `zstd` displays progress notifications
+    and result summary by default.
+    Use `-q` to turn them off.
+  - `zstd` does not accept input from console,
+    but it properly accepts `stdin` when it's not the console.
+  - `zstd` displays a short help page when command line is an error.
+    Use `-q` to turn it off.
+
+`zstd` compresses or decompresses each _file_ according to the selected
+operation mode.
+If no _files_ are given or _file_ is `-`, `zstd` reads from standard input
+and writes the processed data to standard output.
+`zstd` will refuse to write compressed data to standard output
+if it is a terminal : it will display an error message and skip the _file_.
+Similarly, `zstd` will refuse to read compressed data from standard input
+if it is a terminal.
+
+Unless `--stdout` or `-o` is specified, _files_ are written to a new file
+whose name is derived from the source _file_ name:
+
+* When compressing, the suffix `.zst` is appended to the source filename to
+  get the target filename.
+* When decompressing, the `.zst` suffix is removed from the source filename to
+  get the target filename
+
+### Concatenation with .zst files
+It is possible to concatenate `.zst` files as is.
+`zstd` will decompress such files as if they were a single `.zst` file.
+
+OPTIONS
+-------
+
+### Integer suffixes and special values
+In most places where an integer argument is expected,
+an optional suffix is supported to easily indicate large integers.
+There must be no space between the integer and the suffix.
+
+* `KiB`:
+    Multiply the integer by 1,024 (2\^10).
+    `Ki`, `K`, and `KB` are accepted as synonyms for `KiB`.
+* `MiB`:
+    Multiply the integer by 1,048,576 (2\^20).
+    `Mi`, `M`, and `MB` are accepted as synonyms for `MiB`.
+
+### Operation mode
+If multiple operation mode options are given,
+the last one takes effect.
+
+* `-z`, `--compress`:
+    Compress.
+    This is the default operation mode when no operation mode option is specified
+    and no other operation mode is implied from the command name
+    (for example, `unzstd` implies `--decompress`).
+* `-d`, `--decompress`, `--uncompress`:
+    Decompress.
+* `-t`, `--test`:
+    Test the integrity of compressed _files_.
+    This option is equivalent to `--decompress --stdout` except that the
+    decompressed data is discarded instead of being written to standard output.
+    No files are created or removed.
+* `-b#`:
+    Benchmark file(s) using compression level #
+* `--train FILEs`:
+    Use FILEs as a training set to create a dictionary.
+    The training set should contain a lot of small files (> 100).
+
+### Operation modifiers
+
+* `-#`:
+    `#` compression level \[1-19] (default: 3)
+* `--ultra`:
+    unlocks high compression levels 20+ (maximum 22), using a lot more memory.
+    Note that decompression will also require more memory when using these levels.
+* `-D file`:
+    use `file` as Dictionary to compress or decompress FILE(s)
+* `--nodictID`:
+    do not store dictionary ID within frame header (dictionary compression).
+    The decoder will have to rely on implicit knowledge about which dictionary to use,
+    it won't be able to check if it's correct.
+* `-o file`:
+    save result into `file` (only possible with a single INPUT-FILE)
+* `-f`, `--force`:
+    overwrite output without prompting
+* `-c`, `--stdout`:
+    force write to standard output, even if it is the console
+* `--[no-]sparse`:
+    enable / disable sparse FS support,
+    to make files with many zeroes smaller on disk.
+    Creating sparse files may save disk space and speed up decompression by
+    reducing the amount of disk I/O.
+    default : enabled when output is into a file,
+    and disabled when output is stdout.
+    This setting overrides default and can force sparse mode over stdout.
+* `--rm`:
+    remove source file(s) after successful compression or decompression
+* `-k`, `--keep`:
+    keep source file(s) after successful compression or decompression.
+    This is the default behaviour.
+* `-r`:
+    operate recursively on dictionaries
+* `-h`/`-H`, `--help`:
+    display help/long help and exit
+* `-V`, `--version`:
+    display version number and exit
+* `-v`:
+    verbose mode
+* `-q`, `--quiet`:
+    suppress warnings, interactivity, and notifications.
+    specify twice to suppress errors too.
+* `-C`, `--[no-]check`:
+    add integrety check computed from uncompressed data (default : enabled)
+* `--`:
+    All arguments after `--` are treated as files
+
+DICTIONARY BUILDER
+------------------
+`zstd` offers _dictionary_ compression,
+useful for very small files and messages.
+It's possible to train `zstd` with some samples,
+the result of which is saved into a file called a `dictionary`.
+Then during compression and decompression, reference the same dictionary.
+It will improve compression ratio of small files.
+Typical gains range from 10% (at 64KB) to x5 better (at <1KB).
+
+* `--train FILEs`:
+    use FILEs as training set to create a dictionary.
+    The training set should contain a lot of small files (> 100),
+    and weight typically 100x the target dictionary size
+    (for example, 10 MB for a 100 KB dictionary).
+* `-o file`:
+    dictionary saved into `file` (default: dictionary)
+* `--maxdict #`:
+    limit dictionary to specified size (default : (112640)
+* `--dictID #`:
+    A dictionary ID is a locally unique ID that a decoder can use to verify it is
+    using the right dictionary.
+    By default, zstd will create a 4-bytes random number ID.
+    It's possible to give a precise number instead.
+    Short numbers have an advantage : an ID < 256 will only need 1 byte in the
+    compressed frame header, and an ID < 65536 will only need 2 bytes.
+    This compares favorably to 4 bytes default.
+    However, it's up to the dictionary manager to not assign twice the same ID to
+    2 different dictionaries.
+* `-s#`:
+    dictionary selectivity level (default: 9)
+    the smaller the value, the denser the dictionary,
+    improving its efficiency but reducing its possible maximum size.
+* `--cover=k#,d=#`:
+    Use alternate dictionary builder algorithm named cover with parameters
+    _k_ and _d_ with _d_ <= _k_.
+    Selects segments of size _k_ with the highest score to put in the dictionary.
+    The score of a segment is computed by the sum of the frequencies of all the
+    subsegments of of size _d_.
+    Generally _d_ should be in the range [6, 24].
+    Good values for _k_ vary widely based on the input data,
+    but a safe range is [32, 2048].<br />
+    Example: `--train --cover=k=64,d=8 FILEs`.
+
+* `--optimize-cover[=steps=#,k=#,d=#]`:
+    If _steps_ is not specified, the default value of 32 is used.
+    If _k_ is not specified, the _k_ values in [16, 2048] are checked for each
+    value of _d_.
+    If _d_ is not specified, the values checked are [6, 8, ..., 16].
+
+    Runs the cover dictionary builder for each parameter set and saves the
+    optimal parameters and dictionary.
+    Prints the optimal parameters and writes the optimal dictionary to the output file.
+    Supports multithreading if `zstd` is compiled with threading support.
+
+    The parameter _k_ is more sensitive than _d_, and is faster to optimize over.
+    Suggested use is to run with a _steps_ <= 32 with neither _k_ nor _d_ set.
+    Once it completes, use the value of _d_ it selects with a higher _steps_
+    (in the range [256, 1024]).
+
+    `zstd --train --optimize-cover FILEs` <br />
+    `zstd --train --optimize-cover=d=d,steps=512 FILEs`
+
+
+BENCHMARK
+---------
+
+* `-b#`:
+    benchmark file(s) using compression level #
+* `-e#`:
+    benchmark file(s) using multiple compression levels, from `-b#` to `-e#` (inclusive)
+* `-i#`:
+    minimum evaluation time, in seconds (default : 3s), benchmark mode only
+* `-B#`:
+    cut file into independent blocks of size # (default: no block)
+* `--priority=rt`:
+    set process priority to real-time
+
+ADVANCED COMPRESSION OPTIONS
+----------------------------
+### --zstd[=options]:
+`zstd` provides 22 predefined compression levels.
+The selected or default predefined compression level can be changed with
+advanced compression options.
+The _options_ are provided as a comma-separated list.
+You may specify only the options you want to change and the rest will be
+taken from the selected or default compression level.
+The list of available _options_:
+
+- `strategy`=_strat_, `strat`=_strat_:
+    Specify a strategy used by a match finder.
+
+    There are 8 strategies numbered from 0 to 7, from faster to stronger:
+    0=ZSTD\_fast, 1=ZSTD\_dfast, 2=ZSTD\_greedy, 3=ZSTD\_lazy,
+    4=ZSTD\_lazy2, 5=ZSTD\_btlazy2, 6=ZSTD\_btopt, 7=ZSTD\_btopt2.
+
+- `windowLog`=_wlog_, `wlog`=_wlog_:
+    Specify the maximum number of bits for a match distance.
+
+    The higher number of increases the chance to find a match which usually
+    improves compression ratio.
+    It also increases memory requirements for the compressor and decompressor.
+    The minimum _wlog_ is 10 (1 KiB) and the maximum is 27 (128 MiB).
+
+- `hashLog`=_hlog_, `hlog`=_hlog_:
+    Specify the maximum number of bits for a hash table.
+
+    Bigger hash tables cause less collisions which usually makes compression
+    faster, but requires more memory during compression.
+
+    The minimum _hlog_ is 6 (64 B) and the maximum is 26 (128 MiB).
+
+- `chainLog`=_clog_, `clog`=_clog_:
+    Specify the maximum number of bits for a hash chain or a binary tree.
+
+    Higher numbers of bits increases the chance to find a match which usually
+    improves compression ratio.
+    It also slows down compression speed and increases memory requirements for
+    compression.
+    This option is ignored for the ZSTD_fast strategy.
+
+    The minimum _clog_ is 6 (64 B) and the maximum is 28 (256 MiB).
+
+- `searchLog`=_slog_, `slog`=_slog_:
+    Specify the maximum number of searches in a hash chain or a binary tree
+    using logarithmic scale.
+
+    More searches increases the chance to find a match which usually increases
+    compression ratio but decreases compression speed.
+
+    The minimum _slog_ is 1 and the maximum is 26.
+
+- `searchLength`=_slen_, `slen`=_slen_:
+    Specify the minimum searched length of a match in a hash table.
+
+    Larger search lengths usually decrease compression ratio but improve
+    decompression speed.
+
+    The minimum _slen_ is 3 and the maximum is 7.
+
+- `targetLen`=_tlen_, `tlen`=_tlen_:
+    Specify the minimum match length that causes a match finder to stop
+    searching for better matches.
+
+    A larger minimum match length usually improves compression ratio but
+    decreases compression speed.
+    This option is only used with strategies ZSTD_btopt and ZSTD_btopt2.
+
+    The minimum _tlen_ is 4 and the maximum is 999.
+
+### Example
+The following parameters sets advanced compression options to those of
+predefined level 19 for files bigger than 256 KB:
+
+`--zstd`=windowLog=23,chainLog=23,hashLog=22,searchLog=6,searchLength=3,targetLength=48,strategy=6
+
+BUGS
+----
+Report bugs at: https://github.com/facebook/zstd/issues
+
+AUTHOR
+------
+Yann Collet
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@ -49,13 +49,13 @@
 #define AUTHOR "Yann Collet"
 #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR

-#define GZ_EXTENSION ".gz"
-#define ZSTD_EXTENSION ".zst"
 #define ZSTD_UNZSTD "unzstd"
 #define ZSTD_CAT "zstdcat"
 #define ZSTD_GZ "gzip"
 #define ZSTD_GUNZIP "gunzip"
 #define ZSTD_GZCAT "gzcat"
+#define ZSTD_LZMA "lzma"
+#define ZSTD_XZ "xz"

 #define KB *(1 <<10)
 #define MB *(1 <<20)
@ -130,6 +130,10 @@ static int usage_advanced(const char* programName)
 #ifdef ZSTD_GZCOMPRESS
    DISPLAY( "--format=gzip : compress files to the .gz format \n");
 #endif
+#ifdef ZSTD_LZMACOMPRESS
+    DISPLAY( "--format=xz : compress files to the .xz format \n");
+    DISPLAY( "--format=lzma : compress files to the .lzma format \n");
+#endif
 #endif
 #ifndef ZSTD_NODECOMPRESS
    DISPLAY( "--test  : test compressed file integrity \n");
@ -163,7 +167,7 @@ static int usage_advanced(const char* programName)
 static int badusage(const char* programName)
 {
    DISPLAYLEVEL(1, "Incorrect parameters\n");
-    if (displayLevel >= 1) usage(programName);
+    if (displayLevel >= 2) usage(programName);
    return 1;
 }

@ -325,6 +329,8 @@ int main(int argCount, const char* argv[])
    if (!strcmp(programName, ZSTD_GZ)) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); FIO_setRemoveSrcFile(1); }    /* behave like gzip */
    if (!strcmp(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(1); }                                          /* behave like gunzip */
    if (!strcmp(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; displayLevel=1; }  /* behave like gzcat */
+    if (!strcmp(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); }    /* behave like lzma */
+    if (!strcmp(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); }    /* behave like xz */
    memset(&compressionParams, 0, sizeof(compressionParams));

    /* command switches */
@ -370,10 +376,16 @@ int main(int argCount, const char* argv[])
                    if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
                    if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
                    if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
+#ifdef ZSTD_GZCOMPRESS
                    if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; }
+#endif
+#ifdef ZSTD_LZMACOMPRESS
+                    if (!strcmp(argument, "--format=lzma")) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression);  continue; }
+                    if (!strcmp(argument, "--format=xz")) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression);  continue; }
+#endif

                    /* long commands with arguments */
-#ifndef  ZSTD_NODICT
+#ifndef ZSTD_NODICT
                    if (longCommandWArg(&argument, "--cover=")) {
                      cover=1; if (!parseCoverParameters(argument, &coverParams)) CLEAN_RETURN(badusage(programName));
                      continue;
@ -623,7 +635,7 @@ int main(int argCount, const char* argv[])

    /* Check if input/output defined as console; trigger an error in this case */
    if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) CLEAN_RETURN(badusage(programName));
-    if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && strcmp(filenameTable[0], stdinmark) && !(forceStdout && (operation==zom_decompress)))
+    if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && !strcmp(filenameTable[0], stdinmark) && !forceStdout && operation!=zom_decompress)
        CLEAN_RETURN(badusage(programName));

    /* user-selected output filename, only possible with a single file */
--- a/tests/Makefile
+++ b/tests/Makefile
@ -28,8 +28,9 @@ TESTARTEFACT := versionsTest namespaceTest

 CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
 CFLAGS  ?= -O3
-CFLAGS  += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
-           -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
+CFLAGS  += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+           -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
+           -Wstrict-prototypes -Wundef -Wformat-security
 CFLAGS  += $(MOREFLAGS)
 FLAGS    = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)

@ -59,10 +60,10 @@ endif
 MULTITHREAD = $(MULTITHREAD_CPP) $(MULTITHREAD_LD)

 VOID = /dev/null
-ZSTREAM_TESTTIME = -T2mn
+ZSTREAM_TESTTIME ?= -T2mn
 FUZZERTEST ?= -T5mn
 ZSTDRTTEST = --test-large-data
-DECODECORPUS_TESTTIME = -T30
+DECODECORPUS_TESTTIME ?= -T30

 .PHONY: default all all32 dll clean test test32 test-all namespaceTest versionsTest

@ -74,8 +75,6 @@ all32: fullbench32 fuzzer32 zstreamtest32 zbufftest32

 dll: fuzzer-dll zstreamtest-dll zbufftest-dll

-
-
 zstd:
 	$(MAKE) -C $(PRGDIR) $@

@ -168,7 +167,7 @@ longmatch  : $(ZSTD_FILES) longmatch.c
 invalidDictionaries  : $(ZSTD_FILES) invalidDictionaries.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)

-legacy : CFLAGS+= -DZSTD_LEGACY_SUPPORT=1
+legacy : CFLAGS+= -DZSTD_LEGACY_SUPPORT=4
 legacy : CPPFLAGS+= -I$(ZSTDDIR)/legacy
 legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
@ -253,6 +252,11 @@ zstd-playTests: datagen
 	file $(ZSTD)
 	ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)

+shortest: ZSTDRTTEST=
+shortest: test-zstd
+
+fuzztest: test-fuzzer test-zstream test-decodecorpus
+
 test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus
 ifeq ($(QEMU_SYS),)
 test: test-pool
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@ -66,6 +66,7 @@ static clock_t g_displayClock = 0;
 *  Fuzzer functions
 *********************************************************/
 #define MIN(a,b) ((a)<(b)?(a):(b))
+#define MAX(a,b) ((a)>(b)?(a):(b))

 static clock_t FUZ_clockSpan(clock_t cStart)
 {
@ -108,6 +109,7 @@ static int basicUnitTests(U32 seed, double compressibility)
    void* const CNBuffer = malloc(CNBuffSize);
    void* const compressedBuffer = malloc(ZSTD_compressBound(CNBuffSize));
    void* const decodedBuffer = malloc(CNBuffSize);
+    ZSTD_DCtx* dctx = ZSTD_createDCtx();
    int testResult = 0;
    U32 testNb=0;
    size_t cSize;
@ -155,6 +157,16 @@ static int basicUnitTests(U32 seed, double compressibility)
    }   }
    DISPLAYLEVEL(4, "OK \n");

+    DISPLAYLEVEL(4, "test%3i : decompress with null dict : ", testNb++);
+    { size_t const r = ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL, 0);
+      if (r != CNBuffSize) goto _output_error; }
+    DISPLAYLEVEL(4, "OK \n");
+
+    DISPLAYLEVEL(4, "test%3i : decompress with null DDict : ", testNb++);
+    { size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL);
+      if (r != CNBuffSize) goto _output_error; }
+    DISPLAYLEVEL(4, "OK \n");
+
    DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++);
    { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
      if (!ZSTD_isError(r)) goto _output_error;
@ -210,7 +222,6 @@ static int basicUnitTests(U32 seed, double compressibility)
    /* Dictionary and CCtx Duplication tests */
    {   ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx();
        ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();
-        ZSTD_DCtx* const dctx = ZSTD_createDCtx();
        static const size_t dictSize = 551;

        DISPLAYLEVEL(4, "test%3i : copy context too soon : ", testNb++);
@ -283,12 +294,10 @@ static int basicUnitTests(U32 seed, double compressibility)

        ZSTD_freeCCtx(ctxOrig);
        ZSTD_freeCCtx(ctxDuplicated);
-        ZSTD_freeDCtx(dctx);
    }

    /* Dictionary and dictBuilder tests */
    {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();
-        ZSTD_DCtx* const dctx = ZSTD_createDCtx();
        size_t dictSize = 16 KB;
        void* dictBuffer = malloc(dictSize);
        size_t const totalSampleSize = 1 MB;
@ -370,14 +379,12 @@ static int basicUnitTests(U32 seed, double compressibility)
        DISPLAYLEVEL(4, "OK \n");

        ZSTD_freeCCtx(cctx);
-        ZSTD_freeDCtx(dctx);
        free(dictBuffer);
        free(samplesSizes);
    }

    /* COVER dictionary builder tests */
    {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();
-        ZSTD_DCtx* const dctx = ZSTD_createDCtx();
        size_t dictSize = 16 KB;
        size_t optDictSize = dictSize;
        void* dictBuffer = malloc(dictSize);
@ -414,7 +421,7 @@ static int basicUnitTests(U32 seed, double compressibility)
        memset(&params, 0, sizeof(params));
        params.steps = 4;
        optDictSize = COVER_optimizeTrainFromBuffer(dictBuffer, optDictSize,
-                                                    CNBuffer, samplesSizes, nbSamples,
+                                                    CNBuffer, samplesSizes, nbSamples / 4,
                                                    &params);
        if (ZDICT_isError(optDictSize)) goto _output_error;
        DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)optDictSize);
@ -425,7 +432,6 @@ static int basicUnitTests(U32 seed, double compressibility)
        DISPLAYLEVEL(4, "OK : %u \n", dictID);

        ZSTD_freeCCtx(cctx);
-        ZSTD_freeDCtx(dctx);
        free(dictBuffer);
        free(samplesSizes);
    }
@ -445,7 +451,6 @@ static int basicUnitTests(U32 seed, double compressibility)

    /* block API tests */
    {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();
-        ZSTD_DCtx* const dctx = ZSTD_createDCtx();
        static const size_t dictSize = 65 KB;
        static const size_t blockSize = 100 KB;   /* won't cause pb with small dict size */
        size_t cSize2;
@ -795,11 +800,12 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
        /*=====   Streaming compression test, scattered segments and dictionary   =====*/

        {   U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
-            int const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
+            U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
+            int const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (MAX(testLog, dictLog)/3))) + 1;
            maxTestSize = FUZ_rLogLength(&lseed, testLog);
            if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;

-            dictSize = FUZ_randomLength(&lseed, maxSampleLog);   /* needed also for decompression */
+            dictSize = FUZ_rLogLength(&lseed, dictLog);   /* needed also for decompression */
            dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize));

            if (FUZ_rand(&lseed) & 0xF) {
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@ -20,6 +20,12 @@ roundTripTest() {
    $DIFF -q tmp1 tmp2
 }

+isTerminal=false
+if [ -t 0 ] && [ -t 1 ]
+then
+    isTerminal=true
+fi
+
 isWindows=false
 ECHO="echo"
 INTOVOID="/dev/null"
@ -27,6 +33,7 @@ case "$OS" in
  Windows*)
    isWindows=true
    ECHO="echo -e"
+    INTOVOID="NUL"
    ;;
 esac

@ -72,6 +79,12 @@ cp tmp tmp2
 $ZSTD tmp2 -fo && die "-o must be followed by filename "
 $ECHO "test : implied stdout when input is stdin"
 $ECHO bob | $ZSTD | $ZSTD -d
+if [ "$isTerminal" = true ]; then
+$ECHO "test : compressed data to terminal"
+$ECHO bob | $ZSTD && die "should have refused : compressed data to terminal"
+$ECHO "test : compressed data from terminal (a hang here is a test fail, zstd is wrongly waiting on data from terminal)"
+$ZSTD -d > $INTOVOID && die "should have refused : compressed data from terminal"
+fi
 $ECHO "test : null-length file roundtrip"
 $ECHO -n '' | $ZSTD - --stdout | $ZSTD -d --stdout
 $ECHO "test : decompress file with wrong suffix (must fail)"
@ -240,7 +253,7 @@ $ZSTD -f tmp -D tmpDict1 --no-dictID
 $ZSTD -d tmp.zst -D tmpDict -fo result
 $DIFF $TESTFILE result
 $ECHO "- Compress with wrong argument order (must fail)"
-$ZSTD tmp -Df tmpDict1 -c > /dev/null && die "-D must be followed by dictionary name "
+$ZSTD tmp -Df tmpDict1 -c > $INTOVOID && die "-D must be followed by dictionary name "
 $ECHO "- Compress multiple files with dictionary"
 rm -rf dirTestDict
 mkdir dirTestDict
@ -304,11 +317,91 @@ $ECHO "\n**** benchmark mode tests **** "

 $ECHO "bench one file"
 ./datagen > tmp1
-$ZSTD -bi1 tmp1
+$ZSTD -bi0 tmp1
 $ECHO "bench multiple levels"
-$ZSTD -i1b1e3 tmp1
+$ZSTD -i0b0e3 tmp1
 $ECHO "with recursive and quiet modes"
-$ZSTD -rqi1b1e3 tmp1
+$ZSTD -rqi1b1e2 tmp1
+
+
+$ECHO "\n**** gzip compatibility tests **** "
+
+GZIPMODE=1
+$ZSTD --format=gzip -V || GZIPMODE=0
+if [ $GZIPMODE -eq 1 ]; then
+    $ECHO "gzip support detected"
+    GZIPEXE=1
+    gzip -V || GZIPEXE=0
+    if [ $GZIPEXE -eq 1 ]; then
+        ./datagen > tmp
+        $ZSTD --format=gzip -f tmp
+        gzip -t -v tmp.gz
+        gzip -f tmp
+        $ZSTD -d -f -v tmp.gz
+        rm tmp*
+    else
+        $ECHO "gzip binary not detected"
+    fi
+else
+    $ECHO "gzip mode not supported"
+fi
+
+
+$ECHO "\n**** gzip frame tests **** "
+
+if [ $GZIPMODE -eq 1 ]; then
+    ./datagen > tmp
+    $ZSTD -f --format=gzip tmp
+    $ZSTD -f tmp
+    cat tmp.gz tmp.zst tmp.gz tmp.zst | $ZSTD -d -f -o tmp
+    head -c -1 tmp.gz | $ZSTD -t && die "incomplete frame not detected !"
+    rm tmp*
+else
+    $ECHO "gzip mode not supported"
+fi
+
+
+$ECHO "\n**** xz compatibility tests **** "
+
+LZMAMODE=1
+$ZSTD --format=xz -V || LZMAMODE=0
+if [ $LZMAMODE -eq 1 ]; then
+    $ECHO "xz support detected"
+    XZEXE=1
+    xz -V && lzma -V || XZEXE=0
+    if [ $XZEXE -eq 1 ]; then
+        ./datagen > tmp
+        $ZSTD --format=lzma -f tmp
+        $ZSTD --format=xz -f tmp
+        xz -t -v tmp.xz
+        xz -t -v tmp.lzma
+        xz -f -k tmp
+        lzma -f -k --lzma1 tmp
+        $ZSTD -d -f -v tmp.xz
+        $ZSTD -d -f -v tmp.lzma
+        rm tmp*
+    else
+        $ECHO "xz binary not detected"
+    fi
+else
+    $ECHO "xz mode not supported"
+fi
+
+
+$ECHO "\n**** xz frame tests **** "
+
+if [ $LZMAMODE -eq 1 ]; then
+    ./datagen > tmp
+    $ZSTD -f --format=xz tmp
+    $ZSTD -f --format=lzma tmp
+    $ZSTD -f tmp
+    cat tmp.xz tmp.lzma tmp.zst tmp.lzma tmp.xz tmp.zst | $ZSTD -d -f -o tmp
+    head -c -1 tmp.xz | $ZSTD -t && die "incomplete frame not detected !"
+    head -c -1 tmp.lzma | $ZSTD -t && die "incomplete frame not detected !"
+    rm tmp*
+else
+    $ECHO "xz mode not supported"
+fi


 $ECHO "\n**** zstd round-trip tests **** "
@ -317,10 +410,10 @@ roundTripTest
 roundTripTest -g15K       # TableID==3
 roundTripTest -g127K      # TableID==2
 roundTripTest -g255K      # TableID==1
-roundTripTest -g513K      # TableID==0
-roundTripTest -g512K 6    # greedy, hash chain
-roundTripTest -g512K 16   # btlazy2
-roundTripTest -g512K 19   # btopt
+roundTripTest -g522K      # TableID==0
+roundTripTest -g519K 6    # greedy, hash chain
+roundTripTest -g517K 16   # btlazy2
+roundTripTest -g516K 19   # btopt

 rm tmp*

--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@ -218,7 +218,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
    outBuff.pos = 0;
    { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
      if (r != 0) goto _output_error; }
-    if (outBuff.pos != 0) goto _output_error;   /* skippable frame len is 0 */
+    if (outBuff.pos != 0) goto _output_error;   /* skippable frame output len is 0 */
    DISPLAYLEVEL(3, "OK \n");

    /* Basic decompression test */
@ -645,11 +645,12 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
            }
        } else {
            U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
-            U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
+            U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
+            U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (MAX(testLog, dictLog)/3))) + 1;
            maxTestSize = FUZ_rLogLength(&lseed, testLog);
            oldTestLog = testLog;
            /* random dictionary selection */
-            dictSize  = ((FUZ_rand(&lseed)&63)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
+            dictSize  = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
            {   size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
                dict = srcBuffer + dictStart;
            }
@ -886,11 +887,12 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
            }
        } else {
            U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
-            U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
+            U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
+            U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (MAX(testLog, dictLog)/3))) + 1;
            maxTestSize = FUZ_rLogLength(&lseed, testLog);
            oldTestLog = testLog;
            /* random dictionary selection */
-            dictSize  = ((FUZ_rand(&lseed)&63)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
+            dictSize  = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
            {   size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
                dict = srcBuffer + dictStart;
            }