From 60f10aab6c26dda2224583de84fe64ab7a649f55 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 15 Dec 2016 11:32:31 +0100 Subject: [PATCH 01/28] introduced ZSTDLIB_VISIBILITY --- .travis.yml | 2 +- lib/zstd.h | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 148a98f1..c2817c48 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ matrix: # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - - env: Ubu=12.04cont Cmd="make test && make clean && make travis-install" + - env: Ubu=12.04cont Cmd="make test && make clean && make travis-install && make -C tests fullbench-dll fullbench-lib" os: linux sudo: false diff --git a/lib/zstd.h b/lib/zstd.h index b02e16b4..976d41e1 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -18,15 +18,19 @@ extern "C" { #include /* size_t */ +/* ===== ZSTDLIB_API : control library symbols visibility ===== */ /* ===== ZSTDLIB_API : control library symbols visibility ===== */ #if defined(__GNUC__) && (__GNUC__ >= 4) -# define ZSTDLIB_API __attribute__ ((visibility ("default"))) -#elif defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) -# define ZSTDLIB_API __declspec(dllexport) -#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) -# define ZSTDLIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +# define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default"))) #else -# define ZSTDLIB_API +# define ZSTDLIB_VISIBILITY +#endif +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDLIB_API ZSTDLIB_VISIBILITY #endif From 4e10bd339ddaa451aae5de33491251ebbd36a180 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 15 Dec 2016 12:09:23 +0100 Subject: [PATCH 02/28] appveyor.yml: added tests of fullbench-dll fullbench-lib --- .travis.yml | 2 +- appveyor.yml | 8 ++++---- lib/zstd.h | 1 - 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index c2817c48..148a98f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ matrix: # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - - env: Ubu=12.04cont Cmd="make test && make clean && make travis-install && make -C tests fullbench-dll fullbench-lib" + - env: Ubu=12.04cont Cmd="make test && make clean && make travis-install" os: linux sudo: false diff --git a/appveyor.yml b/appveyor.yml index 27c83c04..618f76d3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,10 +2,10 @@ version: 1.0.{build} environment: matrix: - COMPILER: "gcc" - MAKE_PARAMS: "test" + MAKE_PARAMS: "make test && make lib && make -C tests fullbench-dll fullbench-lib" PLATFORM: "mingw64" - COMPILER: "gcc" - MAKE_PARAMS: "test" + MAKE_PARAMS: "make test" PLATFORM: "mingw32" - COMPILER: "visual" CONFIGURATION: "Debug" @@ -62,8 +62,8 @@ build_script: ECHO *** && make -v && cc -v && - ECHO make %MAKE_PARAMS% && - make %MAKE_PARAMS% + ECHO %MAKE_PARAMS% && + sh -c %MAKE_PARAMS% ) - if [%COMPILER%]==[gcc] if [%PLATFORM%]==[mingw64] ( COPY programs\zstd.exe bin\zstd.exe && diff --git a/lib/zstd.h b/lib/zstd.h index 976d41e1..07283f55 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -18,7 +18,6 @@ extern "C" { #include /* size_t */ -/* ===== ZSTDLIB_API : control library symbols visibility ===== */ /* ===== ZSTDLIB_API : control library symbols visibility ===== */ #if defined(__GNUC__) && (__GNUC__ >= 4) # define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default"))) From 0a1caeef6dcf5b50aca2891424dfd81955e69908 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 15 Dec 2016 12:12:46 +0100 Subject: [PATCH 03/28] VS: fixed 32-bit DLL compilation --- build/VS2010/fullbench-dll/fullbench-dll.vcxproj | 2 ++ lib/dll/example/fullbench-dll.vcxproj | 2 ++ 2 files changed, 4 insertions(+) diff --git a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj index 3609f3ac..e697318e 100644 --- a/build/VS2010/fullbench-dll/fullbench-dll.vcxproj +++ b/build/VS2010/fullbench-dll/fullbench-dll.vcxproj @@ -99,6 +99,7 @@ true $(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories) libzstd.lib;%(AdditionalDependencies) + false @@ -138,6 +139,7 @@ true $(SolutionDir)bin\$(Platform)_$(Configuration);%(AdditionalLibraryDirectories) libzstd.lib;%(AdditionalDependencies) + false diff --git a/lib/dll/example/fullbench-dll.vcxproj b/lib/dll/example/fullbench-dll.vcxproj index 3faacbae..44bbaf78 100644 --- a/lib/dll/example/fullbench-dll.vcxproj +++ b/lib/dll/example/fullbench-dll.vcxproj @@ -100,6 +100,7 @@ true $(SolutionDir)..\dll;%(AdditionalLibraryDirectories) libzstd.lib;%(AdditionalDependencies) + false @@ -141,6 +142,7 @@ true $(SolutionDir)..\dll;%(AdditionalLibraryDirectories) libzstd.lib;%(AdditionalDependencies) + false From 06b3f017afce3947919fc74b1938632d8a635db6 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 15 Dec 2016 12:31:18 +0100 Subject: [PATCH 04/28] appveyor.yml: fixed tests of fullbench-dll fullbench-lib --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 618f76d3..5dcbc4ba 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,7 +2,7 @@ version: 1.0.{build} environment: matrix: - COMPILER: "gcc" - MAKE_PARAMS: "make test && make lib && make -C tests fullbench-dll fullbench-lib" + MAKE_PARAMS: '"make test && make lib && make -C tests fullbench-dll fullbench-lib"' PLATFORM: "mingw64" - COMPILER: "gcc" MAKE_PARAMS: "make test" From b3843afcf5cf7771318d93b1ac9b72b3a0bee0db Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 16 Dec 2016 14:13:15 +0100 Subject: [PATCH 05/28] introduced platform.h --- programs/platform.h | 69 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 programs/platform.h diff --git a/programs/platform.h b/programs/platform.h new file mode 100644 index 00000000..978db1db --- /dev/null +++ b/programs/platform.h @@ -0,0 +1,69 @@ +/** + * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#ifndef PLATFORM_H_MODULE +#define PLATFORM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + +/* ************************************** +* Compiler Options +****************************************/ +#if defined(__INTEL_COMPILER) +# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced */ +#endif +#if defined(_MSC_VER) +# define _CRT_SECURE_NO_WARNINGS /* Disable some Visual warning messages for fopen, strncpy */ +# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# if _MSC_VER <= 1800 /* (1800 = Visual Studio 2013) */ +# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ +# endif +#endif + + +/* Unix Large Files support (>4GB) */ +#if !defined(__LP64__) /* No point defining Large file for 64 bit */ +# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ +# if defined(__sun__) && !defined(_LARGEFILE_SOURCE) /* Sun Solaris 32-bits requires specific definitions */ +# define _LARGEFILE_SOURCE /* fseeko, ftello */ +# elif !defined(_LARGEFILE64_SOURCE) +# define _LARGEFILE64_SOURCE /* off64_t, fseeko64, ftello64 */ +# endif +#endif + + +#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) || defined(__midipix__)) + /* UNIX-style OS. ------------------------------------------- */ +# if (defined(__APPLE__) && defined(__MACH__) || defined(__SVR4) || defined(_AIX) || defined(__hpux) \ + || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) \ /* POSIX.1–2001 (SUSv3) conformant */ +# define PLATFORM_POSIX_VERSION 200112L +# else +# if defined(__linux__) || defined(__linux) +# define _POSIX_C_SOURCE 200112L /* use feature test macro */ +# endif +# include /* declares _POSIX_VERSION */ +# if defined(_POSIX_VERSION) /* POSIX compliant */ +# define PLATFORM_POSIX_VERSION _POSIX_VERSION +# endif +# endif +#endif + +#if !defined(PLATFORM_POSIX_VERSION) +# define PLATFORM_POSIX_VERSION 0 +#endif + + +#if defined (__cplusplus) +} +#endif + +#endif /* PLATFORM_H_MODULE */ From b866e72826fdd4b7b6a697b10dfd825b3ba10a79 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 16 Dec 2016 14:24:01 +0100 Subject: [PATCH 06/28] tools use platform.h --- programs/bench.c | 3 ++- programs/dibio.c | 3 ++- programs/fileio.c | 3 ++- programs/platform.h | 13 ++++++++++++- programs/zstdcli.c | 3 ++- tests/datagencli.c | 2 +- tests/fullbench.c | 3 ++- tests/paramgrill.c | 3 ++- zlibWrapper/examples/zwrapbench.c | 3 ++- 9 files changed, 27 insertions(+), 9 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 9104ea89..f58f3e5c 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -11,7 +11,8 @@ /* ************************************* * Includes ***************************************/ -#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_sleep */ +#include "platform.h" /* Compiler options */ +#include "util.h" /* UTIL_GetFileSize, UTIL_sleep */ #include /* malloc, free */ #include /* memset */ #include /* fprintf, fopen, ftello64 */ diff --git a/programs/dibio.c b/programs/dibio.c index a793669c..74dc5405 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -12,7 +12,8 @@ /*-************************************* * Includes ***************************************/ -#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_getTotalFileSize */ +#include "platform.h" /* Compiler options */ +#include "util.h" /* UTIL_GetFileSize, UTIL_getTotalFileSize */ #include /* malloc, free */ #include /* memset */ #include /* fprintf, fopen, ftello64 */ diff --git a/programs/fileio.c b/programs/fileio.c index a493e97c..2ff5947f 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -22,7 +22,8 @@ /*-************************************* * Includes ***************************************/ -#include "util.h" /* Compiler options, UTIL_GetFileSize, _LARGEFILE64_SOURCE */ +#include "platform.h" /* Compiler options */ +#include "util.h" /* UTIL_GetFileSize, _LARGEFILE64_SOURCE */ #include /* fprintf, fopen, fread, _fileno, stdin, stdout */ #include /* malloc, free */ #include /* strcmp, strlen */ diff --git a/programs/platform.h b/programs/platform.h index 978db1db..095e52b8 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -30,7 +30,9 @@ extern "C" { #endif -/* Unix Large Files support (>4GB) */ +/* ************************************** +* Unix Large Files support (>4GB) +****************************************/ #if !defined(__LP64__) /* No point defining Large file for 64 bit */ # define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ # if defined(__sun__) && !defined(_LARGEFILE_SOURCE) /* Sun Solaris 32-bits requires specific definitions */ @@ -41,6 +43,12 @@ extern "C" { #endif +/* *********************************************************** +* Detect POSIX version +* PLATFORM_POSIX_VERSION = 0 for non-Unix e.g. Windows +* PLATFORM_POSIX_VERSION = 1 for Unix-like +* PLATFORM_POSIX_VERSION > 1 is equal to found _POSIX_VERSION +**************************************************************/ #if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) || defined(__midipix__)) /* UNIX-style OS. ------------------------------------------- */ # if (defined(__APPLE__) && defined(__MACH__) || defined(__SVR4) || defined(_AIX) || defined(__hpux) \ @@ -53,6 +61,8 @@ extern "C" { # include /* declares _POSIX_VERSION */ # if defined(_POSIX_VERSION) /* POSIX compliant */ # define PLATFORM_POSIX_VERSION _POSIX_VERSION +# else +# define PLATFORM_POSIX_VERSION 1 # endif # endif #endif @@ -62,6 +72,7 @@ extern "C" { #endif + #if defined (__cplusplus) } #endif diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 561730a5..fa2956ca 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -23,7 +23,8 @@ /*-************************************ * Dependencies **************************************/ -#include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */ +#include "platform.h" /* Compiler options, PLATFORM_POSIX_VERSION */ +#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */ #include /* strcmp, strlen */ #include /* errno */ #include "fileio.h" diff --git a/tests/datagencli.c b/tests/datagencli.c index 2f3ebc4d..0729cdd0 100644 --- a/tests/datagencli.c +++ b/tests/datagencli.c @@ -11,7 +11,7 @@ /*-************************************ * Dependencies **************************************/ -#include "util.h" /* Compiler options */ +#include "platform.h" /* Compiler options */ #include /* fprintf, stderr */ #include "datagen.h" /* RDG_generate */ diff --git a/tests/fullbench.c b/tests/fullbench.c index 233b4e93..e5547d08 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -11,7 +11,8 @@ /*_************************************ * Includes **************************************/ -#include "util.h" /* Compiler options, UTIL_GetFileSize */ +#include "platform.h" /* Compiler options */ +#include "util.h" /* UTIL_GetFileSize */ #include /* malloc */ #include /* fprintf, fopen, ftello64 */ #include /* clock_t, clock, CLOCKS_PER_SEC */ diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 5eabcba2..19658fdd 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -11,7 +11,8 @@ /*-************************************ * Dependencies **************************************/ -#include "util.h" /* Compiler options, UTIL_GetFileSize */ +#include "platform.h" /* Compiler options */ +#include "util.h" /* UTIL_GetFileSize */ #include /* malloc */ #include /* fprintf, fopen, ftello64 */ #include /* strcmp */ diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index e0aca001..c3968dba 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -11,7 +11,8 @@ /* ************************************* * Includes ***************************************/ -#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_sleep */ +#include "platform.h" /* Compiler options */ +#include "util.h" /* UTIL_GetFileSize, UTIL_sleep */ #include /* malloc, free */ #include /* memset */ #include /* fprintf, fopen, ftello64 */ From b0e670a054edf3f8f388fe8e53688372c5883071 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 16 Dec 2016 14:25:12 +0100 Subject: [PATCH 07/28] util.h uses platform.h --- programs/util.h | 29 ++--------------------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/programs/util.h b/programs/util.h index 0fa70577..45fdacb1 100644 --- a/programs/util.h +++ b/programs/util.h @@ -14,37 +14,12 @@ extern "C" { #endif -/* ************************************** -* Compiler Options -****************************************/ -#if defined(__INTEL_COMPILER) -# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced */ -#endif -#if defined(_MSC_VER) -# define _CRT_SECURE_NO_WARNINGS /* Disable some Visual warning messages for fopen, strncpy */ -# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#if _MSC_VER <= 1800 /* (1800 = Visual Studio 2013) */ -# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ -#endif -#endif - - -/* Unix Large Files support (>4GB) */ -#if !defined(__LP64__) /* No point defining Large file for 64 bit */ -# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ -# if defined(__sun__) && !defined(_LARGEFILE_SOURCE) /* Sun Solaris 32-bits requires specific definitions */ -# define _LARGEFILE_SOURCE /* fseeko, ftello */ -# elif !defined(_LARGEFILE64_SOURCE) -# define _LARGEFILE64_SOURCE /* off64_t, fseeko64, ftello64 */ -# endif -#endif - /*-**************************************** * Dependencies ******************************************/ -#include /* features.h with _POSIX_C_SOURCE, malloc */ +#include "platform.h" /* Compiler options, PLATFORM_POSIX_VERSION */ +#include /* malloc */ #include /* fprintf */ #include /* stat, utime */ #include /* stat */ From 3cdfe266cfc73333f67d0a63baa8d98ce27232f8 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 16 Dec 2016 15:00:50 +0100 Subject: [PATCH 08/28] use PLATFORM_POSIX_VERSION --- programs/platform.h | 14 +++++++------- programs/util.h | 2 +- programs/zstdcli.c | 3 +-- tests/datagencli.c | 1 + 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/programs/platform.h b/programs/platform.h index 095e52b8..6135cb58 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -43,12 +43,12 @@ extern "C" { #endif -/* *********************************************************** +/* ************************************************************ * Detect POSIX version -* PLATFORM_POSIX_VERSION = 0 for non-Unix e.g. Windows -* PLATFORM_POSIX_VERSION = 1 for Unix-like -* PLATFORM_POSIX_VERSION > 1 is equal to found _POSIX_VERSION -**************************************************************/ +* PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows +* PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX +* PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION +***************************************************************/ #if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) || defined(__midipix__)) /* UNIX-style OS. ------------------------------------------- */ # if (defined(__APPLE__) && defined(__MACH__) || defined(__SVR4) || defined(_AIX) || defined(__hpux) \ @@ -62,13 +62,13 @@ extern "C" { # if defined(_POSIX_VERSION) /* POSIX compliant */ # define PLATFORM_POSIX_VERSION _POSIX_VERSION # else -# define PLATFORM_POSIX_VERSION 1 +# define PLATFORM_POSIX_VERSION 0 # endif # endif #endif #if !defined(PLATFORM_POSIX_VERSION) -# define PLATFORM_POSIX_VERSION 0 +# define PLATFORM_POSIX_VERSION -1 #endif diff --git a/programs/util.h b/programs/util.h index 45fdacb1..179d5ca9 100644 --- a/programs/util.h +++ b/programs/util.h @@ -73,7 +73,7 @@ extern "C" { # define SET_HIGH_PRIORITY /* disabled */ # endif # define UTIL_sleep(s) sleep(s) -# if defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 199309L)) +# if PLATFORM_POSIX_VERSION >= 200112L /* nanosleep requires POSIX.1-2001 */ # define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); } # else # define UTIL_sleepMilli(milli) /* disabled */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index fa2956ca..fad23753 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -44,8 +44,7 @@ #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) # include /* _isatty */ # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) -#elif defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) || \ - defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */ +#elif PLATFORM_POSIX_VERSION >= 200112L /* isatty requires POSIX.1-2001 */ # include /* isatty */ # define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) #else diff --git a/tests/datagencli.c b/tests/datagencli.c index 0729cdd0..2b3d3b51 100644 --- a/tests/datagencli.c +++ b/tests/datagencli.c @@ -14,6 +14,7 @@ #include "platform.h" /* Compiler options */ #include /* fprintf, stderr */ #include "datagen.h" /* RDG_generate */ +#include "mem.h" /* U32, U64 */ /*-************************************ From b0f3663edc00c775beae7cf67e618347d48a77e9 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 16 Dec 2016 15:41:18 +0100 Subject: [PATCH 09/28] imporved support for POSIX-type OSes --- programs/platform.h | 6 +++--- programs/util.h | 9 +++------ programs/zstdcli.c | 11 ++++++++++- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/programs/platform.h b/programs/platform.h index 6135cb58..c69dcb2d 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -49,10 +49,10 @@ extern "C" { * PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX * PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION ***************************************************************/ -#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) || defined(__midipix__)) +#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) || defined(__midipix__) || defined(__VMS)) /* UNIX-style OS. ------------------------------------------- */ -# if (defined(__APPLE__) && defined(__MACH__) || defined(__SVR4) || defined(_AIX) || defined(__hpux) \ - || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) \ /* POSIX.1–2001 (SUSv3) conformant */ +# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) \ + || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* POSIX.1–2001 (SUSv3) conformant */ # define PLATFORM_POSIX_VERSION 200112L # else # if defined(__linux__) || defined(__linux) diff --git a/programs/util.h b/programs/util.h index 179d5ca9..8c171c3b 100644 --- a/programs/util.h +++ b/programs/util.h @@ -63,7 +63,7 @@ extern "C" { # define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) # define UTIL_sleep(s) Sleep(1000*s) # define UTIL_sleepMilli(milli) Sleep(milli) -#elif (defined(__unix__) || defined(__unix) || defined(__VMS) || defined(__midipix__) || (defined(__APPLE__) && defined(__MACH__))) +#elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */ # include # include /* setpriority */ # include /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */ @@ -73,7 +73,7 @@ extern "C" { # define SET_HIGH_PRIORITY /* disabled */ # endif # define UTIL_sleep(s) sleep(s) -# if PLATFORM_POSIX_VERSION >= 200112L /* nanosleep requires POSIX.1-2001 */ +# if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L) /* nanosleep requires POSIX.1-2001 */ # define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); } # else # define UTIL_sleepMilli(milli) /* disabled */ @@ -300,10 +300,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ return nbFiles; } -#elif (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) || \ - (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) || \ - defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ - (defined(__linux__) && defined(_POSIX_C_SOURCE)) /* opendir */ +#elif (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ # define UTIL_HAS_CREATEFILELIST # include /* opendir, readdir */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index fad23753..197dd3ff 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -44,7 +44,7 @@ #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) # include /* _isatty */ # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) -#elif PLATFORM_POSIX_VERSION >= 200112L /* isatty requires POSIX.1-2001 */ +#elif (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || PLATFORM_POSIX_VERSION >= 200112L /* isatty requires POSIX.1-2001 */ # include /* isatty */ # define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) #else @@ -470,6 +470,15 @@ int main(int argCount, const char* argv[]) /* Welcome message (if verbose) */ DISPLAYLEVEL(3, WELCOME_MESSAGE); +#ifdef _POSIX_C_SOURCE + DISPLAYLEVEL(4, "_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE); +#endif +#ifdef _POSIX_VERSION + DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL\n", (long) _POSIX_VERSION); +#endif +#ifdef PLATFORM_POSIX_VERSION + DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION); +#endif #ifdef UTIL_HAS_CREATEFILELIST if (recursive) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */ From 0b37205098bfcaf9e2ddca670c1bd42d29226a73 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 16 Dec 2016 17:12:23 +0100 Subject: [PATCH 10/28] util.h: minor improvement --- programs/util.h | 2 +- programs/zstdcli.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/util.h b/programs/util.h index 8c171c3b..9dbd7fb1 100644 --- a/programs/util.h +++ b/programs/util.h @@ -300,7 +300,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ return nbFiles; } -#elif (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ +#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ # define UTIL_HAS_CREATEFILELIST # include /* opendir, readdir */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 197dd3ff..68ae3c20 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -44,7 +44,7 @@ #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) # include /* _isatty */ # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) -#elif (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || PLATFORM_POSIX_VERSION >= 200112L /* isatty requires POSIX.1-2001 */ +#elif (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) /* isatty requires POSIX.1-2001 */ # include /* isatty */ # define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) #else From 95ff9a1ed77e31c2a9c76611245a5691d6879358 Mon Sep 17 00:00:00 2001 From: Jacques Germishuys Date: Sun, 18 Dec 2016 18:17:25 +0200 Subject: [PATCH 11/28] cmake msvc: correct resources --- build/cmake/lib/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index 65942b41..b8510f7b 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -110,8 +110,8 @@ IF (ZSTD_LEGACY_SUPPORT) ENDIF (ZSTD_LEGACY_SUPPORT) IF (MSVC) - SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/build/VS2010/zstdlib) - SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstdlib.rc) + SET(MSVC_RESOURCE_DIR ${ROOT_DIR}/build/VS2010/libzstd-dll) + SET(PlatformDependResources ${MSVC_RESOURCE_DIR}/libzstd-dll.rc) ENDIF (MSVC) # Split project to static and shared libraries build From 85a150eab7fe505b53438f8df04c577eda7a4022 Mon Sep 17 00:00:00 2001 From: Jacques Germishuys Date: Mon, 19 Dec 2016 14:15:47 +0200 Subject: [PATCH 12/28] cmake msvc: resources are not required for the static library --- build/cmake/lib/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index b8510f7b..41fe2733 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -115,7 +115,7 @@ IF (MSVC) ENDIF (MSVC) # Split project to static and shared libraries build -ADD_LIBRARY(libzstd_static STATIC ${Sources} ${Headers} ${PlatformDependResources}) +ADD_LIBRARY(libzstd_static STATIC ${Sources} ${Headers}) ADD_LIBRARY(libzstd_shared SHARED ${Sources} ${Headers} ${PlatformDependResources}) # Add specific compile definitions for MSVC project From 4beb51f17ceb14451845fc6397f53c9858e74a50 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 20 Dec 2016 10:17:21 +0100 Subject: [PATCH 13/28] tests of ZSTD_compressContinue_extDict --- tests/fullbench.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/fullbench.c b/tests/fullbench.c index 233b4e93..5c7e2cc3 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -179,6 +179,23 @@ size_t local_ZSTD_compressContinue(void* dst, size_t dstCapacity, void* buff2, c return ZSTD_compressEnd(g_zcc, dst, dstCapacity, src, srcSize); } +#define FIRST_BLOCK_SIZE 8 +size_t local_ZSTD_compressContinue_extDict(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) +{ + BYTE firstBlockBuf[FIRST_BLOCK_SIZE]; + + (void)buff2; + memcpy(firstBlockBuf, src, FIRST_BLOCK_SIZE); + ZSTD_compressBegin(g_zcc, 1); + + { size_t const compressResult = ZSTD_compressContinue(g_zcc, dst, dstCapacity, firstBlockBuf, FIRST_BLOCK_SIZE); + if (ZSTD_isError(compressResult)) { DISPLAY("local_ZSTD_compressContinue_extDict error : %s\n", ZSTD_getErrorName(compressResult)); return compressResult; } + dst = (BYTE*)dst + compressResult; + dstCapacity -= compressResult; + } + return ZSTD_compressEnd(g_zcc, dst, dstCapacity, (const BYTE*)src + FIRST_BLOCK_SIZE, srcSize - FIRST_BLOCK_SIZE); +} + size_t local_ZSTD_decompressContinue(void* dst, size_t dstCapacity, void* buff2, const void* src, size_t srcSize) { size_t regeneratedSize = 0; @@ -229,6 +246,9 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) benchFunction = local_ZSTD_compressContinue; benchName = "ZSTD_compressContinue"; break; case 12: + benchFunction = local_ZSTD_compressContinue_extDict; benchName = "ZSTD_compressContinue_extDict"; + break; + case 13: benchFunction = local_ZSTD_decompressContinue; benchName = "ZSTD_decompressContinue"; break; case 31: @@ -268,6 +288,9 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb) if (g_zcc==NULL) g_zcc = ZSTD_createCCtx(); break; case 12 : + if (g_zcc==NULL) g_zcc = ZSTD_createCCtx(); + break; + case 13 : if (g_zdc==NULL) g_zdc = ZSTD_createDCtx(); g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize, 1); break; From 1c1db6b8454e15d772983343c278905dc14655d0 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 20 Dec 2016 11:21:26 +0100 Subject: [PATCH 14/28] windres updated to v1.1.3 --- programs/windres/zstd32.res | Bin 1044 -> 1044 bytes programs/windres/zstd64.res | Bin 1044 -> 1044 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/programs/windres/zstd32.res b/programs/windres/zstd32.res index 6135dc431fed231ce2143b394625bc0b52b0615a..748192a988648a0be3b17ce47353f28015003815 100644 GIT binary patch delta 33 mcmbQjF@ Date: Tue, 20 Dec 2016 10:47:52 -0800 Subject: [PATCH 15/28] Fix dictionary loading bug causing an MSAN failure Offset rep codes must be in the range `[1, dictSize)`. Fix dictionary loading to reject `0` as a offset rep code. --- lib/compress/zstd_compress.c | 6 +++--- lib/decompress/zstd_decompress.c | 6 +++--- lib/legacy/zstd_v07.c | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 7fd73baa..e88f6a1d 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2557,9 +2557,9 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_ } if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); - cctx->rep[0] = MEM_readLE32(dictPtr+0); if (cctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted); - cctx->rep[1] = MEM_readLE32(dictPtr+4); if (cctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted); - cctx->rep[2] = MEM_readLE32(dictPtr+8); if (cctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted); + cctx->rep[0] = MEM_readLE32(dictPtr+0); if (cctx->rep[0] == 0 || cctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted); + cctx->rep[1] = MEM_readLE32(dictPtr+4); if (cctx->rep[1] == 0 || cctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted); + cctx->rep[2] = MEM_readLE32(dictPtr+8); if (cctx->rep[2] == 0 || cctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted); dictPtr += 12; { U32 offcodeMax = MaxOff; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 085477bf..7addff8d 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1671,9 +1671,9 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c } if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); - dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted); - dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted); - dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted); + dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] == 0 || dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted); + dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] == 0 || dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted); + dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] == 0 || dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted); dictPtr += 12; dctx->litEntropy = dctx->fseEntropy = 1; diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index b607ec37..b1fcdc76 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -4134,9 +4134,9 @@ static size_t ZSTDv07_loadEntropy(ZSTDv07_DCtx* dctx, const void* const dict, si } if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); - dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted); - dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted); - dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted); + dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] == 0 || dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted); + dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] == 0 || dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted); + dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] == 0 || dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted); dictPtr += 12; dctx->litEntropy = dctx->fseEntropy = 1; From 9d085973646782eb2e74c8b19538437860c8aa5e Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Tue, 20 Dec 2016 11:13:45 -0800 Subject: [PATCH 16/28] Add test for invalid offset rep codes --- tests/.gitignore | 1 + tests/Makefile | 10 ++++++-- tests/invalidDictionaries.c | 51 +++++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 tests/invalidDictionaries.c diff --git a/tests/.gitignore b/tests/.gitignore index 586c17b6..e932ad91 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -15,6 +15,7 @@ paramgrill32 roundTripCrash longmatch symbols +invalidDictionaries # Tmp test directory zstdtest diff --git a/tests/Makefile b/tests/Makefile index abf89f2f..f1c196ba 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -146,6 +146,9 @@ roundTripCrash : $(ZSTD_FILES) roundTripCrash.c longmatch : $(ZSTD_FILES) longmatch.c $(CC) $(FLAGS) $^ -o $@$(EXT) +invalidDictionaries : $(ZSTD_FILES) invalidDictionaries.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + symbols : symbols.c $(MAKE) -C $(ZSTDDIR) libzstd ifneq (,$(filter Windows%,$(OS))) @@ -173,7 +176,7 @@ clean: fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT)\ zstreamtest$(EXT) zstreamtest32$(EXT) \ datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \ - symbols$(EXT) + symbols$(EXT) invalidDictionaries$(EXT) @echo Cleaning completed @@ -213,7 +216,7 @@ zstd-playTests: datagen file $(ZSTD) ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST) -test: test-zstd test-fullbench test-fuzzer test-zstream test-longmatch +test: test-zstd test-fullbench test-fuzzer test-zstream test-longmatch test-invalidDictionaries test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32 @@ -273,6 +276,9 @@ test-zstream32: zstreamtest32 test-longmatch: longmatch $(QEMU_SYS) ./longmatch +test-invalidDictionaries: invalidDictionaries + $(QEMU_SYS) ./invalidDictionaries + test-symbols: symbols $(QEMU_SYS) ./symbols diff --git a/tests/invalidDictionaries.c b/tests/invalidDictionaries.c new file mode 100644 index 00000000..fe8b23b5 --- /dev/null +++ b/tests/invalidDictionaries.c @@ -0,0 +1,51 @@ +#include +#include "zstd.h" + +static const char invalidRepCode[] = { + 0x37, 0xa4, 0x30, 0xec, 0x2a, 0x00, 0x00, 0x00, 0x39, 0x10, 0xc0, 0xc2, + 0xa6, 0x00, 0x0c, 0x30, 0xc0, 0x00, 0x03, 0x0c, 0x30, 0x20, 0x72, 0xf8, + 0xb4, 0x6d, 0x4b, 0x9f, 0xfc, 0x97, 0x29, 0x49, 0xb2, 0xdf, 0x4b, 0x29, + 0x7d, 0x4a, 0xfc, 0x83, 0x18, 0x22, 0x75, 0x23, 0x24, 0x44, 0x4d, 0x02, + 0xb7, 0x97, 0x96, 0xf6, 0xcb, 0xd1, 0xcf, 0xe8, 0x22, 0xea, 0x27, 0x36, + 0xb7, 0x2c, 0x40, 0x46, 0x01, 0x08, 0x23, 0x01, 0x00, 0x00, 0x06, 0x1e, + 0x3c, 0x83, 0x81, 0xd6, 0x18, 0xd4, 0x12, 0x3a, 0x04, 0x00, 0x80, 0x03, + 0x08, 0x0e, 0x12, 0x1c, 0x12, 0x11, 0x0d, 0x0e, 0x0a, 0x0b, 0x0a, 0x09, + 0x10, 0x0c, 0x09, 0x05, 0x04, 0x03, 0x06, 0x06, 0x06, 0x02, 0x00, 0x03, + 0x00, 0x00, 0x02, 0x02, 0x00, 0x04, 0x06, 0x03, 0x06, 0x08, 0x24, 0x6b, + 0x0d, 0x01, 0x10, 0x04, 0x81, 0x07, 0x00, 0x00, 0x04, 0xb9, 0x58, 0x18, + 0x06, 0x59, 0x92, 0x43, 0xce, 0x28, 0xa5, 0x08, 0x88, 0xc0, 0x80, 0x88, + 0x8c, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00 +}; + +typedef struct dictionary_s { + const char *data; + size_t size; +} dictionary; + +static const dictionary dictionaries[] = { + {invalidRepCode, sizeof(invalidRepCode)}, + {NULL, 0}, +}; + +int main(int argc, const char** argv) { + const dictionary *dict; + for (dict = dictionaries; dict->data != NULL; ++dict) { + ZSTD_CDict *cdict; + ZSTD_DDict *ddict; + cdict = ZSTD_createCDict(dict->data, dict->size, 1); + if (cdict) { + ZSTD_freeCDict(cdict); + return 1; + } + ddict = ZSTD_createDDict(dict->data, dict->size); + if (ddict) { + ZSTD_freeDDict(ddict); + return 2; + } + } + + (void)argc; + (void)argv; + return 0; +} From ead350bdc0b3e20acd282747f2f6a4560e42e3c0 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 09:04:59 +0100 Subject: [PATCH 17/28] improved util.h and platform.h --- programs/platform.h | 84 +++++++++++++++++++++++++++++++++------- programs/util.h | 94 ++++++++++++++++++++++++++++++--------------- 2 files changed, 135 insertions(+), 43 deletions(-) diff --git a/programs/platform.h b/programs/platform.h index c69dcb2d..b8911f2c 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -14,6 +14,7 @@ extern "C" { #endif + /* ************************************** * Compiler Options ****************************************/ @@ -24,21 +25,45 @@ extern "C" { # define _CRT_SECURE_NO_WARNINGS /* Disable some Visual warning messages for fopen, strncpy */ # define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# if _MSC_VER <= 1800 /* (1800 = Visual Studio 2013) */ +# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ # define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ # endif +# if (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ +# define fseek _fseeki64 +# endif #endif /* ************************************** -* Unix Large Files support (>4GB) +* Detect 64-bit OS +* http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros ****************************************/ -#if !defined(__LP64__) /* No point defining Large file for 64 bit */ -# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ -# if defined(__sun__) && !defined(_LARGEFILE_SOURCE) /* Sun Solaris 32-bits requires specific definitions */ -# define _LARGEFILE_SOURCE /* fseeko, ftello */ -# elif !defined(_LARGEFILE64_SOURCE) -# define _LARGEFILE64_SOURCE /* off64_t, fseeko64, ftello64 */ +#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ \ + || defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ \ + || (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ \ + || defined __x86_64__s || defined _M_X64 /* x86 64-bit */ \ + || defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ \ + || (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ \ + || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || defined _ADDR64 /* Cray */ \ + || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */ +# if !defined(__64BIT__) +# define __64BIT__ 1 +# endif +#endif + + +/* ********************************************************* +* Turn on Large Files support (>4GB) for 32-bit Linux/Unix +***********************************************************/ +#if !defined(__64BIT__) /* No point defining Large file for 64 bit */ +# if !defined(_FILE_OFFSET_BITS) +# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ +# endif +# if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */ +# define _LARGEFILE_SOURCE 1 /* Large File Support extension (LFS) - fseeko, ftello */ +# endif +# if defined(_AIX) || defined(__hpux) +# define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */ # endif #endif @@ -49,10 +74,10 @@ extern "C" { * PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX * PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION ***************************************************************/ -#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) || defined(__midipix__) || defined(__VMS)) - /* UNIX-style OS. ------------------------------------------- */ -# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) \ - || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* POSIX.1–2001 (SUSv3) conformant */ +#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \ + || defined(__midipix__) || defined(__VMS)) +# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1–2001 (SUSv3) conformant */ \ + || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* BSD distros */ # define PLATFORM_POSIX_VERSION 200112L # else # if defined(__linux__) || defined(__linux) @@ -66,12 +91,45 @@ extern "C" { # endif # endif #endif - #if !defined(PLATFORM_POSIX_VERSION) # define PLATFORM_POSIX_VERSION -1 #endif +/*-********************************************* +* Detect if isatty() and fileno() are available +************************************************/ +#if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__) +# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +#elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +# include /* _isatty */ +# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#else +# define IS_CONSOLE(stdStream) 0 +#endif + + +/****************************** +* OS-specific Includes +******************************/ +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) +# include /* _O_BINARY */ +# include /* _setmode, _fileno, _get_osfhandle */ +# if !defined(__DJGPP__) +# include /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ +# include /* FSCTL_SET_SPARSE */ +# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } +# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } +# else +# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) +# define SET_SPARSE_FILE_MODE(file) +# endif +#else +# define SET_BINARY_MODE(file) +# define SET_SPARSE_FILE_MODE(file) +#endif + + #if defined (__cplusplus) } diff --git a/programs/util.h b/programs/util.h index 9dbd7fb1..3e8aaf5d 100644 --- a/programs/util.h +++ b/programs/util.h @@ -1,11 +1,21 @@ -/** - * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - */ +/* + util.h - utility functions + Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ #ifndef UTIL_H_MODULE #define UTIL_H_MODULE @@ -20,6 +30,7 @@ extern "C" { ******************************************/ #include "platform.h" /* Compiler options, PLATFORM_POSIX_VERSION */ #include /* malloc */ +#include /* size_t, ptrdiff_t */ #include /* fprintf */ #include /* stat, utime */ #include /* stat */ @@ -32,27 +43,6 @@ extern "C" { #endif #include /* time */ #include -#include "mem.h" /* U32, U64 */ - - -/* ************************************* -* Constants -***************************************/ -#define LIST_SIZE_INCREASE (8*1024) - - -/*-**************************************** -* Compiler specifics -******************************************/ -#if defined(__GNUC__) -# define UTIL_STATIC static __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define UTIL_STATIC static inline -#elif defined(_MSC_VER) -# define UTIL_STATIC static __inline -#else -# define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ -#endif /*-**************************************** @@ -85,6 +75,49 @@ extern "C" { #endif +/* ************************************* +* Constants +***************************************/ +#define LIST_SIZE_INCREASE (8*1024) + + +/*-************************************************************** +* Basic Types +*****************************************************************/ +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef int16_t S16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef int64_t S64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef signed short S16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; + typedef signed long long S64; +#endif + + +/*-**************************************** +* Compiler specifics +******************************************/ +#if defined(__GNUC__) +# define UTIL_STATIC static __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define UTIL_STATIC static inline +#elif defined(_MSC_VER) +# define UTIL_STATIC static __inline +#else +# define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + + /*-**************************************** * Time functions ******************************************/ @@ -303,6 +336,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ #elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ # define UTIL_HAS_CREATEFILELIST # include /* opendir, readdir */ +# include /* strerror, memcpy */ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) { @@ -364,7 +398,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) { (void)bufStart; (void)bufEnd; (void)pos; - fprintf(stderr, "Directory %s ignored (zstd compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName); + fprintf(stderr, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName); return 0; } From 16ae6563a22e513b6e90ca49975e55852faa5c50 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 09:06:14 +0100 Subject: [PATCH 18/28] executables use new util.h and platform.h --- programs/bench.c | 1 - programs/datagen.c | 14 +------------- programs/dibio.c | 1 - programs/fileio.c | 14 -------------- tests/datagencli.c | 2 +- tests/fullbench.c | 3 +-- tests/fuzzer.c | 5 ++--- tests/paramgrill.c | 4 ++-- tests/zbufftest.c | 5 ++--- tests/zstreamtest.c | 5 ++--- zlibWrapper/examples/zwrapbench.c | 3 +-- 11 files changed, 12 insertions(+), 45 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index a4b57efc..2b7436a4 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -18,7 +18,6 @@ #include /* fprintf, fopen, ftello64 */ #include /* clock_t, clock, CLOCKS_PER_SEC */ -#include "mem.h" #define ZSTD_STATIC_LINKING_ONLY #include "zstd.h" #include "datagen.h" /* RDG_genBuffer */ diff --git a/programs/datagen.c b/programs/datagen.c index 1af5a38a..da5124cf 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -20,22 +20,10 @@ /*-************************************ * Dependencies **************************************/ +#include "util.h" /* U32 */ #include /* malloc, free */ #include /* FILE, fwrite, fprintf */ #include /* memcpy */ -#include "mem.h" /* U32 */ - - -/*-************************************ -* OS-specific Includes -**************************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include /* _O_BINARY */ -# include /* _setmode, _isatty */ -# define SET_BINARY_MODE(file) {int unused = _setmode(_fileno(file), _O_BINARY); (void)unused; } -#else -# define SET_BINARY_MODE(file) -#endif /*-************************************ diff --git a/programs/dibio.c b/programs/dibio.c index 152ea4d0..1c22949f 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -20,7 +20,6 @@ #include /* clock_t, clock, CLOCKS_PER_SEC */ #include /* errno */ -#include "mem.h" /* read */ #include "error_private.h" #include "dibio.h" diff --git a/programs/fileio.c b/programs/fileio.c index 325a17a5..58fe7e35 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -30,7 +30,6 @@ #include /* clock */ #include /* errno */ -#include "mem.h" #include "fileio.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ #include "zstd.h" @@ -42,19 +41,6 @@ #endif -/*-************************************* -* OS-specific Includes -***************************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include /* _O_BINARY */ -# include /* _setmode, _isatty */ -# define SET_BINARY_MODE(file) { if (_setmode(_fileno(file), _O_BINARY) == -1) perror("Cannot set _O_BINARY"); } -#else -# include /* isatty */ -# define SET_BINARY_MODE(file) -#endif - - /*-************************************* * Constants ***************************************/ diff --git a/tests/datagencli.c b/tests/datagencli.c index 2b3d3b51..c4bf4893 100644 --- a/tests/datagencli.c +++ b/tests/datagencli.c @@ -12,9 +12,9 @@ * Dependencies **************************************/ #include "platform.h" /* Compiler options */ +#include "util.h" /* U32 */ #include /* fprintf, stderr */ #include "datagen.h" /* RDG_generate */ -#include "mem.h" /* U32, U64 */ /*-************************************ diff --git a/tests/fullbench.c b/tests/fullbench.c index e5547d08..439de5c7 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -12,12 +12,11 @@ * Includes **************************************/ #include "platform.h" /* Compiler options */ -#include "util.h" /* UTIL_GetFileSize */ +#include "util.h" /* U32 */ #include /* malloc */ #include /* fprintf, fopen, ftello64 */ #include /* clock_t, clock, CLOCKS_PER_SEC */ -#include "mem.h" #ifndef ZSTD_DLL_IMPORT #include "zstd_internal.h" /* ZSTD_blockHeaderSize, blockType_e, KB, MB */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressBegin, ZSTD_compressContinue, etc. */ diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 86d4c6be..9088e248 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -12,8 +12,6 @@ * Compiler specific **************************************/ #ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS /* fgets */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ #endif @@ -21,6 +19,8 @@ /*-************************************ * Includes **************************************/ +#include "platform.h" /* Compiler options */ +#include "util.h" /* U32 */ #include /* free */ #include /* fgets, sscanf */ #include /* strcmp */ @@ -30,7 +30,6 @@ #include "zstd_errors.h" /* ZSTD_getErrorCode */ #include "zdict.h" /* ZDICT_trainFromBuffer */ #include "datagen.h" /* RDG_genBuffer */ -#include "mem.h" #define XXH_STATIC_LINKING_ONLY #include "xxhash.h" /* XXH64 */ diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 19658fdd..0825b37a 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -12,14 +12,14 @@ * Dependencies **************************************/ #include "platform.h" /* Compiler options */ -#include "util.h" /* UTIL_GetFileSize */ +#include "util.h" /* UTIL_getFileSize */ #include /* malloc */ #include /* fprintf, fopen, ftello64 */ #include /* strcmp */ #include /* log */ #include /* clock_t */ -#include "mem.h" +#include "mem.h" /* MEM_32bits() */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_estimateCCtxSize */ #include "zstd.h" #include "datagen.h" diff --git a/tests/zbufftest.c b/tests/zbufftest.c index 14b73923..87cf80b5 100644 --- a/tests/zbufftest.c +++ b/tests/zbufftest.c @@ -12,8 +12,6 @@ * Compiler specific **************************************/ #ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS /* fgets */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ #endif @@ -21,11 +19,12 @@ /*-************************************ * Includes **************************************/ +#include "platform.h" /* Compiler options */ #include /* free */ #include /* fgets, sscanf */ #include /* clock_t, clock() */ #include /* strcmp */ -#include "mem.h" +#include "mem.h" /* MEM_writeLE32 */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */ #include "zstd.h" /* ZSTD_compressBound */ #define ZBUFF_STATIC_LINKING_ONLY /* ZBUFF_createCCtx_advanced */ diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index ce619308..9da0010b 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -12,8 +12,6 @@ * Compiler specific **************************************/ #ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS /* fgets */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ #endif @@ -21,11 +19,12 @@ /*-************************************ * Includes **************************************/ +#include "platform.h" /* Compiler options */ #include /* free */ #include /* fgets, sscanf */ #include /* clock_t, clock() */ #include /* strcmp */ -#include "mem.h" +#include "mem.h" /* MEM_writeLE32 */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel, ZSTD_customMem */ #include "zstd.h" /* ZSTD_compressBound */ #include "zstd_errors.h" /* ZSTD_error_srcSize_wrong */ diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index c3968dba..8999657f 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -12,14 +12,13 @@ * Includes ***************************************/ #include "platform.h" /* Compiler options */ -#include "util.h" /* UTIL_GetFileSize, UTIL_sleep */ +#include "util.h" /* U32, UTIL_GetFileSize, UTIL_sleep */ #include /* malloc, free */ #include /* memset */ #include /* fprintf, fopen, ftello64 */ #include /* clock_t, clock, CLOCKS_PER_SEC */ #include /* toupper */ -#include "mem.h" #define ZSTD_STATIC_LINKING_ONLY #include "zstd.h" #include "datagen.h" /* RDG_genBuffer */ From 20b089e53d855bfc1446d1217e30709701af2f87 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 09:19:15 +0100 Subject: [PATCH 19/28] simplified zstdcli.c --- programs/datagen.c | 9 +-------- programs/fileio.c | 1 - programs/zstdcli.c | 14 -------------- 3 files changed, 1 insertion(+), 23 deletions(-) diff --git a/programs/datagen.c b/programs/datagen.c index da5124cf..8209c851 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -9,17 +9,10 @@ -/* ************************************* -* Compiler Options -***************************************/ -#if defined(_MSC_VER) -# define _CRT_SECURE_NO_WARNINGS /* removes Visual warning on strerror() */ -# define _CRT_SECURE_NO_DEPRECATE /* removes VS2005 warning on strerror() */ -#endif - /*-************************************ * Dependencies **************************************/ +#include "platform.h" /* Compiler options */ #include "util.h" /* U32 */ #include /* malloc, free */ #include /* FILE, fwrite, fprintf */ diff --git a/programs/fileio.c b/programs/fileio.c index 58fe7e35..4f6db992 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -11,7 +11,6 @@ * Compiler Options ***************************************/ #ifdef _MSC_VER /* Visual */ -# define _CRT_SECURE_NO_WARNINGS /* removes Visual warning on strerror() */ # pragma warning(disable : 4204) /* non-constant aggregate initializer */ #endif #if defined(__MINGW32__) && !defined(_POSIX_SOURCE) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 0d7ce8b3..29fcb746 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -38,20 +38,6 @@ #include "zstd.h" /* ZSTD_VERSION_STRING */ -/*-************************************ -* OS-specific Includes -**************************************/ -#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include /* _isatty */ -# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) -#elif (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) /* isatty requires POSIX.1-2001 */ -# include /* isatty */ -# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) -#else -# define IS_CONSOLE(stdStream) 0 -#endif - - /*-************************************ * Constants **************************************/ From 5736db219e7ec71bf50ad0d66e436aa8724cd9df Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 09:26:00 +0100 Subject: [PATCH 20/28] fix basic types redefinition --- lib/common/mem.h | 3 +++ programs/util.h | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/common/mem.h b/lib/common/mem.h index 32c63dd1..fb73ef09 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -46,6 +46,8 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size /*-************************************************************** * Basic Types *****************************************************************/ +#ifndef BASIC_TYPES_DEFINED +#define BASIC_TYPES_DEFINED #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) # include typedef uint8_t BYTE; @@ -66,6 +68,7 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size typedef signed long long S64; typedef ptrdiff_t iPtrDiff; #endif +#endif /*-************************************************************** diff --git a/programs/util.h b/programs/util.h index 3e8aaf5d..409f6d7c 100644 --- a/programs/util.h +++ b/programs/util.h @@ -84,7 +84,9 @@ extern "C" { /*-************************************************************** * Basic Types *****************************************************************/ -#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +#ifndef BASIC_TYPES_DEFINED +#define BASIC_TYPES_DEFINED +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) # include typedef uint8_t BYTE; typedef uint16_t U16; @@ -101,7 +103,8 @@ extern "C" { typedef signed int S32; typedef unsigned long long U64; typedef signed long long S64; -#endif +#endif +#endif /*-**************************************** From a35b9448acf0a1f06e413389dd22c82d9e746bf5 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 11:18:45 +0100 Subject: [PATCH 21/28] improved MinGW support --- programs/fileio.c | 3 --- programs/platform.h | 29 +++++++++++++++++++++-------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 4f6db992..fa52f9af 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -13,9 +13,6 @@ #ifdef _MSC_VER /* Visual */ # pragma warning(disable : 4204) /* non-constant aggregate initializer */ #endif -#if defined(__MINGW32__) && !defined(_POSIX_SOURCE) -# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ -#endif /*-************************************* diff --git a/programs/platform.h b/programs/platform.h index b8911f2c..02ebd4c0 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -1,11 +1,21 @@ -/** - * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - */ +/* + platform.h - compiler and OS detection + Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ #ifndef PLATFORM_H_MODULE #define PLATFORM_H_MODULE @@ -32,6 +42,9 @@ extern "C" { # define fseek _fseeki64 # endif #endif +#if defined(__MINGW32__) && !defined(_POSIX_SOURCE) +# define _POSIX_C_SOURCE 1 /* enable __VA_ARGS__ and disable %llu warnings with MinGW on Windows */ +#endif /* ************************************** From 101df4f6360dd97e8a2ef96510c15200d9af77bd Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 11:43:11 +0100 Subject: [PATCH 22/28] fixed Visual Studio compilation --- programs/platform.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/programs/platform.h b/programs/platform.h index 02ebd4c0..fe4bfded 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -38,9 +38,6 @@ extern "C" { # if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ # define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ # endif -# if (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ -# define fseek _fseeki64 -# endif #endif #if defined(__MINGW32__) && !defined(_POSIX_SOURCE) # define _POSIX_C_SOURCE 1 /* enable __VA_ARGS__ and disable %llu warnings with MinGW on Windows */ From 2f6ccee6af2cdf539c5798566dd7c27fc167673f Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 13:23:34 +0100 Subject: [PATCH 23/28] platform.h: removed Compiler Options --- lib/common/mem.h | 3 -- programs/bench.c | 15 ++++++-- programs/datagen.c | 12 +++++-- programs/dibio.c | 14 ++++++-- programs/fileio.c | 9 +++-- programs/platform.h | 20 +---------- programs/util.h | 58 ++++++++++++------------------- programs/zstdcli.c | 2 +- tests/datagencli.c | 3 +- tests/fullbench.c | 4 +-- tests/fuzzer.c | 5 +-- tests/paramgrill.c | 5 ++- tests/zbufftest.c | 5 +-- tests/zstreamtest.c | 5 +-- zlibWrapper/examples/zwrapbench.c | 4 +-- 15 files changed, 83 insertions(+), 81 deletions(-) diff --git a/lib/common/mem.h b/lib/common/mem.h index fb73ef09..32c63dd1 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -46,8 +46,6 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size /*-************************************************************** * Basic Types *****************************************************************/ -#ifndef BASIC_TYPES_DEFINED -#define BASIC_TYPES_DEFINED #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) # include typedef uint8_t BYTE; @@ -68,7 +66,6 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size typedef signed long long S64; typedef ptrdiff_t iPtrDiff; #endif -#endif /*-************************************************************** diff --git a/programs/bench.c b/programs/bench.c index 2b7436a4..9b7f9888 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -8,16 +8,27 @@ */ + +/* ************************************** +* Compiler Warnings +****************************************/ +#ifdef _MSC_VER +# define _CRT_SECURE_NO_WARNINGS /* fopen */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + /* ************************************* * Includes ***************************************/ -#include "platform.h" /* Compiler options */ -#include "util.h" /* UTIL_GetFileSize, UTIL_sleep */ +#include "platform.h" /* Large Files support */ +#include "util.h" /* UTIL_getFileSize, UTIL_sleep */ #include /* malloc, free */ #include /* memset */ #include /* fprintf, fopen, ftello64 */ #include /* clock_t, clock, CLOCKS_PER_SEC */ +#include "mem.h" #define ZSTD_STATIC_LINKING_ONLY #include "zstd.h" #include "datagen.h" /* RDG_genBuffer */ diff --git a/programs/datagen.c b/programs/datagen.c index 8209c851..06b5ab97 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -9,14 +9,22 @@ +/* ************************************* +* Compiler Options +***************************************/ +#if defined(_MSC_VER) +# define _CRT_SECURE_NO_WARNINGS /* removes Visual warning on strerror() */ +# define _CRT_SECURE_NO_DEPRECATE /* removes VS2005 warning on strerror() */ +#endif + /*-************************************ * Dependencies **************************************/ -#include "platform.h" /* Compiler options */ -#include "util.h" /* U32 */ +#include "platform.h" /* SET_BINARY_MODE */ #include /* malloc, free */ #include /* FILE, fwrite, fprintf */ #include /* memcpy */ +#include "mem.h" /* U32 */ /*-************************************ diff --git a/programs/dibio.c b/programs/dibio.c index 1c22949f..b99e717c 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -9,17 +9,27 @@ +/* ************************************** +* Compiler Warnings +****************************************/ +#ifdef _MSC_VER +# define _CRT_SECURE_NO_WARNINGS /* fopen */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + /*-************************************* * Includes ***************************************/ -#include "platform.h" /* Compiler options */ -#include "util.h" /* UTIL_GetFileSize, UTIL_getTotalFileSize */ +#include "platform.h" /* Large Files support */ +#include "util.h" /* UTIL_getFileSize, UTIL_getTotalFileSize */ #include /* malloc, free */ #include /* memset */ #include /* fprintf, fopen, ftello64 */ #include /* clock_t, clock, CLOCKS_PER_SEC */ #include /* errno */ +#include "mem.h" /* read */ #include "error_private.h" #include "dibio.h" diff --git a/programs/fileio.c b/programs/fileio.c index fa52f9af..e2a90a67 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -11,21 +11,26 @@ * Compiler Options ***************************************/ #ifdef _MSC_VER /* Visual */ +# define _CRT_SECURE_NO_WARNINGS /* removes Visual warning on strerror() */ # pragma warning(disable : 4204) /* non-constant aggregate initializer */ #endif +#if defined(__MINGW32__) && !defined(_POSIX_SOURCE) +# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ +#endif /*-************************************* * Includes ***************************************/ -#include "platform.h" /* Compiler options */ -#include "util.h" /* UTIL_GetFileSize, _LARGEFILE64_SOURCE */ +#include "platform.h" /* Large Files support, SET_BINARY_MODE */ +#include "util.h" /* UTIL_getFileSize */ #include /* fprintf, fopen, fread, _fileno, stdin, stdout */ #include /* malloc, free */ #include /* strcmp, strlen */ #include /* clock */ #include /* errno */ +#include "mem.h" #include "fileio.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ #include "zstd.h" diff --git a/programs/platform.h b/programs/platform.h index fe4bfded..0b82e46b 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -25,25 +25,6 @@ extern "C" { #endif -/* ************************************** -* Compiler Options -****************************************/ -#if defined(__INTEL_COMPILER) -# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced */ -#endif -#if defined(_MSC_VER) -# define _CRT_SECURE_NO_WARNINGS /* Disable some Visual warning messages for fopen, strncpy */ -# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ -# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ -# endif -#endif -#if defined(__MINGW32__) && !defined(_POSIX_SOURCE) -# define _POSIX_C_SOURCE 1 /* enable __VA_ARGS__ and disable %llu warnings with MinGW on Windows */ -#endif - - /* ************************************** * Detect 64-bit OS * http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros @@ -110,6 +91,7 @@ extern "C" { * Detect if isatty() and fileno() are available ************************************************/ #if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__) +# include /* isatty */ # define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) #elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) # include /* _isatty */ diff --git a/programs/util.h b/programs/util.h index 409f6d7c..8b480b22 100644 --- a/programs/util.h +++ b/programs/util.h @@ -25,24 +25,35 @@ extern "C" { #endif +/* ************************************** +* Compiler Options +****************************************/ +#if defined(_MSC_VER) +# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ +# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ +# endif +#endif + + /*-**************************************** * Dependencies ******************************************/ -#include "platform.h" /* Compiler options, PLATFORM_POSIX_VERSION */ -#include /* malloc */ -#include /* size_t, ptrdiff_t */ -#include /* fprintf */ -#include /* stat, utime */ -#include /* stat */ +#include "platform.h" /* PLATFORM_POSIX_VERSION */ +#include /* malloc */ +#include /* size_t, ptrdiff_t */ +#include /* fprintf */ +#include /* stat, utime */ +#include /* stat */ #if defined(_MSC_VER) -# include /* utime */ -# include /* _chmod */ +# include /* utime */ +# include /* _chmod */ #else # include /* chown, stat */ # include /* utime */ #endif -#include /* time */ +#include /* time */ #include +#include "mem.h" /* U32, U64 */ /*-**************************************** @@ -81,35 +92,12 @@ extern "C" { #define LIST_SIZE_INCREASE (8*1024) -/*-************************************************************** -* Basic Types -*****************************************************************/ -#ifndef BASIC_TYPES_DEFINED -#define BASIC_TYPES_DEFINED -#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef int16_t S16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; - typedef int64_t S64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef signed short S16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; - typedef signed long long S64; -#endif -#endif - - /*-**************************************** * Compiler specifics ******************************************/ +#if defined(__INTEL_COMPILER) +# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced, useful with UTIL_STATIC */ +#endif #if defined(__GNUC__) # define UTIL_STATIC static __attribute__((unused)) #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 29fcb746..978ffcfe 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -23,7 +23,7 @@ /*-************************************ * Dependencies **************************************/ -#include "platform.h" /* Compiler options, PLATFORM_POSIX_VERSION */ +#include "platform.h" /* IS_CONSOLE, PLATFORM_POSIX_VERSION */ #include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */ #include /* strcmp, strlen */ #include /* errno */ diff --git a/tests/datagencli.c b/tests/datagencli.c index c4bf4893..2f3ebc4d 100644 --- a/tests/datagencli.c +++ b/tests/datagencli.c @@ -11,8 +11,7 @@ /*-************************************ * Dependencies **************************************/ -#include "platform.h" /* Compiler options */ -#include "util.h" /* U32 */ +#include "util.h" /* Compiler options */ #include /* fprintf, stderr */ #include "datagen.h" /* RDG_generate */ diff --git a/tests/fullbench.c b/tests/fullbench.c index 439de5c7..233b4e93 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -11,12 +11,12 @@ /*_************************************ * Includes **************************************/ -#include "platform.h" /* Compiler options */ -#include "util.h" /* U32 */ +#include "util.h" /* Compiler options, UTIL_GetFileSize */ #include /* malloc */ #include /* fprintf, fopen, ftello64 */ #include /* clock_t, clock, CLOCKS_PER_SEC */ +#include "mem.h" #ifndef ZSTD_DLL_IMPORT #include "zstd_internal.h" /* ZSTD_blockHeaderSize, blockType_e, KB, MB */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressBegin, ZSTD_compressContinue, etc. */ diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 9088e248..86d4c6be 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -12,6 +12,8 @@ * Compiler specific **************************************/ #ifdef _MSC_VER /* Visual Studio */ +# define _CRT_SECURE_NO_WARNINGS /* fgets */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ #endif @@ -19,8 +21,6 @@ /*-************************************ * Includes **************************************/ -#include "platform.h" /* Compiler options */ -#include "util.h" /* U32 */ #include /* free */ #include /* fgets, sscanf */ #include /* strcmp */ @@ -30,6 +30,7 @@ #include "zstd_errors.h" /* ZSTD_getErrorCode */ #include "zdict.h" /* ZDICT_trainFromBuffer */ #include "datagen.h" /* RDG_genBuffer */ +#include "mem.h" #define XXH_STATIC_LINKING_ONLY #include "xxhash.h" /* XXH64 */ diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 0825b37a..5eabcba2 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -11,15 +11,14 @@ /*-************************************ * Dependencies **************************************/ -#include "platform.h" /* Compiler options */ -#include "util.h" /* UTIL_getFileSize */ +#include "util.h" /* Compiler options, UTIL_GetFileSize */ #include /* malloc */ #include /* fprintf, fopen, ftello64 */ #include /* strcmp */ #include /* log */ #include /* clock_t */ -#include "mem.h" /* MEM_32bits() */ +#include "mem.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_estimateCCtxSize */ #include "zstd.h" #include "datagen.h" diff --git a/tests/zbufftest.c b/tests/zbufftest.c index 87cf80b5..14b73923 100644 --- a/tests/zbufftest.c +++ b/tests/zbufftest.c @@ -12,6 +12,8 @@ * Compiler specific **************************************/ #ifdef _MSC_VER /* Visual Studio */ +# define _CRT_SECURE_NO_WARNINGS /* fgets */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ #endif @@ -19,12 +21,11 @@ /*-************************************ * Includes **************************************/ -#include "platform.h" /* Compiler options */ #include /* free */ #include /* fgets, sscanf */ #include /* clock_t, clock() */ #include /* strcmp */ -#include "mem.h" /* MEM_writeLE32 */ +#include "mem.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */ #include "zstd.h" /* ZSTD_compressBound */ #define ZBUFF_STATIC_LINKING_ONLY /* ZBUFF_createCCtx_advanced */ diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 9da0010b..ce619308 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -12,6 +12,8 @@ * Compiler specific **************************************/ #ifdef _MSC_VER /* Visual Studio */ +# define _CRT_SECURE_NO_WARNINGS /* fgets */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ #endif @@ -19,12 +21,11 @@ /*-************************************ * Includes **************************************/ -#include "platform.h" /* Compiler options */ #include /* free */ #include /* fgets, sscanf */ #include /* clock_t, clock() */ #include /* strcmp */ -#include "mem.h" /* MEM_writeLE32 */ +#include "mem.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel, ZSTD_customMem */ #include "zstd.h" /* ZSTD_compressBound */ #include "zstd_errors.h" /* ZSTD_error_srcSize_wrong */ diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index 8999657f..e0aca001 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -11,14 +11,14 @@ /* ************************************* * Includes ***************************************/ -#include "platform.h" /* Compiler options */ -#include "util.h" /* U32, UTIL_GetFileSize, UTIL_sleep */ +#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_sleep */ #include /* malloc, free */ #include /* memset */ #include /* fprintf, fopen, ftello64 */ #include /* clock_t, clock, CLOCKS_PER_SEC */ #include /* toupper */ +#include "mem.h" #define ZSTD_STATIC_LINKING_ONLY #include "zstd.h" #include "datagen.h" /* RDG_genBuffer */ From e679741b188a29631aabd226f3653ed90552d78c Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 13:47:11 +0100 Subject: [PATCH 24/28] _CRT_SECURE_NO_WARNINGS moved to util.h --- programs/bench.c | 2 -- programs/datagen.c | 8 -------- programs/dibio.c | 2 -- programs/fileio.c | 3 +-- programs/util.h | 2 ++ programs/zstdcli.c | 1 - tests/fuzzer.c | 1 - tests/zbufftest.c | 1 - tests/zstreamtest.c | 1 - 9 files changed, 3 insertions(+), 18 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 9b7f9888..5d142529 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -13,7 +13,6 @@ * Compiler Warnings ****************************************/ #ifdef _MSC_VER -# define _CRT_SECURE_NO_WARNINGS /* fopen */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #endif @@ -21,7 +20,6 @@ /* ************************************* * Includes ***************************************/ -#include "platform.h" /* Large Files support */ #include "util.h" /* UTIL_getFileSize, UTIL_sleep */ #include /* malloc, free */ #include /* memset */ diff --git a/programs/datagen.c b/programs/datagen.c index 06b5ab97..d0116b97 100644 --- a/programs/datagen.c +++ b/programs/datagen.c @@ -9,14 +9,6 @@ -/* ************************************* -* Compiler Options -***************************************/ -#if defined(_MSC_VER) -# define _CRT_SECURE_NO_WARNINGS /* removes Visual warning on strerror() */ -# define _CRT_SECURE_NO_DEPRECATE /* removes VS2005 warning on strerror() */ -#endif - /*-************************************ * Dependencies **************************************/ diff --git a/programs/dibio.c b/programs/dibio.c index b99e717c..743d3ef9 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -13,7 +13,6 @@ * Compiler Warnings ****************************************/ #ifdef _MSC_VER -# define _CRT_SECURE_NO_WARNINGS /* fopen */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #endif @@ -21,7 +20,6 @@ /*-************************************* * Includes ***************************************/ -#include "platform.h" /* Large Files support */ #include "util.h" /* UTIL_getFileSize, UTIL_getTotalFileSize */ #include /* malloc, free */ #include /* memset */ diff --git a/programs/fileio.c b/programs/fileio.c index e2a90a67..5544fe30 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -11,7 +11,7 @@ * Compiler Options ***************************************/ #ifdef _MSC_VER /* Visual */ -# define _CRT_SECURE_NO_WARNINGS /* removes Visual warning on strerror() */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4204) /* non-constant aggregate initializer */ #endif #if defined(__MINGW32__) && !defined(_POSIX_SOURCE) @@ -22,7 +22,6 @@ /*-************************************* * Includes ***************************************/ -#include "platform.h" /* Large Files support, SET_BINARY_MODE */ #include "util.h" /* UTIL_getFileSize */ #include /* fprintf, fopen, fread, _fileno, stdin, stdout */ #include /* malloc, free */ diff --git a/programs/util.h b/programs/util.h index 8b480b22..5779fa77 100644 --- a/programs/util.h +++ b/programs/util.h @@ -29,6 +29,8 @@ extern "C" { * Compiler Options ****************************************/ #if defined(_MSC_VER) +# define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */ +# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ # if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ # define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ # endif diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 978ffcfe..20698c78 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -23,7 +23,6 @@ /*-************************************ * Dependencies **************************************/ -#include "platform.h" /* IS_CONSOLE, PLATFORM_POSIX_VERSION */ #include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */ #include /* strcmp, strlen */ #include /* errno */ diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 86d4c6be..b9b0158e 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -12,7 +12,6 @@ * Compiler specific **************************************/ #ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS /* fgets */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ #endif diff --git a/tests/zbufftest.c b/tests/zbufftest.c index 14b73923..0fda993f 100644 --- a/tests/zbufftest.c +++ b/tests/zbufftest.c @@ -12,7 +12,6 @@ * Compiler specific **************************************/ #ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS /* fgets */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ #endif diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index ce619308..b5284504 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -12,7 +12,6 @@ * Compiler specific **************************************/ #ifdef _MSC_VER /* Visual Studio */ -# define _CRT_SECURE_NO_WARNINGS /* fgets */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ #endif From 97a258d71ddf6c7a7723b680f356bc9721d2e0a8 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 14:00:41 +0100 Subject: [PATCH 25/28] updated comments --- programs/fileio.c | 2 +- programs/zstdcli.c | 2 +- tests/fuzzer.c | 1 + tests/zbufftest.c | 1 + tests/zstreamtest.c | 1 + 5 files changed, 5 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 5544fe30..8305fbd7 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -22,7 +22,7 @@ /*-************************************* * Includes ***************************************/ -#include "util.h" /* UTIL_getFileSize */ +#include "util.h" /* SET_BINARY_MODE, UTIL_getFileSize */ #include /* fprintf, fopen, fread, _fileno, stdin, stdout */ #include /* malloc, free */ #include /* strcmp, strlen */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 20698c78..0b6f9241 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -23,7 +23,7 @@ /*-************************************ * Dependencies **************************************/ -#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */ +#include "util.h" /* IS_CONSOLE, UTIL_HAS_CREATEFILELIST, UTIL_createFileList */ #include /* strcmp, strlen */ #include /* errno */ #include "fileio.h" diff --git a/tests/fuzzer.c b/tests/fuzzer.c index b9b0158e..86d4c6be 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -12,6 +12,7 @@ * Compiler specific **************************************/ #ifdef _MSC_VER /* Visual Studio */ +# define _CRT_SECURE_NO_WARNINGS /* fgets */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ #endif diff --git a/tests/zbufftest.c b/tests/zbufftest.c index 0fda993f..14b73923 100644 --- a/tests/zbufftest.c +++ b/tests/zbufftest.c @@ -12,6 +12,7 @@ * Compiler specific **************************************/ #ifdef _MSC_VER /* Visual Studio */ +# define _CRT_SECURE_NO_WARNINGS /* fgets */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ #endif diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index b5284504..ce619308 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -12,6 +12,7 @@ * Compiler specific **************************************/ #ifdef _MSC_VER /* Visual Studio */ +# define _CRT_SECURE_NO_WARNINGS /* fgets */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ #endif From 7a8a03c20d71ede3896a6e3ed9820baae5b0ccf9 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 21 Dec 2016 15:08:44 +0100 Subject: [PATCH 26/28] util.h: restore BSD license for Facebook Open-Source --- programs/bench.c | 1 + programs/dibio.c | 1 + programs/fileio.c | 3 ++- programs/platform.h | 41 +++++++++++++++++++++++------------------ programs/util.h | 39 ++++++++++----------------------------- programs/zstdcli.c | 3 ++- 6 files changed, 39 insertions(+), 49 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 5d142529..4089d6ba 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -20,6 +20,7 @@ /* ************************************* * Includes ***************************************/ +#include "platform.h" /* Large Files support */ #include "util.h" /* UTIL_getFileSize, UTIL_sleep */ #include /* malloc, free */ #include /* memset */ diff --git a/programs/dibio.c b/programs/dibio.c index 743d3ef9..b95bab34 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -20,6 +20,7 @@ /*-************************************* * Includes ***************************************/ +#include "platform.h" /* Large Files support */ #include "util.h" /* UTIL_getFileSize, UTIL_getTotalFileSize */ #include /* malloc, free */ #include /* memset */ diff --git a/programs/fileio.c b/programs/fileio.c index 8305fbd7..a112cc04 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -22,7 +22,8 @@ /*-************************************* * Includes ***************************************/ -#include "util.h" /* SET_BINARY_MODE, UTIL_getFileSize */ +#include "platform.h" /* Large Files support, SET_BINARY_MODE */ +#include "util.h" /* UTIL_getFileSize */ #include /* fprintf, fopen, fread, _fileno, stdin, stdout */ #include /* malloc, free */ #include /* strcmp, strlen */ diff --git a/programs/platform.h b/programs/platform.h index 0b82e46b..f30528aa 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -1,21 +1,13 @@ -/* - platform.h - compiler and OS detection - Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -*/ +/** + * platform.h - compiler and OS detection + * + * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ #ifndef PLATFORM_H_MODULE #define PLATFORM_H_MODULE @@ -25,6 +17,19 @@ extern "C" { #endif + +/* ************************************** +* Compiler Options +****************************************/ +#if defined(_MSC_VER) +# define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */ +# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before and */ +# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ +# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ +# endif +#endif + + /* ************************************** * Detect 64-bit OS * http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros diff --git a/programs/util.h b/programs/util.h index 5779fa77..aaa4b7c1 100644 --- a/programs/util.h +++ b/programs/util.h @@ -1,21 +1,13 @@ -/* - util.h - utility functions - Copyright (C) 2016-present, Przemyslaw Skibinski, Yann Collet - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -*/ +/** + * util.h - utility functions + * + * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ #ifndef UTIL_H_MODULE #define UTIL_H_MODULE @@ -25,17 +17,6 @@ extern "C" { #endif -/* ************************************** -* Compiler Options -****************************************/ -#if defined(_MSC_VER) -# define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */ -# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */ -# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ -# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ -# endif -#endif - /*-**************************************** * Dependencies diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 0b6f9241..978ffcfe 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -23,7 +23,8 @@ /*-************************************ * Dependencies **************************************/ -#include "util.h" /* IS_CONSOLE, UTIL_HAS_CREATEFILELIST, UTIL_createFileList */ +#include "platform.h" /* IS_CONSOLE, PLATFORM_POSIX_VERSION */ +#include "util.h" /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */ #include /* strcmp, strlen */ #include /* errno */ #include "fileio.h" From 1f57c2ed32369b491ae1c47774d14ec29e199dd0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Dec 2016 16:20:11 +0100 Subject: [PATCH 27/28] added : ZSTD_createCDict_byReference() --- Makefile | 3 ++ lib/compress/zstd_compress.c | 48 +++++++++++++++++++++---------- lib/zstd.h | 12 ++++++-- programs/bench.c | 6 ++-- zlibWrapper/examples/zwrapbench.c | 2 +- 5 files changed, 49 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index bb3a4e4c..7d9ff4f8 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,8 @@ all: $(MAKE) -C $(ZSTDDIR) $@ $(MAKE) -C $(PRGDIR) $@ zstd32 $(MAKE) -C $(TESTDIR) $@ all32 + $(MAKE) -C $(ZWRAPDIR) $@ + CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ $@ .PHONY: lib lib: @@ -54,6 +56,7 @@ clean: @$(MAKE) -C $(PRGDIR) $@ > $(VOID) @$(MAKE) -C $(TESTDIR) $@ > $(VOID) @$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID) + @$(MAKE) -C examples/ $@ > $(VOID) @$(RM) zstd$(EXT) tmp* @echo Cleaning completed diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index e88f6a1d..2a9ddd66 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2733,8 +2733,10 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS /* ===== Dictionary API ===== */ struct ZSTD_CDict_s { - void* dictContent; + void* dictBuffer; + const void* dictContent; size_t dictContentSize; + unsigned byReference; ZSTD_CCtx* refContext; }; /* typedef'd tp ZSTD_CDict within "zstd.h" */ @@ -2744,36 +2746,44 @@ size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) return ZSTD_sizeof_CCtx(cdict->refContext) + cdict->dictContentSize; } -ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem) +ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, unsigned byReference, + ZSTD_parameters params, ZSTD_customMem customMem) { if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; if (!customMem.customAlloc || !customMem.customFree) return NULL; { ZSTD_CDict* const cdict = (ZSTD_CDict*) ZSTD_malloc(sizeof(ZSTD_CDict), customMem); - void* const dictContent = ZSTD_malloc(dictSize, customMem); ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem); - if (!dictContent || !cdict || !cctx) { - ZSTD_free(dictContent, customMem); + if (!cdict || !cctx) { ZSTD_free(cdict, customMem); ZSTD_free(cctx, customMem); return NULL; } - if (dictSize) { - memcpy(dictContent, dict, dictSize); + if ((byReference) || (!dictBuffer) || (!dictSize)) { + cdict->dictBuffer = NULL; + cdict->dictContent = dictBuffer; + cdict->byReference = 1; + } else { + void* const internalBuffer = ZSTD_malloc(dictSize, customMem); + if (!internalBuffer) return NULL; + memcpy(internalBuffer, dictBuffer, dictSize); + cdict->dictBuffer = internalBuffer; + cdict->dictContent = internalBuffer; + cdict->byReference = 0; } - { size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0); + + { size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0); if (ZSTD_isError(errorCode)) { - ZSTD_free(dictContent, customMem); - ZSTD_free(cdict, customMem); + ZSTD_free(cdict->dictBuffer, customMem); ZSTD_free(cctx, customMem); + ZSTD_free(cdict, customMem); return NULL; } } - cdict->dictContent = dictContent; - cdict->dictContentSize = dictSize; cdict->refContext = cctx; + cdict->dictContentSize = dictSize; return cdict; } } @@ -2783,7 +2793,15 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL ZSTD_customMem const allocator = { NULL, NULL, NULL }; ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize); params.fParams.contentSizeFlag = 1; - return ZSTD_createCDict_advanced(dict, dictSize, params, allocator); + return ZSTD_createCDict_advanced(dict, dictSize, 0, params, allocator); +} + +ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize); + params.fParams.contentSizeFlag = 1; + return ZSTD_createCDict_advanced(dict, dictSize, 1, params, allocator); } size_t ZSTD_freeCDict(ZSTD_CDict* cdict) @@ -2791,7 +2809,7 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) if (cdict==NULL) return 0; /* support free on NULL */ { ZSTD_customMem const cMem = cdict->refContext->customMem; ZSTD_freeCCtx(cdict->refContext); - ZSTD_free(cdict->dictContent, cMem); + if (!cdict->byReference) ZSTD_free(cdict->dictBuffer, cMem); ZSTD_free(cdict, cMem); return 0; } @@ -2939,7 +2957,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, if (dict) { ZSTD_freeCDict(zcs->cdictLocal); - zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem); + zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0, params, zcs->customMem); if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); zcs->cdict = zcs->cdictLocal; } else zcs->cdict = NULL; diff --git a/lib/zstd.h b/lib/zstd.h index 7eda6987..20682b49 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -173,8 +173,8 @@ typedef struct ZSTD_CDict_s ZSTD_CDict; * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. * ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only. -* `dict` can be released after ZSTD_CDict creation. */ -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel); +* `dictBuffer` can be released after ZSTD_CDict creation, as its content is copied within CDict */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel); /*! ZSTD_freeCDict() : * Function frees memory allocated by ZSTD_createCDict(). */ @@ -400,9 +400,15 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); * Gives the amount of memory used by a given ZSTD_CCtx */ ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); +/*! ZSTD_createCDict_byReference() : + * Create a digested dictionary for compression + * Dictionary content is simply referenced, and therefore stays in dictBuffer. + * It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); + /*! ZSTD_createCDict_advanced() : * Create a ZSTD_CDict using external alloc and free, and customized compression parameters */ -ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference, ZSTD_parameters params, ZSTD_customMem customMem); /*! ZSTD_sizeof_CDict() : diff --git a/programs/bench.c b/programs/bench.c index 6c2383a8..9eac10b7 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -236,7 +236,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, ZSTD_customMem const cmem = { NULL, NULL, NULL }; U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1; U32 nbLoops = 0; - ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem); + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1, zparams, cmem); if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure"); if (comprParams->windowLog) zparams.cParams.windowLog = comprParams->windowLog; if (comprParams->chainLog) zparams.cParams.chainLog = comprParams->chainLog; @@ -452,7 +452,7 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, if (totalSize == 0) EXM_THROW(12, "no data to bench"); } -static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, +static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel, int cLevelLast, ZSTD_compressionParameters *compressionParams) { void* srcBuffer; @@ -523,7 +523,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility } -int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, +int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams) { double const compressibility = (double)g_compressibilityDefault / 100; diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index e0aca001..49a2632e 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -234,7 +234,7 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize, if (compressor == BMK_ZSTD) { ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize); ZSTD_customMem const cmem = { NULL, NULL, NULL }; - ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem); + ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, 1, zparams, cmem); if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure"); do { From 4e5eea61a8646a985b9f1fc7421178ef8c59182d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 21 Dec 2016 16:44:35 +0100 Subject: [PATCH 28/28] added ZSTD_createDDict_byReference() --- NEWS | 1 + lib/compress/zstd_compress.c | 7 ++----- lib/decompress/zstd_decompress.c | 35 ++++++++++++++++++-------------- lib/zstd.h | 15 +++++++++++--- 4 files changed, 35 insertions(+), 23 deletions(-) diff --git a/NEWS b/NEWS index 9a341781..5e927181 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ v1.1.3 cli : new : commands for advanced parameters, by Przemyslaw Skibinski API : fix : all symbols properly exposed in libzstd, by Nick Terrell +API : new : ZSTD_create?Dict_byReference(), requested by Bartosz Taudul v1.1.2 API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 2a9ddd66..afac869c 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2736,7 +2736,6 @@ struct ZSTD_CDict_s { void* dictBuffer; const void* dictContent; size_t dictContentSize; - unsigned byReference; ZSTD_CCtx* refContext; }; /* typedef'd tp ZSTD_CDict within "zstd.h" */ @@ -2764,14 +2763,12 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, u if ((byReference) || (!dictBuffer) || (!dictSize)) { cdict->dictBuffer = NULL; cdict->dictContent = dictBuffer; - cdict->byReference = 1; } else { void* const internalBuffer = ZSTD_malloc(dictSize, customMem); - if (!internalBuffer) return NULL; + if (!internalBuffer) { ZSTD_free(cctx, customMem); ZSTD_free(cdict, customMem); return NULL; } memcpy(internalBuffer, dictBuffer, dictSize); cdict->dictBuffer = internalBuffer; cdict->dictContent = internalBuffer; - cdict->byReference = 0; } { size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0); @@ -2809,7 +2806,7 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) if (cdict==NULL) return 0; /* support free on NULL */ { ZSTD_customMem const cMem = cdict->refContext->customMem; ZSTD_freeCCtx(cdict->refContext); - if (!cdict->byReference) ZSTD_free(cdict->dictBuffer, cMem); + ZSTD_free(cdict->dictBuffer, cMem); ZSTD_free(cdict, cMem); return 0; } diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 7addff8d..19e8287e 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1713,39 +1713,44 @@ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t /* ====== ZSTD_DDict ====== */ struct ZSTD_DDict_s { - void* dict; + void* dictBuffer; + const void* dictContent; size_t dictSize; ZSTD_DCtx* refContext; }; /* typedef'd to ZSTD_DDict within "zstd.h" */ -ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_customMem customMem) +ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem) { if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; if (!customMem.customAlloc || !customMem.customFree) return NULL; { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem); - void* const dictContent = ZSTD_malloc(dictSize, customMem); ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(customMem); - if (!dictContent || !ddict || !dctx) { - ZSTD_free(dictContent, customMem); + if (!ddict || !dctx) { ZSTD_free(ddict, customMem); ZSTD_free(dctx, customMem); return NULL; } - if (dictSize) { - memcpy(dictContent, dict, dictSize); + if ((byReference) || (!dict) || (!dictSize)) { + ddict->dictBuffer = NULL; + ddict->dictContent = dict; + } else { + void* const internalBuffer = ZSTD_malloc(dictSize, customMem); + if (!internalBuffer) { ZSTD_free(dctx, customMem); ZSTD_free(ddict, customMem); return NULL; } + memcpy(internalBuffer, dict, dictSize); + ddict->dictBuffer = internalBuffer; + ddict->dictContent = internalBuffer; } - { size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize); + { size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, ddict->dictContent, dictSize); if (ZSTD_isError(errorCode)) { - ZSTD_free(dictContent, customMem); + ZSTD_free(ddict->dictBuffer, customMem); ZSTD_free(ddict, customMem); ZSTD_free(dctx, customMem); return NULL; } } - ddict->dict = dictContent; ddict->dictSize = dictSize; ddict->refContext = dctx; return ddict; @@ -1758,7 +1763,7 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_cu ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) { ZSTD_customMem const allocator = { NULL, NULL, NULL }; - return ZSTD_createDDict_advanced(dict, dictSize, allocator); + return ZSTD_createDDict_advanced(dict, dictSize, 0, allocator); } size_t ZSTD_freeDDict(ZSTD_DDict* ddict) @@ -1766,7 +1771,7 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict) if (ddict==NULL) return 0; /* support free on NULL */ { ZSTD_customMem const cMem = ddict->refContext->customMem; ZSTD_freeDCtx(ddict->refContext); - ZSTD_free(ddict->dict, cMem); + ZSTD_free(ddict->dictBuffer, cMem); ZSTD_free(ddict, cMem); return 0; } @@ -1796,7 +1801,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) { if (ddict==NULL) return 0; - return ZSTD_getDictID_fromDict(ddict->dict, ddict->dictSize); + return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize); } /*! ZSTD_getDictID_fromFrame() : @@ -1827,7 +1832,7 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) - if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize); + if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dictContent, ddict->dictSize); #endif ZSTD_refDCtx(dctx, ddict->refContext); ZSTD_checkContinuity(dctx, dst); @@ -2007,7 +2012,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) { U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); if (legacyVersion) { - const void* const dict = zds->ddict ? zds->ddict->dict : NULL; + const void* const dict = zds->ddict ? zds->ddict->dictContent : NULL; size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0; CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion, dict, dictSize)); diff --git a/lib/zstd.h b/lib/zstd.h index 20682b49..187ee5c2 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -194,8 +194,8 @@ typedef struct ZSTD_DDict_s ZSTD_DDict; /*! ZSTD_createDDict() : * Create a digested dictionary, ready to start decompression operation without startup delay. -* `dict` can be released after creation. */ -ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize); +* dictBuffer can be released after DDict creation, as its content is copied inside DDict */ +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); /*! ZSTD_freeDDict() : * Function frees memory allocated with ZSTD_createDDict() */ @@ -328,7 +328,7 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output * ***************************************************************************************/ /* --- Constants ---*/ -#define ZSTD_MAGICNUMBER 0xFD2FB528 /* v0.8 */ +#define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */ #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U #define ZSTD_WINDOWLOG_MAX_32 25 @@ -464,6 +464,15 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); * Gives the amount of memory used by a given ZSTD_DCtx */ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * Dictionary content is simply referenced, and therefore stays in dictBuffer. + * It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict */ +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); + +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, + unsigned byReference, ZSTD_customMem customMem); + /*! ZSTD_sizeof_DDict() : * Gives the amount of memory used by a given ZSTD_DDict */ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);