From d872b64f522fdb9b25efc3cb13003c791ffa2c6e Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 2 Nov 2016 12:52:20 +0100 Subject: [PATCH 01/12] added UTIL_setModificationTime, UTIL_getModificationTime --- programs/util.h | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/programs/util.h b/programs/util.h index aabebe96..1bbe2a73 100644 --- a/programs/util.h +++ b/programs/util.h @@ -46,8 +46,10 @@ extern "C" { ******************************************/ #include /* features.h with _POSIX_C_SOURCE, malloc */ #include /* fprintf */ -#include /* stat */ +#include /* stat, utime */ #include /* stat */ +#include /* utime */ +#include /* time */ #include "mem.h" /* U32, U64 */ @@ -142,6 +144,36 @@ UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond) /*-**************************************** * File functions ******************************************/ +UTIL_STATIC int UTIL_setModificationTime(const char *filename, time_t modtime) +{ + struct utimbuf timebuf; + + if (modtime == 0) return -1; + + timebuf.actime = time(NULL); + timebuf.modtime = modtime; + + /* On success, zero is returned. On error, -1 is returned, and errno is set appropriately. */ + return utime(filename, &timebuf); +} + + +UTIL_STATIC time_t UTIL_getModificationTime(const char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); + if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ +#else + struct stat statbuf; + r = stat(infilename, &statbuf); + if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ +#endif + return statbuf.st_mtime; +} + + UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) { int r; From a42794df6170f45499f32896b12795dd4403521d Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 2 Nov 2016 13:08:39 +0100 Subject: [PATCH 02/12] preserve file modification time --- programs/fileio.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index c4c308e0..44b41221 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -388,14 +388,17 @@ static int FIO_compressFilename_dstFile(cRess_t ress, const char* dstFileName, const char* srcFileName) { int result; + time_t modtime = 0; ress.dstFile = FIO_openDstFile(dstFileName); if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ + if (strcmp (srcFileName, stdinmark)) modtime = UTIL_getModificationTime(srcFileName); result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName); if (fclose(ress.dstFile)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; } /* error closing dstFile */ if (result!=0) { if (remove(dstFileName)) EXM_THROW(1, "zstd: %s: %s", dstFileName, strerror(errno)); } /* remove operation artefact */ + else if (strcmp (dstFileName, stdoutmark)) UTIL_setModificationTime(dstFileName, modtime); return result; } @@ -720,16 +723,21 @@ static int FIO_decompressDstFile(dRess_t ress, const char* dstFileName, const char* srcFileName) { int result; + time_t modtime = 0; + ress.dstFile = FIO_openDstFile(dstFileName); if (ress.dstFile==0) return 1; + if (strcmp (srcFileName, stdinmark)) modtime = UTIL_getModificationTime(srcFileName); result = FIO_decompressSrcFile(ress, srcFileName); - if (fclose(ress.dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName); + if (fclose(ress.dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName); + if ( (result != 0) && strcmp(dstFileName, nulmark) /* special case : don't remove() /dev/null (#316) */ && remove(dstFileName) ) result=1; /* don't do anything special if remove() fails */ + else if (strcmp (dstFileName, stdoutmark)) UTIL_setModificationTime(dstFileName, modtime); return result; } From fcf22e3473aa763d5abd19344041cb3960daa2aa Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 2 Nov 2016 14:08:07 +0100 Subject: [PATCH 03/12] set permissions, access and modification times --- programs/fileio.c | 14 ++++++++------ programs/util.h | 42 +++++++++++++++++++++++++++--------------- 2 files changed, 35 insertions(+), 21 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 44b41221..3d31a07c 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -388,17 +388,18 @@ static int FIO_compressFilename_dstFile(cRess_t ress, const char* dstFileName, const char* srcFileName) { int result; - time_t modtime = 0; + stat_t statbuf; + int stat_result = 0; ress.dstFile = FIO_openDstFile(dstFileName); if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ - if (strcmp (srcFileName, stdinmark)) modtime = UTIL_getModificationTime(srcFileName); + if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf)) stat_result = 1; result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName); if (fclose(ress.dstFile)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; } /* error closing dstFile */ if (result!=0) { if (remove(dstFileName)) EXM_THROW(1, "zstd: %s: %s", dstFileName, strerror(errno)); } /* remove operation artefact */ - else if (strcmp (dstFileName, stdoutmark)) UTIL_setModificationTime(dstFileName, modtime); + else if (strcmp (dstFileName, stdoutmark) && stat_result) UTIL_setFileStat(dstFileName, &statbuf); return result; } @@ -723,12 +724,13 @@ static int FIO_decompressDstFile(dRess_t ress, const char* dstFileName, const char* srcFileName) { int result; - time_t modtime = 0; + stat_t statbuf; + int stat_result = 0; ress.dstFile = FIO_openDstFile(dstFileName); if (ress.dstFile==0) return 1; - if (strcmp (srcFileName, stdinmark)) modtime = UTIL_getModificationTime(srcFileName); + if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf)) stat_result = 1; result = FIO_decompressSrcFile(ress, srcFileName); if (fclose(ress.dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName); @@ -737,7 +739,7 @@ static int FIO_decompressDstFile(dRess_t ress, && strcmp(dstFileName, nulmark) /* special case : don't remove() /dev/null (#316) */ && remove(dstFileName) ) result=1; /* don't do anything special if remove() fails */ - else if (strcmp (dstFileName, stdoutmark)) UTIL_setModificationTime(dstFileName, modtime); + else if (strcmp (dstFileName, stdoutmark) && stat_result) UTIL_setFileStat(dstFileName, &statbuf); return result; } diff --git a/programs/util.h b/programs/util.h index 1bbe2a73..7d503d9d 100644 --- a/programs/util.h +++ b/programs/util.h @@ -50,6 +50,7 @@ extern "C" { #include /* stat */ #include /* utime */ #include /* time */ +#include /* chown, stat */ #include "mem.h" /* U32, U64 */ @@ -144,33 +145,44 @@ UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond) /*-**************************************** * File functions ******************************************/ -UTIL_STATIC int UTIL_setModificationTime(const char *filename, time_t modtime) -{ - struct utimbuf timebuf; +#if defined(_MSC_VER) + typedef struct _stat64 stat_t; +#else + typedef struct stat stat_t; +#endif - if (modtime == 0) return -1; + +UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf) +{ + int res = 0; + struct utimbuf timebuf; + +#if !defined(_WIN32) + res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */ +#endif + + res += chmod(filename, statbuf->st_mode & 07777); /* Copy file permissions */ timebuf.actime = time(NULL); - timebuf.modtime = modtime; + timebuf.modtime = statbuf->st_mtime; + res += utime(filename, &timebuf); /* set access and modification times */ - /* On success, zero is returned. On error, -1 is returned, and errno is set appropriately. */ - return utime(filename, &timebuf); + errno = 0; + return -res; /* number of errors is returned */ } -UTIL_STATIC time_t UTIL_getModificationTime(const char* infilename) +UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) { int r; #if defined(_MSC_VER) - struct _stat64 statbuf; - r = _stat64(infilename, &statbuf); - if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ + r = _stat64(infilename, statbuf); + if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */ #else - struct stat statbuf; - r = stat(infilename, &statbuf); - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ + r = stat(infilename, statbuf); + if (r || !S_ISREG(statbuf->st_mode)) return 0; /* No good... */ #endif - return statbuf.st_mtime; + return 1; } From b40884f43d316f7e2acef20cd7d53b752e31c8e8 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 3 Nov 2016 09:54:53 +0100 Subject: [PATCH 04/12] preserve file modification time for Visual C++ --- programs/util.h | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/programs/util.h b/programs/util.h index 7d503d9d..fd07c348 100644 --- a/programs/util.h +++ b/programs/util.h @@ -48,9 +48,15 @@ extern "C" { #include /* fprintf */ #include /* stat, utime */ #include /* stat */ -#include /* utime */ +#if defined(_MSC_VER) + #include /* utime */ + #include /* _chmod */ +#else + #include /* chown, stat */ + #include /* utime */ +#endif #include /* time */ -#include /* chown, stat */ +#include #include "mem.h" /* U32, U64 */ @@ -146,7 +152,8 @@ UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond) * File functions ******************************************/ #if defined(_MSC_VER) - typedef struct _stat64 stat_t; + #define chmod _chmod + typedef struct _stat64 stat_t; #else typedef struct stat stat_t; #endif @@ -157,16 +164,16 @@ UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf) int res = 0; struct utimbuf timebuf; + timebuf.actime = time(NULL); + timebuf.modtime = statbuf->st_mtime; + res += utime(filename, &timebuf); /* set access and modification times */ + #if !defined(_WIN32) res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */ #endif res += chmod(filename, statbuf->st_mode & 07777); /* Copy file permissions */ - timebuf.actime = time(NULL); - timebuf.modtime = statbuf->st_mtime; - res += utime(filename, &timebuf); /* set access and modification times */ - errno = 0; return -res; /* number of errors is returned */ } @@ -322,7 +329,6 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ ((defined(__unix__) || defined(__unix) || defined(__midipix__)) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) /* snprintf, opendir */ # define UTIL_HAS_CREATEFILELIST # include /* opendir, readdir */ -# include UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd) { From 26306fcacff61770cb03df720251abb364529c6f Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 3 Nov 2016 11:38:01 +0100 Subject: [PATCH 05/12] BMK_SetNbIterations renamed to BMK_SetNbSeconds --- programs/bench.c | 18 +++++++++--------- programs/bench.h | 2 +- programs/fileio.c | 4 ++-- programs/zstdcli.c | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index c477b268..7b6e2520 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -33,7 +33,7 @@ # define ZSTD_GIT_COMMIT_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_GIT_COMMIT) #endif -#define NBLOOPS 3 +#define NBSECONDS 3 #define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */ #define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */ #define COOLPERIOD_SEC 10 @@ -82,7 +82,7 @@ static clock_t g_time = 0; /* ************************************* * Benchmark Parameters ***************************************/ -static U32 g_nbIterations = NBLOOPS; +static U32 g_nbSeconds = NBSECONDS; static size_t g_blockSize = 0; int g_additionalParam = 0; @@ -90,10 +90,10 @@ void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; } void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; } -void BMK_SetNbIterations(unsigned nbLoops) +void BMK_SetNbSeconds(unsigned nbSeconds) { - g_nbIterations = nbLoops; - DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbIterations); + g_nbSeconds = nbSeconds; + DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbSeconds); } void BMK_SetBlockSize(size_t blockSize) @@ -175,7 +175,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, { U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL); U64 const crcOrig = XXH64(srcBuffer, srcSize, 0); UTIL_time_t coolTime; - U64 const maxTime = (g_nbIterations * TIMELOOP_MICROSEC) + 100; + U64 const maxTime = (g_nbSeconds * TIMELOOP_MICROSEC) + 100; U64 totalCTime=0, totalDTime=0; U32 cCompleted=0, dCompleted=0; # define NB_MARKS 4 @@ -188,7 +188,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, DISPLAYLEVEL(2, "\r%79s\r", ""); while (!cCompleted | !dCompleted) { UTIL_time_t clockStart; - U64 clockLoop = g_nbIterations ? TIMELOOP_MICROSEC : 1; + U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1; /* overheat protection */ if (UTIL_clockSpanMicro(coolTime, ticksPerSecond) > ACTIVEPERIOD_MICROSEC) { @@ -306,7 +306,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, break; } } /* CRC Checking */ #endif - } /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */ + } /* for (testNb = 1; testNb <= (g_nbSeconds + !g_nbSeconds); testNb++) */ if (g_displayLevel == 1) { double cSpeed = (double)srcSize / fastestC; @@ -361,7 +361,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, SET_HIGH_PRIORITY; if (g_displayLevel == 1 && !g_additionalParam) - DISPLAY("bench %s %s: input %u bytes, %u iterations, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10)); + DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10)); if (cLevelLast < cLevel) cLevelLast = cLevel; diff --git a/programs/bench.h b/programs/bench.h index 7350fd43..1e3e3812 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -17,7 +17,7 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel, int cLevelLast); /* Set Parameters */ -void BMK_SetNbIterations(unsigned nbLoops); +void BMK_SetNbSeconds(unsigned nbLoops); void BMK_SetBlockSize(size_t blockSize); void BMK_setAdditionalParam(int additionalParam); void BMK_setNotificationLevel(unsigned level); diff --git a/programs/fileio.c b/programs/fileio.c index 3d31a07c..4759a786 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -443,7 +443,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile SET_BINARY_MODE(stdout); for (u=0; u /* errno */ #include "fileio.h" #ifndef ZSTD_NOBENCH -# include "bench.h" /* BMK_benchFiles, BMK_SetNbIterations */ +# include "bench.h" /* BMK_benchFiles, BMK_SetNbSeconds */ #endif #ifndef ZSTD_NODICT # include "dibio.h" @@ -383,7 +383,7 @@ int main(int argCount, const char* argv[]) argument++; { U32 const iters = readU32FromChar(&argument); BMK_setNotificationLevel(displayLevel); - BMK_SetNbIterations(iters); + BMK_SetNbSeconds(iters); } break; From 3a415594b156d5d76e03f033aea1e67793513b1c Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 3 Nov 2016 12:59:20 +0100 Subject: [PATCH 06/12] fixed MinGW compilation --- Makefile | 12 +++++++----- lib/Makefile | 2 ++ programs/Makefile | 2 -- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 96f5a9ad..a5758734 100644 --- a/Makefile +++ b/Makefile @@ -14,10 +14,12 @@ ZWRAPDIR = zlibWrapper TESTDIR = tests # Define nul output -ifneq (,$(filter Windows%,$(OS))) -VOID = nul -else VOID = /dev/null + +ifneq (,$(filter Windows%,$(OS))) +EXT =.exe +else +EXT = endif .PHONY: default @@ -35,7 +37,7 @@ lib: zstd: @$(MAKE) -C $(PRGDIR) - cp $(PRGDIR)/zstd . + cp $(PRGDIR)/zstd$(EXT) . .PHONY: zlibwrapper zlibwrapper: @@ -51,7 +53,7 @@ clean: @$(MAKE) -C $(PRGDIR) $@ > $(VOID) @$(MAKE) -C $(TESTDIR) $@ > $(VOID) @$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID) - @$(RM) zstd + @$(RM) zstd$(EXT) @echo Cleaning completed diff --git a/lib/Makefile b/lib/Makefile index 04ebe26b..cfc3028e 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -73,9 +73,11 @@ $(LIBZSTD): LDFLAGS += -shared -fPIC $(LIBZSTD): $(ZSTD_FILES) @echo compiling dynamic library $(LIBVER) @$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@ +ifeq (,$(filter Windows%,$(OS))) @echo creating versioned links @ln -sf $@.$(SHARED_EXT_VER) libzstd.$(SHARED_EXT_MAJOR) @ln -sf $@.$(SHARED_EXT_VER) libzstd.$(SHARED_EXT) +endif libzstd : $(LIBZSTD) diff --git a/programs/Makefile b/programs/Makefile index 64aeb668..b72cebdb 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -56,7 +56,6 @@ endif # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) EXT =.exe -VOID = nul RES64_FILE = windres\zstd64.res RES32_FILE = windres\zstd32.res ifneq (,$(filter x86_64%,$(shell $(CC) -dumpmachine))) @@ -66,7 +65,6 @@ else endif else EXT = -VOID = /dev/null endif From 9adf7bfd8ae433b19aeea50ff0986e329412d3a0 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 3 Nov 2016 15:38:13 +0100 Subject: [PATCH 07/12] fixed MinGW compilation (2) --- tests/Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 80e11d6b..26ae6007 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -50,12 +50,11 @@ ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c # Define *.exe as extension for Windows systems ifneq (,$(filter Windows%,$(OS))) EXT =.exe -VOID = nul else EXT = -VOID = /dev/null endif +VOID = /dev/null ZBUFFTEST = -T2mn FUZZERTEST= -T5mn ZSTDRTTEST= --test-large-data From d007eb5f9fec9fad15e3d22c71de19339e0c2b9b Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 4 Nov 2016 11:20:58 +0100 Subject: [PATCH 08/12] fixed clang warnings in zlibWrapper --- zlibWrapper/Makefile | 1 + zlibWrapper/examples/zwrapbench.c | 14 +++++++------- zlibWrapper/zstd_zlibwrapper.c | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index b36a90d6..25857c94 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -19,6 +19,7 @@ CC ?= gcc CFLAGS ?= -O3 CFLAGS += $(LOC) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -std=gnu99 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef +CFLAGS += $(MOREFLAGS) LDFLAGS = $(LOC) RM = rm -f diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index d16fcfdd..5ef7e525 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -130,7 +130,7 @@ void BMK_SetBlockSize(size_t blockSize) **********************************************************/ typedef struct { - const char* srcPtr; + char* srcPtr; size_t srcSize; char* cPtr; size_t cRoom; @@ -145,7 +145,7 @@ typedef enum { BMK_ZSTD, BMK_ZSTD_STREAM, BMK_ZLIB, BMK_ZWRAP_ZLIB, BMK_ZWRAP_ZS #define MIN(a,b) ((a)<(b) ? (a) : (b)) #define MAX(a,b) ((a)>(b) ? (a) : (b)) -static int BMK_benchMem(const void* srcBuffer, size_t srcSize, +static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* displayName, int cLevel, const size_t* fileSizes, U32 nbFiles, const void* dictBuffer, size_t dictBufferSize, BMK_compressor compressor) @@ -171,7 +171,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, UTIL_initTimer(&ticksPerSecond); /* Init blockTable data */ - { const char* srcPtr = (const char*)srcBuffer; + { char* srcPtr = (char*)srcBuffer; char* cPtr = (char*)compressedBuffer; char* resPtr = (char*)resultBuffer; U32 fileNb; @@ -307,7 +307,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure"); if (ZWRAP_isUsingZSTDcompression()) useSetDict = 0; /* zstd doesn't require deflateSetDictionary after ZWRAP_deflateReset_keepDict */ } - def.next_in = (const void*) blockTable[blockNb].srcPtr; + def.next_in = (void*) blockTable[blockNb].srcPtr; def.avail_in = blockTable[blockNb].srcSize; def.total_in = 0; def.next_out = (void*) blockTable[blockNb].cPtr; @@ -338,7 +338,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, ret = deflateSetDictionary(&def, dictBuffer, dictBufferSize); if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure"); } - def.next_in = (const void*) blockTable[blockNb].srcPtr; + def.next_in = (void*) blockTable[blockNb].srcPtr; def.avail_in = blockTable[blockNb].srcSize; def.total_in = 0; def.next_out = (void*) blockTable[blockNb].cPtr; @@ -443,7 +443,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, else ret = inflateReset(&inf); if (ret != Z_OK) EXM_THROW(1, "inflateReset failure"); - inf.next_in = (const void*) blockTable[blockNb].cPtr; + inf.next_in = (void*) blockTable[blockNb].cPtr; inf.avail_in = blockTable[blockNb].cSize; inf.total_in = 0; inf.next_out = (void*) blockTable[blockNb].resPtr; @@ -475,7 +475,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, inf.opaque = Z_NULL; ret = inflateInit(&inf); if (ret != Z_OK) EXM_THROW(1, "inflateInit failure"); - inf.next_in = (const void*) blockTable[blockNb].cPtr; + inf.next_in = (void*) blockTable[blockNb].cPtr; inf.avail_in = blockTable[blockNb].cSize; inf.total_in = 0; inf.next_out = (void*) blockTable[blockNb].resPtr; diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 31e784a8..c653d749 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -85,7 +85,7 @@ typedef struct { ZSTD_outBuffer outBuffer; ZWRAP_state_t comprState; unsigned long long pledgedSrcSize; -} ZWRAP_CCtx; +} ZWRAP_CCtx __attribute__ ((aligned (4))); size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc) @@ -404,7 +404,7 @@ typedef struct { int windowBits; ZSTD_customMem customMem; z_stream allocFunc; /* copy of zalloc, zfree, opaque */ -} ZWRAP_DCtx; +} ZWRAP_DCtx __attribute__ ((aligned (4))); int ZWRAP_isUsingZSTDdecompression(z_streamp strm) From d0815583d9b5c648fa005ac961a688035e768f5a Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 4 Nov 2016 11:37:27 +0100 Subject: [PATCH 09/12] Changed stdinmark and stdoutmark --- programs/fileio.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/fileio.h b/programs/fileio.h index 60a7e0de..a740f35e 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -18,8 +18,8 @@ extern "C" { /* ************************************* * Special i/o constants **************************************/ -#define stdinmark "stdin" -#define stdoutmark "stdout" +#define stdinmark "/*stdin*\\" +#define stdoutmark "/*stdout*\\" #ifdef _WIN32 # define nulmark "nul" #else From 0694ae2c83fd02bcf488ebb0bbe5bc96dbbc099a Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 4 Nov 2016 16:05:28 +0100 Subject: [PATCH 10/12] typedef ZWRAP_CCtx internal_state --- zlibWrapper/zstd_zlibwrapper.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index c653d749..4098abd8 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -10,7 +10,8 @@ #include /* vsprintf */ #include /* va_list, for z_gzprintf */ -#include +#define NO_DUMMY_DECL +#include /* without #define Z_PREFIX */ #include "zstd_zlibwrapper.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_MAGICNUMBER */ #include "zstd.h" @@ -85,7 +86,10 @@ typedef struct { ZSTD_outBuffer outBuffer; ZWRAP_state_t comprState; unsigned long long pledgedSrcSize; -} ZWRAP_CCtx __attribute__ ((aligned (4))); +} ZWRAP_CCtx; + +typedef ZWRAP_CCtx internal_state; + size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc) @@ -404,7 +408,7 @@ typedef struct { int windowBits; ZSTD_customMem customMem; z_stream allocFunc; /* copy of zalloc, zfree, opaque */ -} ZWRAP_DCtx __attribute__ ((aligned (4))); +} ZWRAP_DCtx; int ZWRAP_isUsingZSTDdecompression(z_streamp strm) From 7e06e6ab1999f78e1ebaaf41f2abca612af89749 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 4 Nov 2016 16:50:39 +0100 Subject: [PATCH 11/12] updated Makefile for zlibWrapper --- zlibWrapper/Makefile | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index 25857c94..48dc1a5b 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -1,7 +1,7 @@ # Makefile for example of using zstd wrapper for zlib # # make - compiles examples -# make LOC=-DZWRAP_USE_ZSTD=1 - compiles examples with zstd compression turned on +# make MOREFLAGS=-DZWRAP_USE_ZSTD=1 - compiles examples with zstd compression turned on # make test - runs examples @@ -15,13 +15,10 @@ ZLIBWRAPPER_PATH = . EXAMPLE_PATH = examples PROGRAMS_PATH = ../programs TEST_FILE = ../doc/zstd_compression_format.md -CC ?= gcc -CFLAGS ?= -O3 -CFLAGS += $(LOC) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -std=gnu99 -CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -CFLAGS += $(MOREFLAGS) -LDFLAGS = $(LOC) -RM = rm -f + +CPPFLAGS = -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) +CFLAGS ?= $(MOREFLAGS) -O3 -std=gnu99 +CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wstrict-aliasing=1 -Wundef all: clean fitblk example zwrapbench @@ -49,8 +46,8 @@ valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench $(VALGRIND) ./zwrapbench -qb3B1K $(TEST_FILE) $(VALGRIND) ./zwrapbench -rqb1e5 ../lib ../programs ../tests -.c.o: - $(CC) $(CFLAGS) -c -o $@ $< +#.c.o: +# $(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $< example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY) $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY) $(ZLIB_LIBRARY) @@ -70,10 +67,10 @@ zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(EXAMPLE_PATH)/zwrapbench.o: $(EXAMPLE_PATH)/zwrapbench.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h - $(CC) $(CFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c + $(CC) $(CFLAGS) $(CPPFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h - $(CC) $(CFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c + $(CC) $(CFLAGS) $(CPPFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZSTDLIBDIR)/libzstd.a: $(MAKE) -C $(ZSTDLIBDIR) libzstd.a From 6cecb35f982935f097d3684ea5b34dd671d9fb14 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 4 Nov 2016 17:49:17 +0100 Subject: [PATCH 12/12] zwrapbench uses z_const --- zlibWrapper/examples/zwrapbench.c | 14 +++++++------- zlibWrapper/zstd_zlibwrapper.c | 1 + zlibWrapper/zstd_zlibwrapper.h | 6 ++---- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index 5ef7e525..ebf76e48 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -130,7 +130,7 @@ void BMK_SetBlockSize(size_t blockSize) **********************************************************/ typedef struct { - char* srcPtr; + z_const char* srcPtr; size_t srcSize; char* cPtr; size_t cRoom; @@ -145,7 +145,7 @@ typedef enum { BMK_ZSTD, BMK_ZSTD_STREAM, BMK_ZLIB, BMK_ZWRAP_ZLIB, BMK_ZWRAP_ZS #define MIN(a,b) ((a)<(b) ? (a) : (b)) #define MAX(a,b) ((a)>(b) ? (a) : (b)) -static int BMK_benchMem(void* srcBuffer, size_t srcSize, +static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize, const char* displayName, int cLevel, const size_t* fileSizes, U32 nbFiles, const void* dictBuffer, size_t dictBufferSize, BMK_compressor compressor) @@ -171,7 +171,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, UTIL_initTimer(&ticksPerSecond); /* Init blockTable data */ - { char* srcPtr = (char*)srcBuffer; + { z_const char* srcPtr = (z_const char*)srcBuffer; char* cPtr = (char*)compressedBuffer; char* resPtr = (char*)resultBuffer; U32 fileNb; @@ -307,7 +307,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure"); if (ZWRAP_isUsingZSTDcompression()) useSetDict = 0; /* zstd doesn't require deflateSetDictionary after ZWRAP_deflateReset_keepDict */ } - def.next_in = (void*) blockTable[blockNb].srcPtr; + def.next_in = (z_const void*) blockTable[blockNb].srcPtr; def.avail_in = blockTable[blockNb].srcSize; def.total_in = 0; def.next_out = (void*) blockTable[blockNb].cPtr; @@ -338,7 +338,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, ret = deflateSetDictionary(&def, dictBuffer, dictBufferSize); if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure"); } - def.next_in = (void*) blockTable[blockNb].srcPtr; + def.next_in = (z_const void*) blockTable[blockNb].srcPtr; def.avail_in = blockTable[blockNb].srcSize; def.total_in = 0; def.next_out = (void*) blockTable[blockNb].cPtr; @@ -443,7 +443,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, else ret = inflateReset(&inf); if (ret != Z_OK) EXM_THROW(1, "inflateReset failure"); - inf.next_in = (void*) blockTable[blockNb].cPtr; + inf.next_in = (z_const void*) blockTable[blockNb].cPtr; inf.avail_in = blockTable[blockNb].cSize; inf.total_in = 0; inf.next_out = (void*) blockTable[blockNb].resPtr; @@ -475,7 +475,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, inf.opaque = Z_NULL; ret = inflateInit(&inf); if (ret != Z_OK) EXM_THROW(1, "inflateInit failure"); - inf.next_in = (void*) blockTable[blockNb].cPtr; + inf.next_in = (z_const void*) blockTable[blockNb].cPtr; inf.avail_in = blockTable[blockNb].cSize; inf.total_in = 0; inf.next_out = (void*) blockTable[blockNb].resPtr; diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 4098abd8..326fea26 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -11,6 +11,7 @@ #include /* vsprintf */ #include /* va_list, for z_gzprintf */ #define NO_DUMMY_DECL +#define ZLIB_CONST #include /* without #define Z_PREFIX */ #include "zstd_zlibwrapper.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_MAGICNUMBER */ diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h index 87341390..45d15bac 100644 --- a/zlibWrapper/zstd_zlibwrapper.h +++ b/zlibWrapper/zstd_zlibwrapper.h @@ -15,16 +15,14 @@ extern "C" { #endif +#define ZLIB_CONST #define Z_PREFIX #include #if !defined(z_const) -#if ZLIB_VERNUM >= 0x1260 - #define z_const const -#else #define z_const #endif -#endif + /* returns a string with version of zstd library */ const char * zstdVersion(void);