Merge pull request #441 from inikep/dev11

preserve owner, file permission and modification time

Makefile

@@ -14,10 +14,12 @@ ZWRAPDIR = zlibWrapper
 TESTDIR  = tests
 
 # Define nul output
-ifneq (,$(filter Windows%,$(OS)))
-VOID = nul
-else
 VOID = /dev/null
+
+ifneq (,$(filter Windows%,$(OS)))
+EXT =.exe
+else
+EXT =
 endif
 
 .PHONY: default
@@ -35,7 +37,7 @@ lib:
 
 zstd:
 	@$(MAKE) -C $(PRGDIR)
-	cp $(PRGDIR)/zstd .
+	cp $(PRGDIR)/zstd$(EXT) .
 
 .PHONY: zlibwrapper
 zlibwrapper:
@@ -51,7 +53,7 @@ clean:
 	@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
 	@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
 	@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
-	@$(RM) zstd
+	@$(RM) zstd$(EXT)
 	@echo Cleaning completed
 
 

lib/Makefile

@@ -73,9 +73,11 @@ $(LIBZSTD): LDFLAGS += -shared -fPIC
 $(LIBZSTD): $(ZSTD_FILES)
 	@echo compiling dynamic library $(LIBVER)
 	@$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
+ifeq (,$(filter Windows%,$(OS)))
 	@echo creating versioned links
 	@ln -sf $(LIBZSTD) libzstd.$(SHARED_EXT_MAJOR)
 	@ln -sf $(LIBZSTD) libzstd.$(SHARED_EXT)
+endif
 
 libzstd : $(LIBZSTD)
 

programs/Makefile

@@ -56,7 +56,6 @@ endif
 # Define *.exe as extension for Windows systems
 ifneq (,$(filter Windows%,$(OS)))
 EXT =.exe
-VOID = nul
 RES64_FILE = windres\zstd64.res
 RES32_FILE = windres\zstd32.res
 ifneq (,$(filter x86_64%,$(shell $(CC) -dumpmachine)))
@@ -66,7 +65,6 @@ else
 endif
 else
 EXT =
-VOID = /dev/null
 endif
 
 

programs/bench.c

@@ -33,7 +33,7 @@
 #  define ZSTD_GIT_COMMIT_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_GIT_COMMIT)
 #endif
 
-#define NBLOOPS               3
+#define NBSECONDS             3
 #define TIMELOOP_MICROSEC     1*1000000ULL /* 1 second */
 #define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */
 #define COOLPERIOD_SEC        10
@@ -82,7 +82,7 @@ static clock_t g_time = 0;
 /* *************************************
 *  Benchmark Parameters
 ***************************************/
-static U32 g_nbIterations = NBLOOPS;
+static U32 g_nbSeconds = NBSECONDS;
 static size_t g_blockSize = 0;
 int g_additionalParam = 0;
 
@@ -90,10 +90,10 @@ void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
 
 void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
 
-void BMK_SetNbIterations(unsigned nbLoops)
+void BMK_SetNbSeconds(unsigned nbSeconds)
 {
-    g_nbIterations = nbLoops;
+    g_nbSeconds = nbSeconds;
-    DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbIterations);
+    DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbSeconds);
 }
 
 void BMK_SetBlockSize(size_t blockSize)
@@ -175,7 +175,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
     {   U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL);
         U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
         UTIL_time_t coolTime;
-        U64 const maxTime = (g_nbIterations * TIMELOOP_MICROSEC) + 100;
+        U64 const maxTime = (g_nbSeconds * TIMELOOP_MICROSEC) + 100;
         U64 totalCTime=0, totalDTime=0;
         U32 cCompleted=0, dCompleted=0;
 #       define NB_MARKS 4
@@ -188,7 +188,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
         DISPLAYLEVEL(2, "\r%79s\r", "");
         while (!cCompleted | !dCompleted) {
             UTIL_time_t clockStart;
-            U64 clockLoop = g_nbIterations ? TIMELOOP_MICROSEC : 1;
+            U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
 
             /* overheat protection */
             if (UTIL_clockSpanMicro(coolTime, ticksPerSecond) > ACTIVEPERIOD_MICROSEC) {
@@ -306,7 +306,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                     break;
             }   }   /* CRC Checking */
 #endif
-        }   /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */
+        }   /* for (testNb = 1; testNb <= (g_nbSeconds + !g_nbSeconds); testNb++) */
 
         if (g_displayLevel == 1) {
             double cSpeed = (double)srcSize / fastestC;
@@ -361,7 +361,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
     SET_HIGH_PRIORITY;
 
     if (g_displayLevel == 1 && !g_additionalParam)
-        DISPLAY("bench %s %s: input %u bytes, %u iterations, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
+        DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10));
 
     if (cLevelLast < cLevel) cLevelLast = cLevel;
 

programs/bench.h

@@ -17,7 +17,7 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
                    const char* dictFileName, int cLevel, int cLevelLast);
 
 /* Set Parameters */
-void BMK_SetNbIterations(unsigned nbLoops);
+void BMK_SetNbSeconds(unsigned nbLoops);
 void BMK_SetBlockSize(size_t blockSize);
 void BMK_setAdditionalParam(int additionalParam);
 void BMK_setNotificationLevel(unsigned level);

programs/fileio.c

@@ -375,14 +375,18 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
                                         const char* dstFileName, const char* srcFileName)
 {
     int result;
+    stat_t statbuf;
+    int stat_result = 0;
 
     ress.dstFile = FIO_openDstFile(dstFileName);
     if (ress.dstFile==NULL) return 1;  /* could not open dstFileName */
 
+    if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf)) stat_result = 1;
     result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName);
 
     if (fclose(ress.dstFile)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; }  /* error closing dstFile */
     if (result!=0) { if (remove(dstFileName)) EXM_THROW(1, "zstd: %s: %s", dstFileName, strerror(errno)); }  /* remove operation artefact */
+    else if (strcmp (dstFileName, stdoutmark) && stat_result) UTIL_setFileStat(dstFileName, &statbuf);
     return result;
 }
 
@@ -426,7 +430,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
         SET_BINARY_MODE(stdout);
         for (u=0; u<nbFiles; u++)
             missed_files += FIO_compressFilename_srcFile(ress, stdoutmark, inFileNamesTable[u]);
-        if (fclose(ress.dstFile)) EXM_THROW(29, "Write error : cannot properly close %s", stdoutmark);
+        if (fclose(ress.dstFile)) EXM_THROW(29, "Write error : cannot properly close stdout");
     } else {
         unsigned u;
         for (u=0; u<nbFiles; u++) {
@@ -702,16 +706,22 @@ static int FIO_decompressDstFile(dRess_t ress,
                                       const char* dstFileName, const char* srcFileName)
 {
     int result;
+    stat_t statbuf;
+    int stat_result = 0;
+
     ress.dstFile = FIO_openDstFile(dstFileName);
     if (ress.dstFile==0) return 1;
 
+    if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf)) stat_result = 1;
     result = FIO_decompressSrcFile(ress, srcFileName);
 
     if (fclose(ress.dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName);    
+
     if ( (result != 0)
        && strcmp(dstFileName, nulmark)  /* special case : don't remove() /dev/null (#316) */
        && remove(dstFileName) )
         result=1;   /* don't do anything special if remove() fails */
+    else if (strcmp (dstFileName, stdoutmark) && stat_result) UTIL_setFileStat(dstFileName, &statbuf);
     return result;
 }
 
@@ -746,7 +756,7 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
         if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", suffix);
         for (u=0; u<nbFiles; u++)
             missingFiles += FIO_decompressSrcFile(ress, srcNamesTable[u]);
-        if (fclose(ress.dstFile)) EXM_THROW(72, "Write error : cannot properly close %s", stdoutmark);
+        if (fclose(ress.dstFile)) EXM_THROW(72, "Write error : cannot properly close stdout");
     } else {
         size_t const suffixSize = strlen(suffix);
         size_t dfnSize = FNSPACE;

programs/fileio.h

@@ -18,8 +18,8 @@ extern "C" {
 /* *************************************
 *  Special i/o constants
 **************************************/
-#define stdinmark "stdin"
+#define stdinmark "/*stdin*\\"
-#define stdoutmark "stdout"
+#define stdoutmark "/*stdout*\\"
 #ifdef _WIN32
 #  define nulmark "nul"
 #else

programs/util.h

@@ -46,8 +46,17 @@ extern "C" {
 ******************************************/
 #include <stdlib.h>     /* features.h with _POSIX_C_SOURCE, malloc */
 #include <stdio.h>      /* fprintf */
-#include <sys/types.h>  /* stat */
+#include <sys/types.h>  /* stat, utime */
 #include <sys/stat.h>   /* stat */
+#if defined(_MSC_VER)
+	#include <sys/utime.h>   /* utime */
+	#include <io.h>          /* _chmod */
+#else
+	#include <unistd.h>     /* chown, stat */
+	#include <utime.h>      /* utime */
+#endif
+#include <time.h>       /* time */
+#include <errno.h>
 #include "mem.h"        /* U32, U64 */
 
 
@@ -142,6 +151,48 @@ UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond)
 /*-****************************************
 *  File functions
 ******************************************/
+#if defined(_MSC_VER)
+	#define chmod _chmod
+	typedef struct _stat64 stat_t;
+#else
+    typedef struct stat stat_t;
+#endif
+
+
+UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf)
+{
+    int res = 0;
+    struct utimbuf timebuf;
+
+	timebuf.actime = time(NULL);
+	timebuf.modtime = statbuf->st_mtime;
+	res += utime(filename, &timebuf);  /* set access and modification times */
+
+#if !defined(_WIN32)
+    res += chown(filename, statbuf->st_uid, statbuf->st_gid);  /* Copy ownership */
+#endif
+
+    res += chmod(filename, statbuf->st_mode & 07777);  /* Copy file permissions */
+
+    errno = 0;
+    return -res; /* number of errors is returned */
+}
+
+
+UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
+{
+    int r;
+#if defined(_MSC_VER)
+    r = _stat64(infilename, statbuf);
+    if (r || !(statbuf->st_mode & S_IFREG)) return 0;   /* No good... */
+#else
+    r = stat(infilename, statbuf);
+    if (r || !S_ISREG(statbuf->st_mode)) return 0;   /* No good... */
+#endif
+    return 1;
+}
+
+
 UTIL_STATIC U64 UTIL_getFileSize(const char* infilename)
 {
     int r;
@@ -278,7 +329,6 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
      ((defined(__unix__) || defined(__unix) || defined(__midipix__)) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) /* snprintf, opendir */
 #  define UTIL_HAS_CREATEFILELIST
 #  include <dirent.h>       /* opendir, readdir */
-#  include <errno.h>
 
 UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd)
 {

programs/zstdcli.c

@@ -28,7 +28,7 @@
 #include <errno.h>    /* errno */
 #include "fileio.h"
 #ifndef ZSTD_NOBENCH
-#  include "bench.h"  /* BMK_benchFiles, BMK_SetNbIterations */
+#  include "bench.h"  /* BMK_benchFiles, BMK_SetNbSeconds */
 #endif
 #ifndef ZSTD_NODICT
 #  include "dibio.h"
@@ -383,7 +383,7 @@ int main(int argCount, const char* argv[])
                         argument++;
                         {   U32 const iters = readU32FromChar(&argument);
                             BMK_setNotificationLevel(displayLevel);
-                            BMK_SetNbIterations(iters);
+                            BMK_SetNbSeconds(iters);
                         }
                         break;
 

tests/Makefile

@@ -50,12 +50,11 @@ ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
 # Define *.exe as extension for Windows systems
 ifneq (,$(filter Windows%,$(OS)))
 EXT =.exe
-VOID = nul
 else
 EXT =
-VOID = /dev/null
 endif
 
+VOID = /dev/null
 ZBUFFTEST = -T2mn
 FUZZERTEST= -T5mn
 ZSTDRTTEST= --test-large-data

zlibWrapper/Makefile

@@ -1,7 +1,7 @@
 # Makefile for example of using zstd wrapper for zlib
 #
 # make - compiles examples
-# make LOC=-DZWRAP_USE_ZSTD=1 - compiles examples with zstd compression turned on
+# make MOREFLAGS=-DZWRAP_USE_ZSTD=1 - compiles examples with zstd compression turned on
 # make test - runs examples
 
 
@@ -15,12 +15,10 @@ ZLIBWRAPPER_PATH = .
 EXAMPLE_PATH = examples
 PROGRAMS_PATH = ../programs
 TEST_FILE = ../doc/zstd_compression_format.md
-CC ?= gcc
+
-CFLAGS ?= -O3
+CPPFLAGS = -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH)
-CFLAGS += $(LOC) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -std=gnu99
+CFLAGS  ?= $(MOREFLAGS) -O3 -std=gnu99
-CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
+CFLAGS  += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wstrict-aliasing=1 -Wundef
-LDFLAGS = $(LOC)
-RM = rm -f
 
 
 all: clean fitblk example zwrapbench
@@ -48,8 +46,8 @@ valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench
 	$(VALGRIND) ./zwrapbench -qb3B1K $(TEST_FILE)
 	$(VALGRIND) ./zwrapbench -rqb1e5 ../lib ../programs ../tests
 
-.c.o:
+#.c.o:
-	$(CC) $(CFLAGS) -c -o $@ $<
+#	$(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $<
 
 example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY)
 	$(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY) $(ZLIB_LIBRARY)
@@ -69,10 +67,10 @@ zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o
 $(EXAMPLE_PATH)/zwrapbench.o: $(EXAMPLE_PATH)/zwrapbench.c
 
 $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
-	$(CC) $(CFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
+	$(CC) $(CFLAGS) $(CPPFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
 
 $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
-	$(CC) $(CFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
+	$(CC) $(CFLAGS) $(CPPFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
 
 $(ZSTDLIBDIR)/libzstd.a:
 	$(MAKE) -C $(ZSTDLIBDIR) libzstd.a

zlibWrapper/examples/zwrapbench.c

@@ -130,7 +130,7 @@ void BMK_SetBlockSize(size_t blockSize)
 **********************************************************/
 typedef struct
 {
-    const char* srcPtr;
+    z_const char* srcPtr;
     size_t srcSize;
     char*  cPtr;
     size_t cRoom;
@@ -145,7 +145,7 @@ typedef enum { BMK_ZSTD, BMK_ZSTD_STREAM, BMK_ZLIB, BMK_ZWRAP_ZLIB, BMK_ZWRAP_ZS
 #define MIN(a,b) ((a)<(b) ? (a) : (b))
 #define MAX(a,b) ((a)>(b) ? (a) : (b))
 
-static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
+static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
                         const char* displayName, int cLevel,
                         const size_t* fileSizes, U32 nbFiles,
                         const void* dictBuffer, size_t dictBufferSize, BMK_compressor compressor)
@@ -171,7 +171,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
     UTIL_initTimer(&ticksPerSecond);
 
     /* Init blockTable data */
-    {   const char* srcPtr = (const char*)srcBuffer;
+    {   z_const char* srcPtr = (z_const char*)srcBuffer;
         char* cPtr = (char*)compressedBuffer;
         char* resPtr = (char*)resultBuffer;
         U32 fileNb;
@@ -307,7 +307,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                                 if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure");
                                 if (ZWRAP_isUsingZSTDcompression()) useSetDict = 0; /* zstd doesn't require deflateSetDictionary after ZWRAP_deflateReset_keepDict */
                             }
-                            def.next_in = (const void*) blockTable[blockNb].srcPtr;
+                            def.next_in = (z_const void*) blockTable[blockNb].srcPtr;
                             def.avail_in = blockTable[blockNb].srcSize;
                             def.total_in = 0;
                             def.next_out = (void*) blockTable[blockNb].cPtr;
@@ -338,7 +338,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                                 ret = deflateSetDictionary(&def, dictBuffer, dictBufferSize);
                                 if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure");
                             }
-                            def.next_in = (const void*) blockTable[blockNb].srcPtr;
+                            def.next_in = (z_const void*) blockTable[blockNb].srcPtr;
                             def.avail_in = blockTable[blockNb].srcSize;
                             def.total_in = 0;
                             def.next_out = (void*) blockTable[blockNb].cPtr;
@@ -443,7 +443,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                             else
                                 ret = inflateReset(&inf);
                             if (ret != Z_OK) EXM_THROW(1, "inflateReset failure");
-                            inf.next_in = (const void*) blockTable[blockNb].cPtr;
+                            inf.next_in = (z_const void*) blockTable[blockNb].cPtr;
                             inf.avail_in = blockTable[blockNb].cSize;
                             inf.total_in = 0;
                             inf.next_out = (void*) blockTable[blockNb].resPtr;
@@ -475,7 +475,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
                             inf.opaque = Z_NULL;
                             ret = inflateInit(&inf);
                             if (ret != Z_OK) EXM_THROW(1, "inflateInit failure");
-                            inf.next_in = (const void*) blockTable[blockNb].cPtr;
+                            inf.next_in = (z_const void*) blockTable[blockNb].cPtr;
                             inf.avail_in = blockTable[blockNb].cSize;
                             inf.total_in = 0;
                             inf.next_out = (void*) blockTable[blockNb].resPtr;

zlibWrapper/zstd_zlibwrapper.c

@@ -10,7 +10,9 @@
 
 #include <stdio.h>                 /* vsprintf */
 #include <stdarg.h>                /* va_list, for z_gzprintf */
-#include <zlib.h>
+#define NO_DUMMY_DECL
+#define ZLIB_CONST
+#include <zlib.h>                  /* without #define Z_PREFIX */
 #include "zstd_zlibwrapper.h"
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_MAGICNUMBER */
 #include "zstd.h"
@@ -87,6 +89,9 @@ typedef struct {
     unsigned long long pledgedSrcSize;
 } ZWRAP_CCtx;
 
+typedef ZWRAP_CCtx internal_state;
+
+
 
 size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc)
 {

zlibWrapper/zstd_zlibwrapper.h

@@ -15,16 +15,14 @@ extern "C" {
 #endif
 
 
+#define ZLIB_CONST
 #define Z_PREFIX
 #include <zlib.h>
 
 #if !defined(z_const)
-#if ZLIB_VERNUM >= 0x1260
-    #define z_const const
-#else
     #define z_const
 #endif
-#endif
+
 
 /* returns a string with version of zstd library */
 const char * zstdVersion(void);