AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #441 from inikep/dev11

Browse Source 
preserve owner, file permission and modification time
This commit is contained in:
Yann Collet 2016-11-04 15:49:56 -07:00 committed by GitHub
commit 2fc035490e
14 changed files with 113 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Makefile
View File

@ -14,10 +14,12 @@ ZWRAPDIR = zlibWrapper
TESTDIR = tests
# Define nul output
ifneq (,$(filter Windows%,$(OS)))
VOID = nul
else
VOID = /dev/null
ifneq (,$(filter Windows%,$(OS)))
EXT =.exe
else
EXT =
endif
.PHONY: default
@ -35,7 +37,7 @@ lib:
zstd:
@$(MAKE) -C $(PRGDIR)
cp $(PRGDIR)/zstd .
cp $(PRGDIR)/zstd$(EXT) .
.PHONY: zlibwrapper
zlibwrapper:
@ -51,7 +53,7 @@ clean:
@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
@$(RM) zstd
@$(RM) zstd$(EXT)
@echo Cleaning completed

2
lib/Makefile
View File

@ -73,9 +73,11 @@ $(LIBZSTD): LDFLAGS += -shared -fPIC
$(LIBZSTD): $(ZSTD_FILES)
@echo compiling dynamic library $(LIBVER)
@$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
ifeq (,$(filter Windows%,$(OS)))
@echo creating versioned links
@ln -sf $(LIBZSTD) libzstd.$(SHARED_EXT_MAJOR)
@ln -sf $(LIBZSTD) libzstd.$(SHARED_EXT)
endif
libzstd : $(LIBZSTD)

2
programs/Makefile
View File

@ -56,7 +56,6 @@ endif
# Define *.exe as extension for Windows systems
ifneq (,$(filter Windows%,$(OS)))
EXT =.exe
VOID = nul
RES64_FILE = windres\zstd64.res
RES32_FILE = windres\zstd32.res
ifneq (,$(filter x86_64%,$(shell $(CC) -dumpmachine)))
@ -66,7 +65,6 @@ else
endif
else
EXT =
VOID = /dev/null
endif

18
programs/bench.c
View File

@ -33,7 +33,7 @@
# define ZSTD_GIT_COMMIT_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_GIT_COMMIT)
#endif
#define NBLOOPS 3
#define NBSECONDS 3
#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */
#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */
#define COOLPERIOD_SEC 10
@ -82,7 +82,7 @@ static clock_t g_time = 0;
/* *************************************
* Benchmark Parameters
***************************************/
static U32 g_nbIterations = NBLOOPS;
static U32 g_nbSeconds = NBSECONDS;
static size_t g_blockSize = 0;
int g_additionalParam = 0;
@ -90,10 +90,10 @@ void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
void BMK_SetNbIterations(unsigned nbLoops)
void BMK_SetNbSeconds(unsigned nbSeconds)
{
g_nbIterations = nbLoops;
DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbIterations);
g_nbSeconds = nbSeconds;
DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbSeconds);
}
void BMK_SetBlockSize(size_t blockSize)
@ -175,7 +175,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
{ U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL);
U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
UTIL_time_t coolTime;
U64 const maxTime = (g_nbIterations * TIMELOOP_MICROSEC) + 100;
U64 const maxTime = (g_nbSeconds * TIMELOOP_MICROSEC) + 100;
U64 totalCTime=0, totalDTime=0;
U32 cCompleted=0, dCompleted=0;
# define NB_MARKS 4
@ -188,7 +188,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
DISPLAYLEVEL(2, "\r%79s\r", "");
while (!cCompleted | !dCompleted) {
UTIL_time_t clockStart;
U64 clockLoop = g_nbIterations ? TIMELOOP_MICROSEC : 1;
U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1;
/* overheat protection */
if (UTIL_clockSpanMicro(coolTime, ticksPerSecond) > ACTIVEPERIOD_MICROSEC) {
@ -306,7 +306,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
break;
} } /* CRC Checking */
#endif
} /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */
} /* for (testNb = 1; testNb <= (g_nbSeconds + !g_nbSeconds); testNb++) */
if (g_displayLevel == 1) {
double cSpeed = (double)srcSize / fastestC;
@ -361,7 +361,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
SET_HIGH_PRIORITY;
if (g_displayLevel == 1 && !g_additionalParam)
DISPLAY("bench %s %s: input %u bytes, %u iterations, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10));
if (cLevelLast < cLevel) cLevelLast = cLevel;

2
programs/bench.h
View File

@ -17,7 +17,7 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName, int cLevel, int cLevelLast);
/* Set Parameters */
void BMK_SetNbIterations(unsigned nbLoops);
void BMK_SetNbSeconds(unsigned nbLoops);
void BMK_SetBlockSize(size_t blockSize);
void BMK_setAdditionalParam(int additionalParam);
void BMK_setNotificationLevel(unsigned level);

16
programs/fileio.c
View File

@ -375,14 +375,18 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
const char* dstFileName, const char* srcFileName)
{
int result;
stat_t statbuf;
int stat_result = 0;
ress.dstFile = FIO_openDstFile(dstFileName);
if (ress.dstFile==NULL) return 1; /* could not open dstFileName */
if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf)) stat_result = 1;
result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName);
if (fclose(ress.dstFile)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; } /* error closing dstFile */
if (result!=0) { if (remove(dstFileName)) EXM_THROW(1, "zstd: %s: %s", dstFileName, strerror(errno)); } /* remove operation artefact */
else if (strcmp (dstFileName, stdoutmark) && stat_result) UTIL_setFileStat(dstFileName, &statbuf);
return result;
}
@ -426,7 +430,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
SET_BINARY_MODE(stdout);
for (u=0; u<nbFiles; u++)
missed_files += FIO_compressFilename_srcFile(ress, stdoutmark, inFileNamesTable[u]);
if (fclose(ress.dstFile)) EXM_THROW(29, "Write error : cannot properly close %s", stdoutmark);
if (fclose(ress.dstFile)) EXM_THROW(29, "Write error : cannot properly close stdout");
} else {
unsigned u;
for (u=0; u<nbFiles; u++) {
@ -702,16 +706,22 @@ static int FIO_decompressDstFile(dRess_t ress,
const char* dstFileName, const char* srcFileName)
{
int result;
stat_t statbuf;
int stat_result = 0;
ress.dstFile = FIO_openDstFile(dstFileName);
if (ress.dstFile==0) return 1;
if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf)) stat_result = 1;
result = FIO_decompressSrcFile(ress, srcFileName);
if (fclose(ress.dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName);
if (fclose(ress.dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName);
if ( (result != 0)
&& strcmp(dstFileName, nulmark) /* special case : don't remove() /dev/null (#316) */
&& remove(dstFileName) )
result=1; /* don't do anything special if remove() fails */
else if (strcmp (dstFileName, stdoutmark) && stat_result) UTIL_setFileStat(dstFileName, &statbuf);
return result;
}
@ -746,7 +756,7 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", suffix);
for (u=0; u<nbFiles; u++)
missingFiles += FIO_decompressSrcFile(ress, srcNamesTable[u]);
if (fclose(ress.dstFile)) EXM_THROW(72, "Write error : cannot properly close %s", stdoutmark);
if (fclose(ress.dstFile)) EXM_THROW(72, "Write error : cannot properly close stdout");
} else {
size_t const suffixSize = strlen(suffix);
size_t dfnSize = FNSPACE;

4
programs/fileio.h
View File

@ -18,8 +18,8 @@ extern "C" {
/* *************************************
* Special i/o constants
**************************************/
#define stdinmark "stdin"
#define stdoutmark "stdout"
#define stdinmark "/*stdin*\\"
#define stdoutmark "/*stdout*\\"
#ifdef _WIN32
# define nulmark "nul"
#else

54
programs/util.h
View File

@ -46,8 +46,17 @@ extern "C" {
******************************************/
#include <stdlib.h> /* features.h with _POSIX_C_SOURCE, malloc */
#include <stdio.h> /* fprintf */
#include <sys/types.h> /* stat */
#include <sys/types.h> /* stat, utime */
#include <sys/stat.h> /* stat */
#if defined(_MSC_VER)
#include <sys/utime.h> /* utime */
#include <io.h> /* _chmod */
#else
#include <unistd.h> /* chown, stat */
#include <utime.h> /* utime */
#endif
#include <time.h> /* time */
#include <errno.h>
#include "mem.h" /* U32, U64 */
@ -142,6 +151,48 @@ UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond)
/*-****************************************
* File functions
******************************************/
#if defined(_MSC_VER)
#define chmod _chmod
typedef struct _stat64 stat_t;
#else
typedef struct stat stat_t;
#endif
UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf)
{
int res = 0;
struct utimbuf timebuf;
timebuf.actime = time(NULL);
timebuf.modtime = statbuf->st_mtime;
res += utime(filename, &timebuf); /* set access and modification times */
#if !defined(_WIN32)
res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */
#endif
res += chmod(filename, statbuf->st_mode & 07777); /* Copy file permissions */
errno = 0;
return -res; /* number of errors is returned */
}
UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
{
int r;
#if defined(_MSC_VER)
r = _stat64(infilename, statbuf);
if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */
#else
r = stat(infilename, statbuf);
if (r || !S_ISREG(statbuf->st_mode)) return 0; /* No good... */
#endif
return 1;
}
UTIL_STATIC U64 UTIL_getFileSize(const char* infilename)
{
int r;
@ -278,7 +329,6 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
((defined(__unix__) || defined(__unix) || defined(__midipix__)) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L)) /* snprintf, opendir */
# define UTIL_HAS_CREATEFILELIST
# include <dirent.h> /* opendir, readdir */
# include <errno.h>
UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd)
{

4
programs/zstdcli.c
View File

@ -28,7 +28,7 @@
#include <errno.h> /* errno */
#include "fileio.h"
#ifndef ZSTD_NOBENCH
# include "bench.h" /* BMK_benchFiles, BMK_SetNbIterations */
# include "bench.h" /* BMK_benchFiles, BMK_SetNbSeconds */
#endif
#ifndef ZSTD_NODICT
# include "dibio.h"
@ -383,7 +383,7 @@ int main(int argCount, const char* argv[])
argument++;
{ U32 const iters = readU32FromChar(&argument);
BMK_setNotificationLevel(displayLevel);
BMK_SetNbIterations(iters);
BMK_SetNbSeconds(iters);
}
break;

3
tests/Makefile
View File

@ -50,12 +50,11 @@ ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
# Define *.exe as extension for Windows systems
ifneq (,$(filter Windows%,$(OS)))
EXT =.exe
VOID = nul
else
EXT =
VOID = /dev/null
endif
VOID = /dev/null
ZBUFFTEST = -T2mn
FUZZERTEST= -T5mn
ZSTDRTTEST= --test-large-data

20
zlibWrapper/Makefile
View File

@ -1,7 +1,7 @@
# Makefile for example of using zstd wrapper for zlib
#
# make - compiles examples
# make LOC=-DZWRAP_USE_ZSTD=1 - compiles examples with zstd compression turned on
# make MOREFLAGS=-DZWRAP_USE_ZSTD=1 - compiles examples with zstd compression turned on
# make test - runs examples
@ -15,12 +15,10 @@ ZLIBWRAPPER_PATH = .
EXAMPLE_PATH = examples
PROGRAMS_PATH = ../programs
TEST_FILE = ../doc/zstd_compression_format.md
CC ?= gcc
CFLAGS ?= -O3
CFLAGS += $(LOC) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -std=gnu99
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
LDFLAGS = $(LOC)
RM = rm -f
CPPFLAGS = -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH)
CFLAGS ?= $(MOREFLAGS) -O3 -std=gnu99
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wstrict-aliasing=1 -Wundef
all: clean fitblk example zwrapbench
@ -48,8 +46,8 @@ valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench
$(VALGRIND) ./zwrapbench -qb3B1K $(TEST_FILE)
$(VALGRIND) ./zwrapbench -rqb1e5 ../lib ../programs ../tests
.c.o:
$(CC) $(CFLAGS) -c -o $@ $<
#.c.o:
# $(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $<
example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY)
$(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY) $(ZLIB_LIBRARY)
@ -69,10 +67,10 @@ zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o
$(EXAMPLE_PATH)/zwrapbench.o: $(EXAMPLE_PATH)/zwrapbench.c
$(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
$(CC) $(CFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
$(CC) $(CFLAGS) $(CPPFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
$(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
$(CC) $(CFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
$(CC) $(CFLAGS) $(CPPFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
$(ZSTDLIBDIR)/libzstd.a:
$(MAKE) -C $(ZSTDLIBDIR) libzstd.a

14
zlibWrapper/examples/zwrapbench.c
View File

@ -130,7 +130,7 @@ void BMK_SetBlockSize(size_t blockSize)
**********************************************************/
typedef struct
{
const char* srcPtr;
z_const char* srcPtr;
size_t srcSize;
char* cPtr;
size_t cRoom;
@ -145,7 +145,7 @@ typedef enum { BMK_ZSTD, BMK_ZSTD_STREAM, BMK_ZLIB, BMK_ZWRAP_ZLIB, BMK_ZWRAP_ZS
#define MIN(a,b) ((a)<(b) ? (a) : (b))
#define MAX(a,b) ((a)>(b) ? (a) : (b))
static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize,
const char* displayName, int cLevel,
const size_t* fileSizes, U32 nbFiles,
const void* dictBuffer, size_t dictBufferSize, BMK_compressor compressor)
@ -171,7 +171,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
UTIL_initTimer(&ticksPerSecond);
/* Init blockTable data */
{ const char* srcPtr = (const char*)srcBuffer;
{ z_const char* srcPtr = (z_const char*)srcBuffer;
char* cPtr = (char*)compressedBuffer;
char* resPtr = (char*)resultBuffer;
U32 fileNb;
@ -307,7 +307,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure");
if (ZWRAP_isUsingZSTDcompression()) useSetDict = 0; /* zstd doesn't require deflateSetDictionary after ZWRAP_deflateReset_keepDict */
}
def.next_in = (const void*) blockTable[blockNb].srcPtr;
def.next_in = (z_const void*) blockTable[blockNb].srcPtr;
def.avail_in = blockTable[blockNb].srcSize;
def.total_in = 0;
def.next_out = (void*) blockTable[blockNb].cPtr;
@ -338,7 +338,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
ret = deflateSetDictionary(&def, dictBuffer, dictBufferSize);
if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure");
}
def.next_in = (const void*) blockTable[blockNb].srcPtr;
def.next_in = (z_const void*) blockTable[blockNb].srcPtr;
def.avail_in = blockTable[blockNb].srcSize;
def.total_in = 0;
def.next_out = (void*) blockTable[blockNb].cPtr;
@ -443,7 +443,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
else
ret = inflateReset(&inf);
if (ret != Z_OK) EXM_THROW(1, "inflateReset failure");
inf.next_in = (const void*) blockTable[blockNb].cPtr;
inf.next_in = (z_const void*) blockTable[blockNb].cPtr;
inf.avail_in = blockTable[blockNb].cSize;
inf.total_in = 0;
inf.next_out = (void*) blockTable[blockNb].resPtr;
@ -475,7 +475,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
inf.opaque = Z_NULL;
ret = inflateInit(&inf);
if (ret != Z_OK) EXM_THROW(1, "inflateInit failure");
inf.next_in = (const void*) blockTable[blockNb].cPtr;
inf.next_in = (z_const void*) blockTable[blockNb].cPtr;
inf.avail_in = blockTable[blockNb].cSize;
inf.total_in = 0;
inf.next_out = (void*) blockTable[blockNb].resPtr;

7
zlibWrapper/zstd_zlibwrapper.c
View File

@ -10,7 +10,9 @@
#include <stdio.h> /* vsprintf */
#include <stdarg.h> /* va_list, for z_gzprintf */
#include <zlib.h>
#define NO_DUMMY_DECL
#define ZLIB_CONST
#include <zlib.h> /* without #define Z_PREFIX */
#include "zstd_zlibwrapper.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_MAGICNUMBER */
#include "zstd.h"
@ -87,6 +89,9 @@ typedef struct {
unsigned long long pledgedSrcSize;
} ZWRAP_CCtx;
typedef ZWRAP_CCtx internal_state;
size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc)
{

6
zlibWrapper/zstd_zlibwrapper.h
View File

@ -15,16 +15,14 @@ extern "C" {
#endif
#define ZLIB_CONST
#define Z_PREFIX
#include <zlib.h>
#if !defined(z_const)
#if ZLIB_VERNUM >= 0x1260
#define z_const const
#else
#define z_const
#endif
#endif
/* returns a string with version of zstd library */
const char * zstdVersion(void);