diff --git a/circle.yml b/circle.yml new file mode 100644 index 0000000..b3faae0 --- /dev/null +++ b/circle.yml @@ -0,0 +1,39 @@ +dependencies: + override: + - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update + - sudo apt-get -y install qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu + - sudo apt-get -y install qemu-system-arm gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross + - sudo apt-get -y install libc6-dev-i386 clang gcc-5 gcc-5-multilib gcc-6 valgrind + +test: + override: + # Tests compilers and C standards + - clang -v; make clangtest && make clean + - g++ -v; make gpptest && make clean + - gcc -v; make c_standards && make clean + - gcc-5 -v; make -C tests test-lz4 CC=gcc-5 MOREFLAGS=-Werror && make clean + - gcc-5 -v; make -C tests test-lz4c32 CC=gcc-5 MOREFLAGS="-I/usr/include/x86_64-linux-gnu -Werror" && make clean + - gcc-6 -v; make c_standards CC=gcc-6 && make clean + - gcc-6 -v; make -C tests test-lz4 CC=gcc-6 MOREFLAGS=-Werror && make clean +# Shorter tests + - make cmake && make clean + - make -C tests test-lz4 + - make -C tests test-lz4c + - make -C tests test-fasttest + - make -C tests test-frametest + - make -C tests test-fullbench + - make -C tests test-fuzzer && make clean + - make -C lib all && make clean + - pyenv global 3.4.4; CFLAGS=-I/usr/include/x86_64-linux-gnu make versionsTest && make clean + - make travis-install && make clean + # Longer tests + - gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean + - make usan && make clean + - clang -v; make staticAnalyze && make clean + # Valgrind tests + - make -C tests test-mem && make clean + # ARM, AArch64, PowerPC, PowerPC64 tests + - make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static && make clean + - make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS=-m64 && make clean + - make platformTest CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static && make clean + - make platformTest CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static && make clean diff --git a/programs/bench.c b/programs/bench.c index 8625802..71d3896 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -393,7 +393,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, if (!pch) pch = strrchr(displayName, '/'); /* Linux */ if (pch) displayName = pch+1; - SET_HIGH_PRIORITY; + SET_REALTIME_PRIORITY; if (g_displayLevel == 1 && !g_additionalParam) DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", LZ4_VERSION_STRING, LZ4_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10)); diff --git a/programs/lz4io.c b/programs/lz4io.c index 640c76d..5264b14 100644 --- a/programs/lz4io.c +++ b/programs/lz4io.c @@ -59,17 +59,6 @@ #include "lz4frame.h" -/* ************************************** -* Compiler Options -****************************************/ -#if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Avoid MSVC fseek()'s 2GiB barrier */ -# define fseek _fseeki64 -#endif -#if !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ -# define fseek fseeko -#endif - - /***************************** * Constants *****************************/ @@ -633,7 +622,7 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer /* avoid int overflow */ if (storedSkips > 1 GB) { - int const seekResult = fseek(file, 1 GB, SEEK_CUR); + int const seekResult = UTIL_fseek(file, 1 GB, SEEK_CUR); if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)"); storedSkips -= 1 GB; } @@ -650,7 +639,7 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer if (nb0T != seg0SizeT) { /* not all 0s */ errno = 0; - { int const seekResult = fseek(file, storedSkips, SEEK_CUR); + { int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR); if (seekResult) EXM_THROW(72, "Sparse skip error(%d): %s ; try --no-sparse", (int)errno, strerror(errno)); } storedSkips = 0; @@ -670,7 +659,7 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer for (; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; storedSkips += (unsigned) (restPtr - restStart); if (restPtr != restEnd) { - int const seekResult = fseek(file, storedSkips, SEEK_CUR); + int const seekResult = UTIL_fseek(file, storedSkips, SEEK_CUR); if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse"); storedSkips = 0; { size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file); @@ -684,7 +673,7 @@ static unsigned LZ4IO_fwriteSparse(FILE* file, const void* buffer, size_t buffer static void LZ4IO_fwriteSparseEnd(FILE* file, unsigned storedSkips) { if (storedSkips>0) { /* implies g_sparseFileSupport>0 */ - int const seekResult = fseek(file, storedSkips-1, SEEK_CUR); + int const seekResult = UTIL_fseek(file, storedSkips-1, SEEK_CUR); if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n"); { const char lastZeroByte[1] = { 0 }; size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file); @@ -873,7 +862,7 @@ static int fseek_u32(FILE *fp, unsigned offset, int where) while (offset > 0) { unsigned s = offset; if (s > stepMax) s = stepMax; - errorNb = fseek(fp, (long) s, SEEK_CUR); + errorNb = UTIL_fseek(fp, (long) s, SEEK_CUR); if (errorNb != 0) break; offset -= s; } diff --git a/programs/platform.h b/programs/platform.h index f9eb81a..51ce1ac 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -59,7 +59,7 @@ extern "C" { /* ********************************************************* * Turn on Large Files support (>4GB) for 32-bit Linux/Unix ***********************************************************/ -#if !defined(__64BIT__) || defined(__MINGW32__) /* No point defining Large file for 64 bit but MinGW requires it */ +#if !defined(__64BIT__) || defined(__MINGW32__) /* No point defining Large file for 64 bit but MinGW-w64 requires it */ # if !defined(_FILE_OFFSET_BITS) # define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ # endif diff --git a/programs/util.h b/programs/util.h index f3ff1b2..044085f 100644 --- a/programs/util.h +++ b/programs/util.h @@ -70,12 +70,26 @@ extern "C" { #endif +/* ************************************************************ +* Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW +***************************************************************/ +#if defined(_MSC_VER) && (_MSC_VER >= 1400) +# define UTIL_fseek _fseeki64 +#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ +# define UTIL_fseek fseeko +#elif defined(__MINGW32__) && defined(__MSVCRT__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) +# define UTIL_fseek fseeko64 +#else +# define UTIL_fseek fseek +#endif + + /*-**************************************** * Sleep functions: Windows - Posix - others ******************************************/ #if defined(_WIN32) # include -# define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) +# define SET_REALTIME_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) # define UTIL_sleep(s) Sleep(1000*s) # define UTIL_sleepMilli(milli) Sleep(milli) #elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */ @@ -83,9 +97,9 @@ extern "C" { # include /* setpriority */ # include /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */ # if defined(PRIO_PROCESS) -# define SET_HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20) +# define SET_REALTIME_PRIORITY setpriority(PRIO_PROCESS, 0, -20) # else -# define SET_HIGH_PRIORITY /* disabled */ +# define SET_REALTIME_PRIORITY /* disabled */ # endif # define UTIL_sleep(s) sleep(s) # if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L) /* nanosleep requires POSIX.1-2001 */ @@ -94,7 +108,7 @@ extern "C" { # define UTIL_sleepMilli(milli) /* disabled */ # endif #else -# define SET_HIGH_PRIORITY /* disabled */ +# define SET_REALTIME_PRIORITY /* disabled */ # define UTIL_sleep(s) /* disabled */ # define UTIL_sleepMilli(milli) /* disabled */ #endif @@ -126,18 +140,26 @@ extern "C" { /*-**************************************** * Time functions ******************************************/ -#if !defined(_WIN32) - typedef clock_t UTIL_time_t; - UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=0; } - UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { *x = clock(); } - UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } - UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } -#else +#if (PLATFORM_POSIX_VERSION >= 1) +#include +#include /* times */ + typedef U64 UTIL_time_t; + UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=sysconf(_SC_CLK_TCK); } + UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { struct tms junk; clock_t newTicks = (clock_t) times(&junk); (void)junk; *x = (UTIL_time_t)newTicks; } + UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / ticksPerSecond; } + UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / ticksPerSecond; } +#elif defined(_WIN32) /* Windows */ typedef LARGE_INTEGER UTIL_time_t; UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { if (!QueryPerformanceFrequency(ticksPerSecond)) fprintf(stderr, "ERROR: QueryPerformance not present\n"); } UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { QueryPerformanceCounter(x); } UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; } UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; } +#else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */ + typedef clock_t UTIL_time_t; + UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=0; } + UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { *x = clock(); } + UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } + UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } #endif @@ -166,7 +188,7 @@ UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond) ******************************************/ #if defined(_MSC_VER) #define chmod _chmod - typedef struct _stat64 stat_t; + typedef struct __stat64 stat_t; #else typedef struct stat stat_t; #endif @@ -206,13 +228,39 @@ UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) } +UTIL_STATIC int UTIL_isRegFile(const char* infilename) +{ + stat_t statbuf; + return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */ +} + + +UTIL_STATIC U32 UTIL_isDirectory(const char* infilename) +{ + int r; + stat_t statbuf; +#if defined(_MSC_VER) + r = _stat64(infilename, &statbuf); + if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; +#else + r = stat(infilename, &statbuf); + if (!r && S_ISDIR(statbuf.st_mode)) return 1; +#endif + return 0; +} + + UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) { int r; #if defined(_MSC_VER) - struct _stat64 statbuf; + struct __stat64 statbuf; r = _stat64(infilename, &statbuf); if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ +#elif defined(__MINGW32__) && defined (__MSVCRT__) + struct _stati64 statbuf; + r = _stati64(infilename, &statbuf); + if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ #else struct stat statbuf; r = stat(infilename, &statbuf); @@ -232,37 +280,6 @@ UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFi } -UTIL_STATIC int UTIL_doesFileExists(const char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct _stat64 statbuf; - r = _stat64(infilename, &statbuf); - if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ -#else - struct stat statbuf; - r = stat(infilename, &statbuf); - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ -#endif - return 1; -} - - -UTIL_STATIC U32 UTIL_isDirectory(const char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct _stat64 statbuf; - r = _stat64(infilename, &statbuf); - if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; -#else - struct stat statbuf; - r = stat(infilename, &statbuf); - if (!r && S_ISDIR(statbuf.st_mode)) return 1; -#endif - return 0; -} - /* * A modified version of realloc(). * If UTIL_realloc() fails the original block is freed. diff --git a/tests/Makefile b/tests/Makefile index ebab278..d1117c6 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -301,6 +301,7 @@ test-lz4c: lz4c datagen ./datagen -g256MB | $(LZ4)c -l -v | $(LZ4)c -t test-lz4c32: CFLAGS+=-m32 +test-lz4c32: MOREFLAGS+="-I/usr/include/x86_64-linux-gnu" test-lz4c32: test-lz4 test-interop-32-64: lz4 lz4c32 datagen