Merge branch 'dev' into multiframe
This commit is contained in:
Sean Purcell
GitHub
commit
5069b6c2c3
@ -41,7 +41,7 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
|
||||
|
||||
/*! ZSTD_getErrorString() :
|
||||
* provides error code string from enum */
|
||||
const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); }
|
||||
const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); }
|
||||
|
||||
|
||||
/*=**************************************************************
|
||||
|
||||
@ -42,7 +42,9 @@ extern "C" {
|
||||
#ifdef ZBUFF_DISABLE_DEPRECATE_WARNINGS
|
||||
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API /* disable deprecation warnings */
|
||||
#else
|
||||
# if (defined(__GNUC__) && (__GNUC__ >= 5)) || defined(__clang__)
|
||||
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
|
||||
# define ZBUFF_DEPRECATED(message) [[deprecated(message)]] ZSTDLIB_API
|
||||
# elif (defined(__GNUC__) && (__GNUC__ >= 5)) || defined(__clang__)
|
||||
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated(message)))
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3)
|
||||
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated))
|
||||
|
||||
@ -174,7 +174,7 @@ ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBuffer
|
||||
#else
|
||||
# define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
||||
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
|
||||
# define ZDICT_DEPRECATED(message) ZDICTLIB_API [[deprecated(message)]]
|
||||
# define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API
|
||||
# elif (ZDICT_GCC_VERSION >= 405) || defined(__clang__)
|
||||
# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message)))
|
||||
# elif (ZDICT_GCC_VERSION >= 301)
|
||||
|
||||
@ -4,6 +4,7 @@ COPY tests\fullbench.c bin\example\
|
||||
COPY programs\datagen.c bin\example\
|
||||
COPY programs\datagen.h bin\example\
|
||||
COPY programs\util.h bin\example\
|
||||
COPY programs\platform.h bin\example\
|
||||
COPY lib\common\mem.h bin\example\
|
||||
COPY lib\common\zstd_errors.h bin\example\
|
||||
COPY lib\common\zstd_internal.h bin\example\
|
||||
|
||||
@ -67,8 +67,9 @@ endif
|
||||
|
||||
# zlib detection
|
||||
VOID = /dev/null
|
||||
HAVE_ZLIB := $(shell echo "\#include <zlib.h>\nint main(){}" | $(CC) -o $(VOID) -x c - -lz 2> $(VOID) && echo 1 || echo 0)
|
||||
HAVE_ZLIB := $(shell printf '\#include <zlib.h>\nint main(){}' | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && echo 1 || echo 0)
|
||||
ifeq ($(HAVE_ZLIB), 1)
|
||||
TEMP := $(shell rm have_zlib$(EXT))
|
||||
ZLIBCPP = -DZSTD_GZDECOMPRESS
|
||||
ZLIBLD = -lz
|
||||
endif
|
||||
@ -152,7 +153,7 @@ clean:
|
||||
@$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda
|
||||
@$(RM) core *.o tmp* result* *.gcda dictionary *.zst \
|
||||
zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \
|
||||
*.gcda default.profraw have_zlib
|
||||
*.gcda default.profraw have_zlib$(EXT)
|
||||
@echo Cleaning completed
|
||||
|
||||
clean_decomp_o:
|
||||
|
||||
@ -449,7 +449,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
|
||||
const char* displayName, int cLevel, int cLevelLast,
|
||||
const size_t* fileSizes, unsigned nbFiles,
|
||||
const void* dictBuffer, size_t dictBufferSize,
|
||||
ZSTD_compressionParameters *compressionParams)
|
||||
ZSTD_compressionParameters *compressionParams, int setRealTimePrio)
|
||||
{
|
||||
int l;
|
||||
|
||||
@ -457,7 +457,10 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
|
||||
if (!pch) pch = strrchr(displayName, '/'); /* Linux */
|
||||
if (pch) displayName = pch+1;
|
||||
|
||||
SET_HIGH_PRIORITY;
|
||||
if (setRealTimePrio) {
|
||||
DISPLAYLEVEL(2, "Note : switching to a real-time priority \n");
|
||||
SET_REALTIME_PRIORITY;
|
||||
}
|
||||
|
||||
if (g_displayLevel == 1 && !g_additionalParam)
|
||||
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10));
|
||||
@ -505,8 +508,8 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
|
||||
if (totalSize == 0) EXM_THROW(12, "no data to bench");
|
||||
}
|
||||
|
||||
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
|
||||
int cLevel, int cLevelLast, ZSTD_compressionParameters *compressionParams)
|
||||
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel,
|
||||
int cLevelLast, ZSTD_compressionParameters *compressionParams, int setRealTimePrio)
|
||||
{
|
||||
void* srcBuffer;
|
||||
size_t benchedSize;
|
||||
@ -545,7 +548,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co
|
||||
BMK_benchCLevel(srcBuffer, benchedSize,
|
||||
displayName, cLevel, cLevelLast,
|
||||
fileSizes, nbFiles,
|
||||
dictBuffer, dictBufferSize, compressionParams);
|
||||
dictBuffer, dictBufferSize, compressionParams, setRealTimePrio);
|
||||
}
|
||||
|
||||
/* clean up */
|
||||
@ -555,7 +558,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co
|
||||
}
|
||||
|
||||
|
||||
static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams)
|
||||
static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams, int setRealTimePrio)
|
||||
{
|
||||
char name[20] = {0};
|
||||
size_t benchedSize = 10000000;
|
||||
@ -569,7 +572,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
|
||||
|
||||
/* Bench */
|
||||
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
|
||||
BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams);
|
||||
BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams, setRealTimePrio);
|
||||
|
||||
/* clean up */
|
||||
free(srcBuffer);
|
||||
@ -577,7 +580,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
|
||||
|
||||
|
||||
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
|
||||
int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams)
|
||||
int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio)
|
||||
{
|
||||
double const compressibility = (double)g_compressibilityDefault / 100;
|
||||
|
||||
@ -587,8 +590,8 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* di
|
||||
if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
|
||||
|
||||
if (nbFiles == 0)
|
||||
BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams);
|
||||
BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams, setRealTimePrio);
|
||||
else
|
||||
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams);
|
||||
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, setRealTimePrio);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
#include "zstd.h" /* ZSTD_compressionParameters */
|
||||
|
||||
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,const char* dictFileName,
|
||||
int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams);
|
||||
int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio);
|
||||
|
||||
/* Set Parameters */
|
||||
void BMK_setNbSeconds(unsigned nbLoops);
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
/**
|
||||
* platform.h - compiler and OS detection
|
||||
*
|
||||
*
|
||||
* Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
@ -23,7 +23,7 @@ extern "C" {
|
||||
****************************************/
|
||||
#if defined(_MSC_VER)
|
||||
# define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */
|
||||
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before <io.h> and <windows.h> */
|
||||
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before <io.h> and <windows.h> */
|
||||
# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */
|
||||
# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */
|
||||
# endif
|
||||
@ -52,7 +52,7 @@ extern "C" {
|
||||
* Turn on Large Files support (>4GB) for 32-bit Linux/Unix
|
||||
***********************************************************/
|
||||
#if !defined(__64BIT__) /* No point defining Large file for 64 bit */
|
||||
# if !defined(_FILE_OFFSET_BITS)
|
||||
# if !defined(_FILE_OFFSET_BITS)
|
||||
# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */
|
||||
# endif
|
||||
# if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */
|
||||
@ -77,7 +77,9 @@ extern "C" {
|
||||
# define PLATFORM_POSIX_VERSION 200112L
|
||||
# else
|
||||
# if defined(__linux__) || defined(__linux)
|
||||
# define _POSIX_C_SOURCE 200112L /* use feature test macro */
|
||||
# ifndef _POSIX_C_SOURCE
|
||||
# define _POSIX_C_SOURCE 200112L /* use feature test macro */
|
||||
# endif
|
||||
# endif
|
||||
# include <unistd.h> /* declares _POSIX_VERSION */
|
||||
# if defined(_POSIX_VERSION) /* POSIX compliant */
|
||||
|
||||
@ -44,7 +44,7 @@ extern "C" {
|
||||
******************************************/
|
||||
#if defined(_WIN32)
|
||||
# include <windows.h>
|
||||
# define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
|
||||
# define SET_REALTIME_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
|
||||
# define UTIL_sleep(s) Sleep(1000*s)
|
||||
# define UTIL_sleepMilli(milli) Sleep(milli)
|
||||
#elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */
|
||||
@ -52,9 +52,9 @@ extern "C" {
|
||||
# include <sys/resource.h> /* setpriority */
|
||||
# include <time.h> /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */
|
||||
# if defined(PRIO_PROCESS)
|
||||
# define SET_HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
|
||||
# define SET_REALTIME_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
|
||||
# else
|
||||
# define SET_HIGH_PRIORITY /* disabled */
|
||||
# define SET_REALTIME_PRIORITY /* disabled */
|
||||
# endif
|
||||
# define UTIL_sleep(s) sleep(s)
|
||||
# if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L) /* nanosleep requires POSIX.1-2001 */
|
||||
@ -63,7 +63,7 @@ extern "C" {
|
||||
# define UTIL_sleepMilli(milli) /* disabled */
|
||||
# endif
|
||||
#else
|
||||
# define SET_HIGH_PRIORITY /* disabled */
|
||||
# define SET_REALTIME_PRIORITY /* disabled */
|
||||
# define UTIL_sleep(s) /* disabled */
|
||||
# define UTIL_sleepMilli(milli) /* disabled */
|
||||
#endif
|
||||
|
||||
@ -289,7 +289,8 @@ and weight typically 100x the target dictionary size (for example, 10 MB for a 1
|
||||
.TP
|
||||
.B \-B#
|
||||
cut file into independent blocks of size # (default: no block)
|
||||
|
||||
.B \--priority=rt
|
||||
set process priority to real-time
|
||||
|
||||
.SH ADVANCED COMPRESSION OPTIONS
|
||||
.TP
|
||||
|
||||
@ -148,6 +148,7 @@ static int usage_advanced(const char* programName)
|
||||
DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n");
|
||||
DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n");
|
||||
DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n");
|
||||
DISPLAY( "--priority=rt : set process priority to real-time\n");
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
@ -267,7 +268,8 @@ int main(int argCount, const char* argv[])
|
||||
nextArgumentsAreFiles=0,
|
||||
ultra=0,
|
||||
lastCommand = 0,
|
||||
nbThreads = 1;
|
||||
nbThreads = 1,
|
||||
setRealTimePrio = 0;
|
||||
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
|
||||
size_t blockSize = 0;
|
||||
zstd_operation_mode operation = zom_compress;
|
||||
@ -356,6 +358,7 @@ int main(int argCount, const char* argv[])
|
||||
if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
|
||||
if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
|
||||
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
|
||||
if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
|
||||
|
||||
/* long commands with arguments */
|
||||
#ifndef ZSTD_NODICT
|
||||
@ -574,7 +577,7 @@ int main(int argCount, const char* argv[])
|
||||
BMK_setBlockSize(blockSize);
|
||||
BMK_setNbThreads(nbThreads);
|
||||
BMK_setNbSeconds(bench_nbSeconds);
|
||||
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams);
|
||||
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, setRealTimePrio);
|
||||
#endif
|
||||
(void)bench_nbSeconds;
|
||||
goto _end;
|
||||
|
||||
@ -230,7 +230,10 @@ zstd-playTests: datagen
|
||||
file $(ZSTD)
|
||||
ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
|
||||
|
||||
test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-pool test-legacy
|
||||
test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy
|
||||
ifneq ($(QEMU_SYS),qemu-ppc64-static)
|
||||
test: test-pool
|
||||
endif
|
||||
|
||||
test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32
|
||||
|
||||
|
||||
@ -545,6 +545,16 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
if (r != _3BYTESTESTLENGTH) goto _output_error; }
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* error string tests */
|
||||
DISPLAYLEVEL(4, "test%3i : testing ZSTD error code strings : ", testNb++);
|
||||
if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error;
|
||||
if (strcmp("No error detected", ZSTD_getErrorString(ZSTD_error_no_error)) != 0) goto _output_error;
|
||||
if (strcmp("Unspecified error code", ZSTD_getErrorString((ZSTD_ErrorCode)(0-ZSTD_error_GENERIC))) != 0) goto _output_error;
|
||||
if (strcmp("Error (generic)", ZSTD_getErrorName((size_t)0-ZSTD_error_GENERIC)) != 0) goto _output_error;
|
||||
if (strcmp("Error (generic)", ZSTD_getErrorString(ZSTD_error_GENERIC)) != 0) goto _output_error;
|
||||
if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
_end:
|
||||
free(CNBuffer);
|
||||
free(compressedBuffer);
|
||||
|
||||
@ -591,7 +591,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
|
||||
if (!pch) pch = strrchr(displayName, '/'); /* Linux */
|
||||
if (pch) displayName = pch+1;
|
||||
|
||||
SET_HIGH_PRIORITY;
|
||||
SET_REALTIME_PRIORITY;
|
||||
|
||||
if (g_displayLevel == 1 && !g_additionalParam)
|
||||
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
|
||||
|
||||
Loading…
Reference in New Issue
Block a user