AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'dev' into multiframe

Browse Source
This commit is contained in:
Sean Purcell 2017-02-10 10:08:55 -08:00 committed by GitHub
commit 5069b6c2c3
14 changed files with 55 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/common/zstd_common.c
View File

@ -41,7 +41,7 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
/*! ZSTD_getErrorString() :
* provides error code string from enum */
const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); }
const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); }
/*=**************************************************************

4
lib/deprecated/zbuff.h
View File

@ -42,7 +42,9 @@ extern "C" {
#ifdef ZBUFF_DISABLE_DEPRECATE_WARNINGS
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API /* disable deprecation warnings */
#else
# if (defined(__GNUC__) && (__GNUC__ >= 5)) || defined(__clang__)
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
# define ZBUFF_DEPRECATED(message) [[deprecated(message)]] ZSTDLIB_API
# elif (defined(__GNUC__) && (__GNUC__ >= 5)) || defined(__clang__)
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated(message)))
# elif defined(__GNUC__) && (__GNUC__ >= 3)
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated))

2
lib/dictBuilder/zdict.h
View File

@ -174,7 +174,7 @@ ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBuffer
#else
# define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
# define ZDICT_DEPRECATED(message) ZDICTLIB_API [[deprecated(message)]]
# define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API
# elif (ZDICT_GCC_VERSION >= 405) || defined(__clang__)
# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message)))
# elif (ZDICT_GCC_VERSION >= 301)

1
lib/dll/example/build_package.bat
View File

@ -4,6 +4,7 @@ COPY tests\fullbench.c bin\example\
COPY programs\datagen.c bin\example\
COPY programs\datagen.h bin\example\
COPY programs\util.h bin\example\
COPY programs\platform.h bin\example\
COPY lib\common\mem.h bin\example\
COPY lib\common\zstd_errors.h bin\example\
COPY lib\common\zstd_internal.h bin\example\

5
programs/Makefile
View File

@ -67,8 +67,9 @@ endif
# zlib detection
VOID = /dev/null
HAVE_ZLIB := $(shell echo "\#include <zlib.h>\nint main(){}" | $(CC) -o $(VOID) -x c - -lz 2> $(VOID) && echo 1 || echo 0)
HAVE_ZLIB := $(shell printf '\#include <zlib.h>\nint main(){}' | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && echo 1 || echo 0)
ifeq ($(HAVE_ZLIB), 1)
TEMP := $(shell rm have_zlib$(EXT))
ZLIBCPP = -DZSTD_GZDECOMPRESS
ZLIBLD = -lz
endif
@ -152,7 +153,7 @@ clean:
@$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda
@$(RM) core *.o tmp* result* *.gcda dictionary *.zst \
zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \
*.gcda default.profraw have_zlib
*.gcda default.profraw have_zlib$(EXT)
@echo Cleaning completed
clean_decomp_o:

23
programs/bench.c
View File

@ -449,7 +449,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
const char* displayName, int cLevel, int cLevelLast,
const size_t* fileSizes, unsigned nbFiles,
const void* dictBuffer, size_t dictBufferSize,
ZSTD_compressionParameters *compressionParams)
ZSTD_compressionParameters *compressionParams, int setRealTimePrio)
{
int l;
@ -457,7 +457,10 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
if (!pch) pch = strrchr(displayName, '/'); /* Linux */
if (pch) displayName = pch+1;
SET_HIGH_PRIORITY;
if (setRealTimePrio) {
DISPLAYLEVEL(2, "Note : switching to a real-time priority \n");
SET_REALTIME_PRIORITY;
}
if (g_displayLevel == 1 && !g_additionalParam)
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10));
@ -505,8 +508,8 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
if (totalSize == 0) EXM_THROW(12, "no data to bench");
}
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
int cLevel, int cLevelLast, ZSTD_compressionParameters *compressionParams)
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel,
int cLevelLast, ZSTD_compressionParameters *compressionParams, int setRealTimePrio)
{
void* srcBuffer;
size_t benchedSize;
@ -545,7 +548,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co
BMK_benchCLevel(srcBuffer, benchedSize,
displayName, cLevel, cLevelLast,
fileSizes, nbFiles,
dictBuffer, dictBufferSize, compressionParams);
dictBuffer, dictBufferSize, compressionParams, setRealTimePrio);
}
/* clean up */
@ -555,7 +558,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co
}
static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams)
static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams, int setRealTimePrio)
{
char name[20] = {0};
size_t benchedSize = 10000000;
@ -569,7 +572,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
/* Bench */
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams);
BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams, setRealTimePrio);
/* clean up */
free(srcBuffer);
@ -577,7 +580,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName,
int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams)
int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio)
{
double const compressibility = (double)g_compressibilityDefault / 100;
@ -587,8 +590,8 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* di
if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
if (nbFiles == 0)
BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams);
BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams, setRealTimePrio);
else
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams);
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, setRealTimePrio);
return 0;
}

2
programs/bench.h
View File

@ -16,7 +16,7 @@
#include "zstd.h" /* ZSTD_compressionParameters */
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,const char* dictFileName,
int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams);
int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio);
/* Set Parameters */
void BMK_setNbSeconds(unsigned nbLoops);

2
programs/platform.h
View File

@ -77,8 +77,10 @@ extern "C" {
# define PLATFORM_POSIX_VERSION 200112L
# else
# if defined(__linux__) || defined(__linux)
# ifndef _POSIX_C_SOURCE
# define _POSIX_C_SOURCE 200112L /* use feature test macro */
# endif
# endif
# include <unistd.h> /* declares _POSIX_VERSION */
# if defined(_POSIX_VERSION) /* POSIX compliant */
# define PLATFORM_POSIX_VERSION _POSIX_VERSION

8
programs/util.h
View File

@ -44,7 +44,7 @@ extern "C" {
******************************************/
#if defined(_WIN32)
# include <windows.h>
# define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
# define SET_REALTIME_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
# define UTIL_sleep(s) Sleep(1000*s)
# define UTIL_sleepMilli(milli) Sleep(milli)
#elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */
@ -52,9 +52,9 @@ extern "C" {
# include <sys/resource.h> /* setpriority */
# include <time.h> /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */
# if defined(PRIO_PROCESS)
# define SET_HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
# define SET_REALTIME_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
# else
# define SET_HIGH_PRIORITY /* disabled */
# define SET_REALTIME_PRIORITY /* disabled */
# endif
# define UTIL_sleep(s) sleep(s)
# if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L) /* nanosleep requires POSIX.1-2001 */
@ -63,7 +63,7 @@ extern "C" {
# define UTIL_sleepMilli(milli) /* disabled */
# endif
#else
# define SET_HIGH_PRIORITY /* disabled */
# define SET_REALTIME_PRIORITY /* disabled */
# define UTIL_sleep(s) /* disabled */
# define UTIL_sleepMilli(milli) /* disabled */
#endif

3
programs/zstd.1
View File

@ -289,7 +289,8 @@ and weight typically 100x the target dictionary size (for example, 10 MB for a 1
.TP
.B \-B#
cut file into independent blocks of size # (default: no block)
.B \--priority=rt
set process priority to real-time
.SH ADVANCED COMPRESSION OPTIONS
.TP

7
programs/zstdcli.c
View File

@ -148,6 +148,7 @@ static int usage_advanced(const char* programName)
DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n");
DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n");
DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n");
DISPLAY( "--priority=rt : set process priority to real-time\n");
#endif
return 0;
}
@ -267,7 +268,8 @@ int main(int argCount, const char* argv[])
nextArgumentsAreFiles=0,
ultra=0,
lastCommand = 0,
nbThreads = 1;
nbThreads = 1,
setRealTimePrio = 0;
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
size_t blockSize = 0;
zstd_operation_mode operation = zom_compress;
@ -356,6 +358,7 @@ int main(int argCount, const char* argv[])
if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
/* long commands with arguments */
#ifndef ZSTD_NODICT
@ -574,7 +577,7 @@ int main(int argCount, const char* argv[])
BMK_setBlockSize(blockSize);
BMK_setNbThreads(nbThreads);
BMK_setNbSeconds(bench_nbSeconds);
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams);
BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, setRealTimePrio);
#endif
(void)bench_nbSeconds;
goto _end;

5
tests/Makefile
View File

@ -230,7 +230,10 @@ zstd-playTests: datagen
file $(ZSTD)
ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-pool test-legacy
test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy
ifneq ($(QEMU_SYS),qemu-ppc64-static)
test: test-pool
endif
test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32

10
tests/fuzzer.c
View File

@ -545,6 +545,16 @@ static int basicUnitTests(U32 seed, double compressibility)
if (r != _3BYTESTESTLENGTH) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
/* error string tests */
DISPLAYLEVEL(4, "test%3i : testing ZSTD error code strings : ", testNb++);
if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error;
if (strcmp("No error detected", ZSTD_getErrorString(ZSTD_error_no_error)) != 0) goto _output_error;
if (strcmp("Unspecified error code", ZSTD_getErrorString((ZSTD_ErrorCode)(0-ZSTD_error_GENERIC))) != 0) goto _output_error;
if (strcmp("Error (generic)", ZSTD_getErrorName((size_t)0-ZSTD_error_GENERIC)) != 0) goto _output_error;
if (strcmp("Error (generic)", ZSTD_getErrorString(ZSTD_error_GENERIC)) != 0) goto _output_error;
if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
_end:
free(CNBuffer);
free(compressedBuffer);

2
zlibWrapper/examples/zwrapbench.c
View File

@ -591,7 +591,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
if (!pch) pch = strrchr(displayName, '/'); /* Linux */
if (pch) displayName = pch+1;
SET_HIGH_PRIORITY;
SET_REALTIME_PRIORITY;
if (g_displayLevel == 1 && !g_additionalParam)
DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));