From 7060aee8c2de9080135133d07221237e7db6adb7 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 6 Feb 2017 19:43:13 +0100 Subject: [PATCH 01/10] platform.h added to build_package.bat --- lib/dll/example/build_package.bat | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/dll/example/build_package.bat b/lib/dll/example/build_package.bat index ce738a56..b225af8d 100644 --- a/lib/dll/example/build_package.bat +++ b/lib/dll/example/build_package.bat @@ -4,6 +4,7 @@ COPY tests\fullbench.c bin\example\ COPY programs\datagen.c bin\example\ COPY programs\datagen.h bin\example\ COPY programs\util.h bin\example\ +COPY programs\platform.h bin\example\ COPY lib\common\mem.h bin\example\ COPY lib\common\zstd_errors.h bin\example\ COPY lib\common\zstd_internal.h bin\example\ From 94abd6a26cad4b2993fc0b5fce19a8b01d4a35ad Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 7 Feb 2017 16:36:19 +0100 Subject: [PATCH 02/10] SET_REALTIME_PRIORITY --- programs/bench.c | 2 +- programs/util.h | 8 ++++---- zlibWrapper/examples/zwrapbench.c | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index dcb23b1f..c8f1dcf0 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -457,7 +457,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, if (!pch) pch = strrchr(displayName, '/'); /* Linux */ if (pch) displayName = pch+1; - SET_HIGH_PRIORITY; + SET_REALTIME_PRIORITY; if (g_displayLevel == 1 && !g_additionalParam) DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10)); diff --git a/programs/util.h b/programs/util.h index 651027ba..656b3a96 100644 --- a/programs/util.h +++ b/programs/util.h @@ -44,7 +44,7 @@ extern "C" { ******************************************/ #if defined(_WIN32) # include -# define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) +# define SET_REALTIME_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) # define UTIL_sleep(s) Sleep(1000*s) # define UTIL_sleepMilli(milli) Sleep(milli) #elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */ @@ -52,9 +52,9 @@ extern "C" { # include /* setpriority */ # include /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */ # if defined(PRIO_PROCESS) -# define SET_HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20) +# define SET_REALTIME_PRIORITY setpriority(PRIO_PROCESS, 0, -20) # else -# define SET_HIGH_PRIORITY /* disabled */ +# define SET_REALTIME_PRIORITY /* disabled */ # endif # define UTIL_sleep(s) sleep(s) # if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L) /* nanosleep requires POSIX.1-2001 */ @@ -63,7 +63,7 @@ extern "C" { # define UTIL_sleepMilli(milli) /* disabled */ # endif #else -# define SET_HIGH_PRIORITY /* disabled */ +# define SET_REALTIME_PRIORITY /* disabled */ # define UTIL_sleep(s) /* disabled */ # define UTIL_sleepMilli(milli) /* disabled */ #endif diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index e5c54438..328c1096 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -591,7 +591,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, if (!pch) pch = strrchr(displayName, '/'); /* Linux */ if (pch) displayName = pch+1; - SET_HIGH_PRIORITY; + SET_REALTIME_PRIORITY; if (g_displayLevel == 1 && !g_additionalParam) DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10)); From d05014c7390ec99a6b13ba32c46529acfb5759d1 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 7 Feb 2017 16:48:01 +0100 Subject: [PATCH 03/10] added the "--rt-prio" option --- programs/bench.c | 23 +++++++++++++---------- programs/bench.h | 2 +- programs/zstdcli.c | 6 ++++-- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index c8f1dcf0..5870eaf7 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -449,7 +449,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, const char* displayName, int cLevel, int cLevelLast, const size_t* fileSizes, unsigned nbFiles, const void* dictBuffer, size_t dictBufferSize, - ZSTD_compressionParameters *compressionParams) + ZSTD_compressionParameters *compressionParams, int setRealTimePrio) { int l; @@ -457,7 +457,10 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, if (!pch) pch = strrchr(displayName, '/'); /* Linux */ if (pch) displayName = pch+1; - SET_REALTIME_PRIORITY; + if (setRealTimePrio) { + DISPLAYLEVEL(2, "Note : switching to a real-time priority \n"); + SET_REALTIME_PRIORITY; + } if (g_displayLevel == 1 && !g_additionalParam) DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10)); @@ -505,8 +508,8 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, if (totalSize == 0) EXM_THROW(12, "no data to bench"); } -static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, - int cLevel, int cLevelLast, ZSTD_compressionParameters *compressionParams) +static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel, + int cLevelLast, ZSTD_compressionParameters *compressionParams, int setRealTimePrio) { void* srcBuffer; size_t benchedSize; @@ -545,7 +548,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co BMK_benchCLevel(srcBuffer, benchedSize, displayName, cLevel, cLevelLast, fileSizes, nbFiles, - dictBuffer, dictBufferSize, compressionParams); + dictBuffer, dictBufferSize, compressionParams, setRealTimePrio); } /* clean up */ @@ -555,7 +558,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co } -static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams) +static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams, int setRealTimePrio) { char name[20] = {0}; size_t benchedSize = 10000000; @@ -569,7 +572,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility /* Bench */ snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); - BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams); + BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams, setRealTimePrio); /* clean up */ free(srcBuffer); @@ -577,7 +580,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, - int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams) + int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio) { double const compressibility = (double)g_compressibilityDefault / 100; @@ -587,8 +590,8 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* di if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); if (nbFiles == 0) - BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams); + BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams, setRealTimePrio); else - BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams); + BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, setRealTimePrio); return 0; } diff --git a/programs/bench.h b/programs/bench.h index 2918c02b..77a527f8 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -16,7 +16,7 @@ #include "zstd.h" /* ZSTD_compressionParameters */ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,const char* dictFileName, - int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams); + int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio); /* Set Parameters */ void BMK_setNbSeconds(unsigned nbLoops); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 6ca294fc..30391eb0 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -267,7 +267,8 @@ int main(int argCount, const char* argv[]) nextArgumentsAreFiles=0, ultra=0, lastCommand = 0, - nbThreads = 1; + nbThreads = 1, + setRealTimePrio = 0; unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ size_t blockSize = 0; zstd_operation_mode operation = zom_compress; @@ -356,6 +357,7 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; } if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } + if (!strcmp(argument, "--rt-prio")) { setRealTimePrio = 1; continue; } /* long commands with arguments */ #ifndef ZSTD_NODICT @@ -574,7 +576,7 @@ int main(int argCount, const char* argv[]) BMK_setBlockSize(blockSize); BMK_setNbThreads(nbThreads); BMK_setNbSeconds(bench_nbSeconds); - BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams); + BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, setRealTimePrio); #endif (void)bench_nbSeconds; goto _end; From 0665a359aa95ccc5386a1bb7cad42909fc2cca86 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 7 Feb 2017 20:12:59 +0100 Subject: [PATCH 04/10] "--rt-prio" renamed to "--priority=rt" --- programs/zstdcli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 30391eb0..604e4c5f 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -357,7 +357,7 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; } if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } - if (!strcmp(argument, "--rt-prio")) { setRealTimePrio = 1; continue; } + if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } /* long commands with arguments */ #ifndef ZSTD_NODICT From 40580ff669a4e5bb4d681973fb61281b2ac2d140 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 8 Feb 2017 13:49:06 +0100 Subject: [PATCH 05/10] added description of "--priority=rt" --- programs/zstd.1 | 3 ++- programs/zstdcli.c | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/programs/zstd.1 b/programs/zstd.1 index 384f69e3..684fb868 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -289,7 +289,8 @@ and weight typically 100x the target dictionary size (for example, 10 MB for a 1 .TP .B \-B# cut file into independent blocks of size # (default: no block) - +.B \--priority=rt + set process priority to real-time .SH ADVANCED COMPRESSION OPTIONS .TP diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 604e4c5f..a7dbda31 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -148,6 +148,7 @@ static int usage_advanced(const char* programName) DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n"); DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n"); DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n"); + DISPLAY( "--priority=rt : set process priority to real-time\n"); #endif return 0; } From 4b4f8c2d71ddd22df117b1330495acaa1100d95d Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 8 Feb 2017 13:58:04 +0100 Subject: [PATCH 06/10] turn off test-pool for qemu-ppc64-static --- tests/Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index f49d2301..07069d3a 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -225,7 +225,10 @@ zstd-playTests: datagen file $(ZSTD) ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST) -test: test-zstd test-fullbench test-fuzzer test-zstream test-longmatch test-invalidDictionaries test-pool +test: test-zstd test-fullbench test-fuzzer test-zstream test-longmatch test-invalidDictionaries +ifneq ($(QEMU_SYS),qemu-ppc64-static) +test: test-pool +endif test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32 From ca20edd96024e71011f83e526d831028c877e30f Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 8 Feb 2017 14:32:49 +0100 Subject: [PATCH 07/10] fixed zlib detection with MinGW --- programs/Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index ae798c2a..b189224f 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -67,7 +67,10 @@ endif # zlib detection VOID = /dev/null -HAVE_ZLIB := $(shell echo "\#include \nint main(){}" | $(CC) -o $(VOID) -x c - -lz 2> $(VOID) && echo 1 || echo 0) +HAVE_ZLIB := $(shell echo -e "\#include \nint main(){}" | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && echo 1 || echo 0) +ifeq ($(HAVE_ZLIB), 1) +TEMP := $(shell rm have_zlib$(EXT)) +endif ifeq ($(HAVE_ZLIB), 1) ZLIBCPP = -DZSTD_GZDECOMPRESS ZLIBLD = -lz @@ -152,7 +155,7 @@ clean: @$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda @$(RM) core *.o tmp* result* *.gcda dictionary *.zst \ zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \ - *.gcda default.profraw have_zlib + *.gcda default.profraw have_zlib$(EXT) @echo Cleaning completed clean_decomp_o: From 93901fe85ca3eb342dfbbbc6c4778c5f653a438c Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 8 Feb 2017 21:11:18 +0100 Subject: [PATCH 08/10] remove redundant "ifeq ($(HAVE_ZLIB), 1)" --- programs/Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index b189224f..8257e394 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -70,8 +70,6 @@ VOID = /dev/null HAVE_ZLIB := $(shell echo -e "\#include \nint main(){}" | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && echo 1 || echo 0) ifeq ($(HAVE_ZLIB), 1) TEMP := $(shell rm have_zlib$(EXT)) -endif -ifeq ($(HAVE_ZLIB), 1) ZLIBCPP = -DZSTD_GZDECOMPRESS ZLIBLD = -lz endif From 13127fd05b7ce947b57a0175c54a1b1c7353916f Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 9 Feb 2017 11:32:14 +0100 Subject: [PATCH 09/10] don't use "echo -e" --- programs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/Makefile b/programs/Makefile index 8257e394..a7de2e1a 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -67,7 +67,7 @@ endif # zlib detection VOID = /dev/null -HAVE_ZLIB := $(shell echo -e "\#include \nint main(){}" | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && echo 1 || echo 0) +HAVE_ZLIB := $(shell echo $$'\#include \nint main(){}' | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && echo 1 || echo 0) ifeq ($(HAVE_ZLIB), 1) TEMP := $(shell rm have_zlib$(EXT)) ZLIBCPP = -DZSTD_GZDECOMPRESS From 896638a8a296674cac145aff51b6dc7f724c4b94 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 9 Feb 2017 17:01:17 +0100 Subject: [PATCH 10/10] echo replaced with printf --- programs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/Makefile b/programs/Makefile index a7de2e1a..862ba35d 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -67,7 +67,7 @@ endif # zlib detection VOID = /dev/null -HAVE_ZLIB := $(shell echo $$'\#include \nint main(){}' | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && echo 1 || echo 0) +HAVE_ZLIB := $(shell printf '\#include \nint main(){}' | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && echo 1 || echo 0) ifeq ($(HAVE_ZLIB), 1) TEMP := $(shell rm have_zlib$(EXT)) ZLIBCPP = -DZSTD_GZDECOMPRESS