Merge pull request #779 from paulcruz74/adapt-approach-4

Patching warnings, adding ability to bound compression variation

contrib/adaptive-compression/Makefile

@@ -6,6 +6,7 @@ ZSTDCOMP_FILES   := $(ZSTDDIR)/compress/*.c
 ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c
 ZSTD_FILES  := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
 
+MULTITHREAD_LDFLAGS = -pthread
 DEBUGFLAGS= -g -DZSTD_DEBUG=1
 CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
             -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
@@ -17,7 +18,7 @@ CFLAGS   += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow                 \
             -Wredundant-decls
 CFLAGS   += $(DEBUGFLAGS)
 CFLAGS   += $(MOREFLAGS)
-FLAGS     = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+FLAGS     = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MULTITHREAD_LDFLAGS)
 
 all: adapt datagen
 
@@ -28,7 +29,7 @@ adapt-debug: $(ZSTD_FILES) adapt.c
 	$(CC) $(FLAGS) -DDEBUG_MODE=2 $^ -o adapt
 
 datagen : $(PRGDIR)/datagen.c datagencli.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+	$(CC)      $(FLAGS) $^ -o $@
 
 test-adapt-correctness: datagen adapt
 	@./test-correctness.sh
@@ -45,3 +46,31 @@ clean:
 	@$(RM) -f tests/*.zst
 	@$(RM) -f tests/tmp*
 	@echo "finished cleaning"
+
+#-----------------------------------------------------------------------------
+# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
+#-----------------------------------------------------------------------------
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
+
+ifneq (,$(filter $(shell uname),SunOS))
+INSTALL ?= ginstall
+else
+INSTALL ?= install
+endif
+
+PREFIX  ?= /usr/local
+DESTDIR ?=
+BINDIR  ?= $(PREFIX)/bin
+
+INSTALL_PROGRAM ?= $(INSTALL) -m 755
+
+install: adapt
+	@echo Installing binaries
+	@$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/
+	@$(INSTALL_PROGRAM) adapt $(DESTDIR)$(BINDIR)/zstd-adaptive
+	@echo zstd-adaptive installation completed
+
+uninstall:
+	@$(RM) $(DESTDIR)$(BINDIR)/zstd-adaptive
+	@echo zstd-adaptive programs successfully uninstalled
+endif

contrib/adaptive-compression/adapt.c

@@ -42,6 +42,8 @@ static size_t g_streamedSize = 0;
 static unsigned g_useProgressBar = 1;
 static UTIL_freq_t g_ticksPerSecond;
 static unsigned g_forceCompressionLevel = 0;
+static unsigned g_minCLevel = 1;
+static unsigned g_maxCLevel;
 
 typedef struct {
     void* start;
@@ -331,7 +333,7 @@ static void signalErrorToThreads(adaptCCtx* ctx)
     pthread_mutex_unlock(&ctx->jobReady_mutex.pMutex);
 
     pthread_mutex_lock(&ctx->jobCompressed_mutex.pMutex);
-    pthread_cond_signal(&ctx->jobCompressed_cond.pCond);
+    pthread_cond_broadcast(&ctx->jobCompressed_cond.pCond);
     pthread_mutex_unlock(&ctx->jobReady_mutex.pMutex);
 
     pthread_mutex_lock(&ctx->jobWrite_mutex.pMutex);
@@ -412,6 +414,8 @@ static void adaptCompressionLevel(adaptCCtx* ctx)
     pthread_mutex_unlock(&ctx->createCompletion_mutex.pMutex);
     DEBUG(2, "convergence counter: %u\n", ctx->convergenceCounter);
 
+    assert(g_minCLevel <= ctx->compressionLevel && g_maxCLevel >= ctx->compressionLevel);
+
     /* adaptation logic */
     if (ctx->cooldown) ctx->cooldown--;
 
@@ -420,7 +424,7 @@ static void adaptCompressionLevel(adaptCCtx* ctx)
         /* use whichever one waited less because it was slower */
         double const completion = MAX(createWaitCompressionCompletion, writeWaitCompressionCompletion);
         unsigned const change = convertCompletionToChange(completion);
-        unsigned const boundChange = MIN(change, ctx->compressionLevel - 1);
+        unsigned const boundChange = MIN(change, ctx->compressionLevel - g_minCLevel);
         if (ctx->convergenceCounter >= CONVERGENCE_LOWER_BOUND && boundChange != 0) {
             /* reset convergence counter, might have been a spike */
             ctx->convergenceCounter = 0;
@@ -438,7 +442,7 @@ static void adaptCompressionLevel(adaptCCtx* ctx)
         /* compress waiting on write */
         double const completion = MIN(compressWaitWriteCompletion, compressWaitCreateCompletion);
         unsigned const change = convertCompletionToChange(completion);
-        unsigned const boundChange = MIN(change, ZSTD_maxCLevel() - ctx->compressionLevel);
+        unsigned const boundChange = MIN(change, g_maxCLevel - ctx->compressionLevel);
         if (ctx->convergenceCounter >= CONVERGENCE_LOWER_BOUND && boundChange != 0) {
             /* reset convergence counter, might have been a spike */
             ctx->convergenceCounter = 0;
@@ -619,18 +623,20 @@ static void* compressionThread(void* arg)
 
 static void displayProgress(unsigned cLevel, unsigned last)
 {
-    if (!g_useProgressBar) return;
     UTIL_time_t currTime;
     UTIL_getTime(&currTime);
-    double const timeElapsed = (double)(UTIL_getSpanTimeMicro(g_ticksPerSecond, g_startTime, currTime) / 1000.0);
+    if (!g_useProgressBar) return;
-    double const sizeMB = (double)g_streamedSize / (1 << 20);
+    {
-    double const avgCompRate = sizeMB * 1000 / timeElapsed;
+        double const timeElapsed = (double)(UTIL_getSpanTimeMicro(g_ticksPerSecond, g_startTime, currTime) / 1000.0);
-    fprintf(stderr, "\r| Comp. Level: %2u | Time Elapsed: %7.2f s | Data Size: %7.1f MB | Avg Comp. Rate: %6.2f MB/s |", cLevel, timeElapsed/1000.0, sizeMB, avgCompRate);
+        double const sizeMB = (double)g_streamedSize / (1 << 20);
-    if (last) {
+        double const avgCompRate = sizeMB * 1000 / timeElapsed;
-        fprintf(stderr, "\n");
+        fprintf(stderr, "\r| Comp. Level: %2u | Time Elapsed: %7.2f s | Data Size: %7.1f MB | Avg Comp. Rate: %6.2f MB/s |", cLevel, timeElapsed/1000.0, sizeMB, avgCompRate);
-    }
+        if (last) {
-    else {
+            fprintf(stderr, "\n");
-        fflush(stderr);
+        }
+        else {
+            fflush(stderr);
+        }
     }
 }
 
@@ -928,9 +934,9 @@ static int freeFileCompressionResources(fcResources* fcr)
 static int compressFilename(const char* const srcFilename, const char* const dstFilenameOrNull)
 {
     int ret = 0;
+    fcResources fcr = createFileCompressionResources(srcFilename, dstFilenameOrNull);
     UTIL_getTime(&g_startTime);
     g_streamedSize = 0;
-    fcResources fcr = createFileCompressionResources(srcFilename, dstFilenameOrNull);
     ret |= performCompression(fcr.ctx, fcr.srcFile, fcr.otArg);
     ret |= freeFileCompressionResources(&fcr);
     return ret;
@@ -973,19 +979,21 @@ static unsigned readU32FromChar(const char** stringPtr)
     return result;
 }
 
-static void help()
+static void help(void)
 {
     PRINT("Usage:\n");
     PRINT("  ./multi [options] [file(s)]\n");
     PRINT("\n");
     PRINT("Options:\n");
     PRINT("  -oFILE : specify the output file name\n");
-    PRINT("  -i#    : provide initial compression level\n");
+    PRINT("  -i#    : provide initial compression level -- default %d, must be in the range [L, U] where L and U are bound values (see below for defaults)\n", DEFAULT_COMPRESSION_LEVEL);
     PRINT("  -h     : display help/information\n");
     PRINT("  -f     : force the compression level to stay constant\n");
     PRINT("  -c     : force write to stdout\n");
     PRINT("  -p     : hide progress bar\n");
     PRINT("  -q     : quiet mode -- do not show progress bar or other information\n");
+    PRINT("  -l#    : provide lower bound for compression level -- default 1\n");
+    PRINT("  -u#    : provide upper bound for compression level -- default %u\n", ZSTD_maxCLevel());
 }
 /* return 0 if successful, else return error */
 int main(int argCount, const char* argv[])
@@ -993,10 +1001,12 @@ int main(int argCount, const char* argv[])
     const char* outFilename = NULL;
     const char** filenameTable = (const char**)malloc(argCount*sizeof(const char*));
     unsigned filenameIdx = 0;
-    filenameTable[0] = stdinmark;
     unsigned forceStdout = 0;
+    unsigned providedInitialCLevel = 0;
     int ret = 0;
     int argNum;
+    filenameTable[0] = stdinmark;
+    g_maxCLevel = ZSTD_maxCLevel();
 
     UTIL_initTimer(&g_ticksPerSecond);
 
@@ -1018,6 +1028,7 @@ int main(int argCount, const char* argv[])
                 case 'i':
                     argument += 2;
                     g_compressionLevel = readU32FromChar(&argument);
+                    providedInitialCLevel = 1;
                     break;
                 case 'h':
                     help();
@@ -1036,6 +1047,14 @@ int main(int argCount, const char* argv[])
                     g_useProgressBar = 0;
                     g_displayLevel = 0;
                     break;
+                case 'l':
+                    argument += 2;
+                    g_minCLevel = readU32FromChar(&argument);
+                    break;
+                case 'u':
+                    argument += 2;
+                    g_maxCLevel = readU32FromChar(&argument);
+                    break;
                 default:
                     DISPLAY("Error: invalid argument provided\n");
                     ret = 1;
@@ -1048,6 +1067,20 @@ int main(int argCount, const char* argv[])
         filenameTable[filenameIdx++] = argument;
     }
 
+    /* check initial, max, and min compression levels */
+    {
+        unsigned const minMaxInconsistent = g_minCLevel > g_maxCLevel;
+        unsigned const initialNotInRange = g_minCLevel > g_compressionLevel || g_maxCLevel < g_compressionLevel;
+        if (minMaxInconsistent || (initialNotInRange && providedInitialCLevel)) {
+            DISPLAY("Error: provided compression level parameters are invalid\n");
+            ret = 1;
+            goto _main_exit;
+        }
+        else if (initialNotInRange) {
+            g_compressionLevel = g_minCLevel;
+        }
+    }
+
     /* error checking with number of files */
     if (filenameIdx > 1 && (outFilename != NULL && strcmp(outFilename, stdoutmark))) {
         DISPLAY("Error: multiple input files provided, cannot use specified output file\n");

contrib/adaptive-compression/test-correctness.sh

@@ -242,4 +242,11 @@ echo -e "\ncorrectness tests -- window size test"
 ./datagen -s39 -g1GB | pv -L 25m -q | ./adapt -i1 | pv -q > tmp.zst
 zstd -d tmp.zst
 rm tmp*
+
+echo -e "\ncorrectness tests -- testing bounds"
+./datagen -s40 -g1GB | pv -L 25m -q | ./adapt -i1 -u4 | pv -q > tmp.zst
+rm tmp*
+
+./datagen -s41 -g1GB | ./adapt -i14 -l4 > tmp.zst
+rm tmp*
 make clean