Merge pull request #779 from paulcruz74/adapt-approach-4
Patching warnings, adding ability to bound compression variation
This commit is contained in:
Yann Collet
GitHub
commit
a1fe656165
@ -6,6 +6,7 @@ ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c
|
||||
ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c
|
||||
ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
|
||||
|
||||
MULTITHREAD_LDFLAGS = -pthread
|
||||
DEBUGFLAGS= -g -DZSTD_DEBUG=1
|
||||
CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
|
||||
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
|
||||
@ -17,7 +18,7 @@ CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
|
||||
-Wredundant-decls
|
||||
CFLAGS += $(DEBUGFLAGS)
|
||||
CFLAGS += $(MOREFLAGS)
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MULTITHREAD_LDFLAGS)
|
||||
|
||||
all: adapt datagen
|
||||
|
||||
@ -28,7 +29,7 @@ adapt-debug: $(ZSTD_FILES) adapt.c
|
||||
$(CC) $(FLAGS) -DDEBUG_MODE=2 $^ -o adapt
|
||||
|
||||
datagen : $(PRGDIR)/datagen.c datagencli.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
$(CC) $(FLAGS) $^ -o $@
|
||||
|
||||
test-adapt-correctness: datagen adapt
|
||||
@./test-correctness.sh
|
||||
@ -45,3 +46,31 @@ clean:
|
||||
@$(RM) -f tests/*.zst
|
||||
@$(RM) -f tests/tmp*
|
||||
@echo "finished cleaning"
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
|
||||
#-----------------------------------------------------------------------------
|
||||
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
|
||||
|
||||
ifneq (,$(filter $(shell uname),SunOS))
|
||||
INSTALL ?= ginstall
|
||||
else
|
||||
INSTALL ?= install
|
||||
endif
|
||||
|
||||
PREFIX ?= /usr/local
|
||||
DESTDIR ?=
|
||||
BINDIR ?= $(PREFIX)/bin
|
||||
|
||||
INSTALL_PROGRAM ?= $(INSTALL) -m 755
|
||||
|
||||
install: adapt
|
||||
@echo Installing binaries
|
||||
@$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/
|
||||
@$(INSTALL_PROGRAM) adapt $(DESTDIR)$(BINDIR)/zstd-adaptive
|
||||
@echo zstd-adaptive installation completed
|
||||
|
||||
uninstall:
|
||||
@$(RM) $(DESTDIR)$(BINDIR)/zstd-adaptive
|
||||
@echo zstd-adaptive programs successfully uninstalled
|
||||
endif
|
||||
|
||||
@ -42,6 +42,8 @@ static size_t g_streamedSize = 0;
|
||||
static unsigned g_useProgressBar = 1;
|
||||
static UTIL_freq_t g_ticksPerSecond;
|
||||
static unsigned g_forceCompressionLevel = 0;
|
||||
static unsigned g_minCLevel = 1;
|
||||
static unsigned g_maxCLevel;
|
||||
|
||||
typedef struct {
|
||||
void* start;
|
||||
@ -331,7 +333,7 @@ static void signalErrorToThreads(adaptCCtx* ctx)
|
||||
pthread_mutex_unlock(&ctx->jobReady_mutex.pMutex);
|
||||
|
||||
pthread_mutex_lock(&ctx->jobCompressed_mutex.pMutex);
|
||||
pthread_cond_signal(&ctx->jobCompressed_cond.pCond);
|
||||
pthread_cond_broadcast(&ctx->jobCompressed_cond.pCond);
|
||||
pthread_mutex_unlock(&ctx->jobReady_mutex.pMutex);
|
||||
|
||||
pthread_mutex_lock(&ctx->jobWrite_mutex.pMutex);
|
||||
@ -412,6 +414,8 @@ static void adaptCompressionLevel(adaptCCtx* ctx)
|
||||
pthread_mutex_unlock(&ctx->createCompletion_mutex.pMutex);
|
||||
DEBUG(2, "convergence counter: %u\n", ctx->convergenceCounter);
|
||||
|
||||
assert(g_minCLevel <= ctx->compressionLevel && g_maxCLevel >= ctx->compressionLevel);
|
||||
|
||||
/* adaptation logic */
|
||||
if (ctx->cooldown) ctx->cooldown--;
|
||||
|
||||
@ -420,7 +424,7 @@ static void adaptCompressionLevel(adaptCCtx* ctx)
|
||||
/* use whichever one waited less because it was slower */
|
||||
double const completion = MAX(createWaitCompressionCompletion, writeWaitCompressionCompletion);
|
||||
unsigned const change = convertCompletionToChange(completion);
|
||||
unsigned const boundChange = MIN(change, ctx->compressionLevel - 1);
|
||||
unsigned const boundChange = MIN(change, ctx->compressionLevel - g_minCLevel);
|
||||
if (ctx->convergenceCounter >= CONVERGENCE_LOWER_BOUND && boundChange != 0) {
|
||||
/* reset convergence counter, might have been a spike */
|
||||
ctx->convergenceCounter = 0;
|
||||
@ -438,7 +442,7 @@ static void adaptCompressionLevel(adaptCCtx* ctx)
|
||||
/* compress waiting on write */
|
||||
double const completion = MIN(compressWaitWriteCompletion, compressWaitCreateCompletion);
|
||||
unsigned const change = convertCompletionToChange(completion);
|
||||
unsigned const boundChange = MIN(change, ZSTD_maxCLevel() - ctx->compressionLevel);
|
||||
unsigned const boundChange = MIN(change, g_maxCLevel - ctx->compressionLevel);
|
||||
if (ctx->convergenceCounter >= CONVERGENCE_LOWER_BOUND && boundChange != 0) {
|
||||
/* reset convergence counter, might have been a spike */
|
||||
ctx->convergenceCounter = 0;
|
||||
@ -619,18 +623,20 @@ static void* compressionThread(void* arg)
|
||||
|
||||
static void displayProgress(unsigned cLevel, unsigned last)
|
||||
{
|
||||
if (!g_useProgressBar) return;
|
||||
UTIL_time_t currTime;
|
||||
UTIL_getTime(&currTime);
|
||||
double const timeElapsed = (double)(UTIL_getSpanTimeMicro(g_ticksPerSecond, g_startTime, currTime) / 1000.0);
|
||||
double const sizeMB = (double)g_streamedSize / (1 << 20);
|
||||
double const avgCompRate = sizeMB * 1000 / timeElapsed;
|
||||
fprintf(stderr, "\r| Comp. Level: %2u | Time Elapsed: %7.2f s | Data Size: %7.1f MB | Avg Comp. Rate: %6.2f MB/s |", cLevel, timeElapsed/1000.0, sizeMB, avgCompRate);
|
||||
if (last) {
|
||||
fprintf(stderr, "\n");
|
||||
}
|
||||
else {
|
||||
fflush(stderr);
|
||||
if (!g_useProgressBar) return;
|
||||
{
|
||||
double const timeElapsed = (double)(UTIL_getSpanTimeMicro(g_ticksPerSecond, g_startTime, currTime) / 1000.0);
|
||||
double const sizeMB = (double)g_streamedSize / (1 << 20);
|
||||
double const avgCompRate = sizeMB * 1000 / timeElapsed;
|
||||
fprintf(stderr, "\r| Comp. Level: %2u | Time Elapsed: %7.2f s | Data Size: %7.1f MB | Avg Comp. Rate: %6.2f MB/s |", cLevel, timeElapsed/1000.0, sizeMB, avgCompRate);
|
||||
if (last) {
|
||||
fprintf(stderr, "\n");
|
||||
}
|
||||
else {
|
||||
fflush(stderr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -928,9 +934,9 @@ static int freeFileCompressionResources(fcResources* fcr)
|
||||
static int compressFilename(const char* const srcFilename, const char* const dstFilenameOrNull)
|
||||
{
|
||||
int ret = 0;
|
||||
fcResources fcr = createFileCompressionResources(srcFilename, dstFilenameOrNull);
|
||||
UTIL_getTime(&g_startTime);
|
||||
g_streamedSize = 0;
|
||||
fcResources fcr = createFileCompressionResources(srcFilename, dstFilenameOrNull);
|
||||
ret |= performCompression(fcr.ctx, fcr.srcFile, fcr.otArg);
|
||||
ret |= freeFileCompressionResources(&fcr);
|
||||
return ret;
|
||||
@ -973,19 +979,21 @@ static unsigned readU32FromChar(const char** stringPtr)
|
||||
return result;
|
||||
}
|
||||
|
||||
static void help()
|
||||
static void help(void)
|
||||
{
|
||||
PRINT("Usage:\n");
|
||||
PRINT(" ./multi [options] [file(s)]\n");
|
||||
PRINT("\n");
|
||||
PRINT("Options:\n");
|
||||
PRINT(" -oFILE : specify the output file name\n");
|
||||
PRINT(" -i# : provide initial compression level\n");
|
||||
PRINT(" -i# : provide initial compression level -- default %d, must be in the range [L, U] where L and U are bound values (see below for defaults)\n", DEFAULT_COMPRESSION_LEVEL);
|
||||
PRINT(" -h : display help/information\n");
|
||||
PRINT(" -f : force the compression level to stay constant\n");
|
||||
PRINT(" -c : force write to stdout\n");
|
||||
PRINT(" -p : hide progress bar\n");
|
||||
PRINT(" -q : quiet mode -- do not show progress bar or other information\n");
|
||||
PRINT(" -l# : provide lower bound for compression level -- default 1\n");
|
||||
PRINT(" -u# : provide upper bound for compression level -- default %u\n", ZSTD_maxCLevel());
|
||||
}
|
||||
/* return 0 if successful, else return error */
|
||||
int main(int argCount, const char* argv[])
|
||||
@ -993,10 +1001,12 @@ int main(int argCount, const char* argv[])
|
||||
const char* outFilename = NULL;
|
||||
const char** filenameTable = (const char**)malloc(argCount*sizeof(const char*));
|
||||
unsigned filenameIdx = 0;
|
||||
filenameTable[0] = stdinmark;
|
||||
unsigned forceStdout = 0;
|
||||
unsigned providedInitialCLevel = 0;
|
||||
int ret = 0;
|
||||
int argNum;
|
||||
filenameTable[0] = stdinmark;
|
||||
g_maxCLevel = ZSTD_maxCLevel();
|
||||
|
||||
UTIL_initTimer(&g_ticksPerSecond);
|
||||
|
||||
@ -1018,6 +1028,7 @@ int main(int argCount, const char* argv[])
|
||||
case 'i':
|
||||
argument += 2;
|
||||
g_compressionLevel = readU32FromChar(&argument);
|
||||
providedInitialCLevel = 1;
|
||||
break;
|
||||
case 'h':
|
||||
help();
|
||||
@ -1036,6 +1047,14 @@ int main(int argCount, const char* argv[])
|
||||
g_useProgressBar = 0;
|
||||
g_displayLevel = 0;
|
||||
break;
|
||||
case 'l':
|
||||
argument += 2;
|
||||
g_minCLevel = readU32FromChar(&argument);
|
||||
break;
|
||||
case 'u':
|
||||
argument += 2;
|
||||
g_maxCLevel = readU32FromChar(&argument);
|
||||
break;
|
||||
default:
|
||||
DISPLAY("Error: invalid argument provided\n");
|
||||
ret = 1;
|
||||
@ -1048,6 +1067,20 @@ int main(int argCount, const char* argv[])
|
||||
filenameTable[filenameIdx++] = argument;
|
||||
}
|
||||
|
||||
/* check initial, max, and min compression levels */
|
||||
{
|
||||
unsigned const minMaxInconsistent = g_minCLevel > g_maxCLevel;
|
||||
unsigned const initialNotInRange = g_minCLevel > g_compressionLevel || g_maxCLevel < g_compressionLevel;
|
||||
if (minMaxInconsistent || (initialNotInRange && providedInitialCLevel)) {
|
||||
DISPLAY("Error: provided compression level parameters are invalid\n");
|
||||
ret = 1;
|
||||
goto _main_exit;
|
||||
}
|
||||
else if (initialNotInRange) {
|
||||
g_compressionLevel = g_minCLevel;
|
||||
}
|
||||
}
|
||||
|
||||
/* error checking with number of files */
|
||||
if (filenameIdx > 1 && (outFilename != NULL && strcmp(outFilename, stdoutmark))) {
|
||||
DISPLAY("Error: multiple input files provided, cannot use specified output file\n");
|
||||
|
||||
@ -242,4 +242,11 @@ echo -e "\ncorrectness tests -- window size test"
|
||||
./datagen -s39 -g1GB | pv -L 25m -q | ./adapt -i1 | pv -q > tmp.zst
|
||||
zstd -d tmp.zst
|
||||
rm tmp*
|
||||
|
||||
echo -e "\ncorrectness tests -- testing bounds"
|
||||
./datagen -s40 -g1GB | pv -L 25m -q | ./adapt -i1 -u4 | pv -q > tmp.zst
|
||||
rm tmp*
|
||||
|
||||
./datagen -s41 -g1GB | ./adapt -i14 -l4 > tmp.zst
|
||||
rm tmp*
|
||||
make clean
|
||||
|
||||
Loading…
Reference in New Issue
Block a user