From 3dcfe5cc2c19b5a4fd864c5d1c40aec4a9e5f1dc Mon Sep 17 00:00:00 2001 From: George Lu Date: Tue, 14 Aug 2018 14:24:05 -0700 Subject: [PATCH 1/6] begin display changes --- tests/paramgrill.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 4a7fc387..c56f0a71 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -53,6 +53,8 @@ static const int g_maxNbVariations = 64; * Macros **************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(n, ...) if(g_displayLevel >= n) { fprintf(stderr, __VA_ARGS__); } + #define TIMED 0 #ifndef DEBUG # define DEBUG 0 @@ -233,6 +235,7 @@ static U32 g_noSeed = 0; static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */ static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ static U32 g_memoTableLog = PARAM_UNSET; +static U32 g_displayLevel = 3; typedef enum { directMap, @@ -1282,7 +1285,6 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res /* the table */ fprintf(f, "================================\n"); for(n = g_winners; n != NULL; n = n->next) { - fprintf(f, "\r%79s\r", ""); BMK_displayOneResult(f, n->res, srcSize); } fprintf(f, "================================\n"); @@ -2313,7 +2315,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); - BMK_translateAdvancedParams(stdout, winner.params); } } @@ -2364,7 +2365,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* end summary */ BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); - BMK_translateAdvancedParams(stdout, winner.params); DISPLAY("grillParams size - optimizer completed \n"); } @@ -2653,6 +2653,14 @@ int main(int argc, const char** argv) seperateFiles = 1; break; + case 'q': + g_displayLevel--; + break; + + case 'v': + g_displayLevel++; + break; + /* load dictionary file (only applicable for optimizer rn) */ case 'D': if(i == argc - 1) { /* last argument, return error. */ From 4d9c6f51b8f34970fc292d82f23a2fa7f3bdede0 Mon Sep 17 00:00:00 2001 From: George Lu Date: Tue, 14 Aug 2018 15:54:07 -0700 Subject: [PATCH 2/6] -q -v options --- tests/paramgrill.c | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index c56f0a71..ef6d6d46 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -1257,22 +1257,22 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res /* global winner used for constraints */ /* cSize, cSpeed, dSpeed, cMem */ static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } }; - if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { + if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize) || g_displayLevel >= 4) { if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { DISPLAY("New Winner: \n"); } - BMK_printWinner(f, cLevel, result, params, srcSize); + if(g_displayLevel >= 2) { BMK_printWinner(f, cLevel, result, params, srcSize); } if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { - BMK_translateAdvancedParams(f, params); + if(g_displayLevel >= 1) { BMK_translateAdvancedParams(f, params); } g_winner.result = result; g_winner.params = params; } } //prints out tradeoff table if using lvloptimize - if(g_optmode && g_optimizer) { + if(g_optmode && g_optimizer && (DEBUG || g_displayLevel == 3)) { winnerInfo_t w; winner_ll_node* n; w.result = result; @@ -2205,9 +2205,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } if(nbFiles == 1) { - DISPLAY("Loading %s... \r", fileNamesTable[0]); + DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[0]); } else { - DISPLAY("Loading %lu Files... \r", (unsigned long)nbFiles); + DISPLAYLEVEL(2, "Loading %lu Files... \r", (unsigned long)nbFiles); } /* sanitize paramTarget */ @@ -2273,16 +2273,16 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* bench */ - DISPLAY("\r%79s\r", ""); + DISPLAYLEVEL(2, "\r%79s\r", ""); if(nbFiles == 1) { - DISPLAY("optimizing for %s", fileNamesTable[0]); + DISPLAYLEVEL(2, "optimizing for %s", fileNamesTable[0]); } else { - DISPLAY("optimizing for %lu Files", (unsigned long)nbFiles); + DISPLAYLEVEL(2, "optimizing for %lu Files", (unsigned long)nbFiles); } - if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed >> 20); } - if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed >> 20); } - if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem >> 20); } + if(target.cSpeed != 0) { DISPLAYLEVEL(2," - limit compression speed %u MB/s", target.cSpeed >> 20); } + if(target.dSpeed != 0) { DISPLAYLEVEL(2, " - limit decompression speed %u MB/s", target.dSpeed >> 20); } + if(target.cMem != (U32)-1) { DISPLAYLEVEL(2, " - limit memory %u MB", target.cMem >> 20); } DISPLAY("\n"); findClockGranularity(); @@ -2364,7 +2364,8 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } /* end summary */ - BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); + BMK_displayOneResult(stdout, winner, buf.srcSize); + BMK_translateAdvancedParams(stdout, winner.params); DISPLAY("grillParams size - optimizer completed \n"); } @@ -2501,7 +2502,7 @@ int main(int argc, const char** argv) assert(argc>=1); /* for exename */ /* Welcome message */ - DISPLAY(WELCOME_MESSAGE); + DISPLAYLEVEL(2, WELCOME_MESSAGE); for(i=1; i Date: Tue, 14 Aug 2018 16:51:39 -0700 Subject: [PATCH 3/6] Clean up repetitive display Add documentation --- tests/README.md | 2 ++ tests/paramgrill.c | 28 +++++++++++++--------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/README.md b/tests/README.md index bdc9fff9..60fcebfb 100644 --- a/tests/README.md +++ b/tests/README.md @@ -127,6 +127,8 @@ Full list of arguments -v : Prints Benchmarking output -D : Next argument dictionary file -s : Benchmark all files separately + -q : Quiet, repeat for more quiet + -v : Verbose, cancels quiet, repeat for more volume ``` Any inputs afterwards are treated as files to benchmark. diff --git a/tests/paramgrill.c b/tests/paramgrill.c index ef6d6d46..ef005dd9 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -235,7 +235,7 @@ static U32 g_noSeed = 0; static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */ static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ static U32 g_memoTableLog = PARAM_UNSET; -static U32 g_displayLevel = 3; +static int g_displayLevel = 3; typedef enum { directMap, @@ -889,7 +889,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab goto _cleanUp; } - DISPLAY("Loading %s... \r", fileNamesTable[n]); + DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]); if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n; /* buffer too small - stop after this file */ { @@ -2020,7 +2020,6 @@ static winnerInfo_t climbOnce(const constraint_t target, better = 0; DEBUGOUTPUT("Start\n"); cparam = winnerInfo.params; - BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); candidateInfo.params = cparam; /* all dist-1 candidates */ for(i = 0; i < varLen; i++) { @@ -2036,7 +2035,6 @@ static winnerInfo_t climbOnce(const constraint_t target, DEBUGOUTPUT("Res: %d\n", res); if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */ winnerInfo = candidateInfo; - BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize); better = 1; if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { bestFeasible1 = winnerInfo; @@ -2063,7 +2061,6 @@ static winnerInfo_t climbOnce(const constraint_t target, DEBUGOUTPUT("Res: %d\n", res); if(res == BETTER_RESULT) { /* synonymous with better in this case*/ winnerInfo = candidateInfo; - BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize); better = 1; if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { bestFeasible1 = winnerInfo; @@ -2284,7 +2281,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ if(target.dSpeed != 0) { DISPLAYLEVEL(2, " - limit decompression speed %u MB/s", target.dSpeed >> 20); } if(target.cMem != (U32)-1) { DISPLAYLEVEL(2, " - limit memory %u MB", target.cMem >> 20); } - DISPLAY("\n"); + DISPLAYLEVEL(2, "\n"); findClockGranularity(); { @@ -2309,7 +2306,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ winner.params = CParams; } - CHECKTIMEGT(ret, 0, _cleanUp); /* if pass time limit, stop */ + CHECKTIMEGT(ret, 0, _displayCleanUp); /* if pass time limit, stop */ /* if the current params are too slow, just stop. */ if(target.cSpeed > candidate.cSpeed * 3 / 2) { break; } } @@ -2332,7 +2329,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) { winner = w1; } - CHECKTIMEGT(ret, 0, _cleanUp); + CHECKTIMEGT(ret, 0, _displayCleanUp); } while(st && tries > 0) { @@ -2349,7 +2346,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ st = nextStrategy(st, bestStrategy); tries -= TRY_DECAY; } - CHECKTIMEGT(ret, 0, _cleanUp); + CHECKTIMEGT(ret, 0, _displayCleanUp); } } else { winner = optimizeFixedStrategy(buf, ctx, target, paramBase, paramTarget.vals[strt_ind], allMT, g_maxTries); @@ -2364,12 +2361,13 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } /* end summary */ - BMK_displayOneResult(stdout, winner, buf.srcSize); +_displayCleanUp: + if(g_displayLevel >= 0) { BMK_displayOneResult(stdout, winner, buf.srcSize); } BMK_translateAdvancedParams(stdout, winner.params); - DISPLAY("grillParams size - optimizer completed \n"); + DISPLAYLEVEL(1, "grillParams size - optimizer completed \n"); } -_cleanUp: +_cleanUp: freeContexts(ctx); freeBuffers(buf); freeMemoTableArray(allMT); @@ -2501,9 +2499,6 @@ int main(int argc, const char** argv) assert(argc>=1); /* for exename */ - /* Welcome message */ - DISPLAYLEVEL(2, WELCOME_MESSAGE); - for(i=1; i Date: Tue, 14 Aug 2018 18:04:58 -0700 Subject: [PATCH 4/6] silencing params --- tests/README.md | 1 + tests/paramgrill.c | 70 ++++++++++++++++++++++++++++++++++++---------- 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/tests/README.md b/tests/README.md index 60fcebfb..3af08f7b 100644 --- a/tests/README.md +++ b/tests/README.md @@ -122,6 +122,7 @@ Full list of arguments tries= : Maximum number of random restarts on a single strategy before switching (Default 3) Higher values will make optimizer run longer, more chances to find better solution. memLog : Limits the log of the size of each memotable (1 per strategy). Setting memLog = 0 turns off memoization + --display= : which params to display, uses all --zstd parameter names and 'cParams' to display only compression parameters. -P# : generated sample compressibility -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -v : Prints Benchmarking output diff --git a/tests/paramgrill.c b/tests/paramgrill.c index ef005dd9..2f00c198 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -237,6 +237,8 @@ static UTIL_time_t g_time; /* to be used to compare solution finding speeds to c static U32 g_memoTableLog = PARAM_UNSET; static int g_displayLevel = 3; +static BYTE g_silenceParams[NUM_PARAMS]; + typedef enum { directMap, xxhashMap, @@ -447,31 +449,34 @@ static paramValues_t cParamUnsetMin(paramValues_t paramTarget) { } static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) { - U32 i; + varInds_t v; + int first = 1; fprintf(f,"--zstd="); - for(i = 0; i < NUM_PARAMS; i++) { - fprintf(f,"%s=", g_paramNames[i]); + for(v = 0; v < NUM_PARAMS; v++) { + if(g_silenceParams[v]) { continue; } + if(!first) { fprintf(f, ","); } + fprintf(f,"%s=", g_paramNames[v]); - if(i == strt_ind) { fprintf(f,"%u", params.vals[i]); } - else { displayParamVal(f, i, params.vals[i], 0); } - - if(i != NUM_PARAMS - 1) { - fprintf(f, ","); - } + if(v == strt_ind) { fprintf(f,"%u", params.vals[v]); } + else { displayParamVal(f, v, params.vals[v], 0); } + first = 0; } fprintf(f, "\n"); } static void BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize) { varInds_t v; + int first = 1; res.params = cParamUnsetMin(res.params); fprintf(f," {"); for(v = 0; v < NUM_PARAMS; v++) { - if(v != 0) { fprintf(f, ","); } + if(g_silenceParams[v]) { continue; } + if(!first) { fprintf(f, ","); } displayParamVal(f, v, res.params.vals[v], 3); + first = 0; } - fprintf(f, "}, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + fprintf(f, " }, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", (double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB)); } @@ -2545,6 +2550,43 @@ int main(int argc, const char** argv) } continue; /* if not return, success */ + + } else if (longCommandWArg(&argument, "--display=")) { + /* Decode command (note : aggregated commands are allowed) */ + memset(g_silenceParams, 1, sizeof(g_silenceParams)); + for ( ; ;) { + int found = 0; + varInds_t v; + for(v = 0; v < NUM_PARAMS; v++) { + if(longCommandWArg(&argument, g_shortParamNames[v]) || longCommandWArg(&argument, g_paramNames[v])) { + g_silenceParams[v] = 0; + found = 1; + } + } + if(longCommandWArg(&argument, "compressionParameters") || longCommandWArg(&argument, "cParams")) { + for(v = 0; v <= strt_ind; v++) { + g_silenceParams[v] = 0; + } + found = 1; + } + + + if(found) { + if(argument[0]==',') { + continue; + } else { + break; + } + } + DISPLAY("invalid parameter name parameter \n"); + return 1; + } + + if (argument[0] != 0) { + DISPLAY("invalid --display format\n"); + return 1; /* check the end of string */ + } + continue; } else if (argument[0]=='-') { argument++; @@ -2650,13 +2692,11 @@ int main(int argc, const char** argv) break; case 'q': - argument++; - g_displayLevel--; + while (argument[0] == 'q') { argument++; g_displayLevel--; } break; case 'v': - argument++; - g_displayLevel++; + while (argument[0] == 'v') { argument++; g_displayLevel++; } break; /* load dictionary file (only applicable for optimizer rn) */ From ee77ddc28ddb0ff9bae63ee61e58a3ecfb315728 Mon Sep 17 00:00:00 2001 From: George Lu Date: Wed, 15 Aug 2018 11:46:19 -0700 Subject: [PATCH 5/6] Fix wraparound --- tests/paramgrill.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 2f00c198..0d814bf7 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -144,8 +144,8 @@ static const char* g_shortParamNames[NUM_PARAMS] = { "wlog", "clog", "hlog","slog", "slen", "tlen", "strt", "fadt" }; /* maps value from { 0 to rangetable[param] - 1 } to valid paramvalues */ -static U32 rangeMap(varInds_t param, U32 ind) { - ind = MIN(ind, rangetable[param] - 1); +static U32 rangeMap(varInds_t param, int ind) { + ind = MAX(MIN(ind, (int)rangetable[param] - 1), 0); switch(param) { case tlen_ind: return tlen_table[ind]; @@ -166,7 +166,7 @@ static U32 rangeMap(varInds_t param, U32 ind) { } /* inverse of rangeMap */ -static U32 invRangeMap(varInds_t param, U32 value) { +static int invRangeMap(varInds_t param, U32 value) { value = MIN(MAX(mintable[param], value), maxtable[param]); switch(param) { case tlen_ind: /* bin search */ @@ -186,7 +186,7 @@ static U32 invRangeMap(varInds_t param, U32 value) { return lo; } case fadt_ind: - return value + 1; + return (int)value + 1; case wlog_ind: case clog_ind: case hlog_ind: @@ -196,7 +196,7 @@ static U32 invRangeMap(varInds_t param, U32 value) { return value - mintable[param]; case NUM_PARAMS: DISPLAY("Error, not a valid param\n "); - return (U32)-1; + return -2; } return 0; /* should never happen, stop compiler warnings */ } @@ -1545,7 +1545,7 @@ static unsigned memoTableIndDirect(const paramValues_t* ptr, const varInds_t* va for(i = 0; i < varyLen; i++) { varInds_t v = varyParams[i]; if(v == strt_ind) continue; /* exclude strategy from memotable */ - ind *= rangetable[v]; ind += invRangeMap(v, ptr->vals[v]); + ind *= rangetable[v]; ind += (unsigned)invRangeMap(v, ptr->vals[v]); } return ind; } From b234870c33745401138af634909acf46ec3363fc Mon Sep 17 00:00:00 2001 From: George Lu Date: Wed, 15 Aug 2018 14:29:49 -0700 Subject: [PATCH 6/6] clarify display README --- tests/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 3af08f7b..442285db 100644 --- a/tests/README.md +++ b/tests/README.md @@ -122,7 +122,10 @@ Full list of arguments tries= : Maximum number of random restarts on a single strategy before switching (Default 3) Higher values will make optimizer run longer, more chances to find better solution. memLog : Limits the log of the size of each memotable (1 per strategy). Setting memLog = 0 turns off memoization - --display= : which params to display, uses all --zstd parameter names and 'cParams' to display only compression parameters. + --display= : specifiy which parameters are included in the output + can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters + (Default: display all params available) + -P# : generated sample compressibility -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -v : Prints Benchmarking output