From d41f707e884f54bd2662b759b2bb96c55f9a12f0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 24 Mar 2017 17:56:45 -0700 Subject: [PATCH 1/7] minor improvement : remove duplicates with 1 char prefix difference --- lib/dictBuilder/zdict.c | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index ed53197a..1b3c74f1 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -361,14 +361,28 @@ static dictItem ZDICT_analyzePos( } +static int isIncluded(const void* in, const void* container, size_t length) +{ + const char* const ip = (const char*) in; + const char* const into = (const char*) container; + size_t u; + + for (u=0; upos; const U32 eltEnd = elt.pos + elt.length; + const char* const buf = buffer; /* tail overlap */ U32 u; for (u=1; u= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */ /* append */ int addedLength = (int)eltEnd - (table[u].pos + table[u].length); @@ -405,7 +420,18 @@ static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip) table[u] = table[u-1], u--; table[u] = elt; return u; - } } + } + + if (MEM_read64(buf + table[u].pos) == MEM_read64(buf + elt.pos + 1)) { + if (isIncluded(buf + table[u].pos, buf + elt.pos + 1, table[u].length)) { + size_t const addedLength = MAX( (int)elt.length - (int)table[u].length , 1 ); + table[u].pos = elt.pos; + table[u].savings += elt.savings * addedLength / elt.length; + table[u].length = MIN(elt.length, table[u].length + 1); + return u; + } + } + } return 0; } @@ -423,14 +449,14 @@ static void ZDICT_removeDictItem(dictItem* table, U32 id) } -static void ZDICT_insertDictItem(dictItem* table, U32 maxSize, dictItem elt) +static void ZDICT_insertDictItem(dictItem* table, U32 maxSize, dictItem elt, const void* buffer) { /* merge if possible */ - U32 mergeId = ZDICT_checkMerge(table, elt, 0); + U32 mergeId = ZDICT_checkMerge(table, elt, 0, buffer); if (mergeId) { U32 newMerge = 1; while (newMerge) { - newMerge = ZDICT_checkMerge(table, table[mergeId], mergeId); + newMerge = ZDICT_checkMerge(table, table[mergeId], mergeId, buffer); if (newMerge) ZDICT_removeDictItem(table, mergeId); mergeId = newMerge; } @@ -519,7 +545,7 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize, if (doneMarks[cursor]) { cursor++; continue; } solution = ZDICT_analyzePos(doneMarks, suffix, reverseSuffix[cursor], buffer, minRatio, notificationLevel); if (solution.length==0) { cursor++; continue; } - ZDICT_insertDictItem(dictList, dictListSize, solution); + ZDICT_insertDictItem(dictList, dictListSize, solution, buffer); cursor += solution.length; DISPLAYUPDATE(2, "\r%4.2f %% \r", (double)cursor / bufferSize * 100); } } From ecee9f2ef8ec41ad5fa5bf8f23f430d094722e52 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 26 Mar 2017 00:59:14 -0700 Subject: [PATCH 2/7] fixed conversion warnings --- lib/dictBuilder/zdict.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 1b3c74f1..955d98aa 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -382,7 +382,7 @@ static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, cons { const U32 tableSize = table->pos; const U32 eltEnd = elt.pos + elt.length; - const char* const buf = buffer; + const char* const buf = (const char*) buffer; /* tail overlap */ U32 u; for (u=1; u Date: Sun, 26 Mar 2017 02:50:00 -0700 Subject: [PATCH 3/7] fixed dictBuilder issue dictionary loading would fail during entropy analysis --- lib/dictBuilder/zdict.c | 51 +++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 955d98aa..0cd9cc88 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -393,7 +393,7 @@ static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, cons table[u].length += addedLength; table[u].pos = elt.pos; table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */ - table[u].savings += elt.length / 8; /* rough approx bonus */ + // table[u].savings += elt.length / 8; /* rough approx bonus */ elt = table[u]; /* sort : improve rank */ while ((u>1) && (table[u-1].savings < elt.savings)) @@ -422,14 +422,13 @@ static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, cons return u; } - if (MEM_read64(buf + table[u].pos) == MEM_read64(buf + elt.pos + 1)) { - if (isIncluded(buf + table[u].pos, buf + elt.pos + 1, table[u].length)) { - size_t const addedLength = MAX( (int)elt.length - (int)table[u].length , 1 ); - table[u].pos = elt.pos; - table[u].savings += (U32)(elt.savings * addedLength / elt.length); - table[u].length = MIN(elt.length, table[u].length + 1); - return u; - } + if ( (MEM_read64(buf + table[u].pos) == MEM_read64(buf + elt.pos + 1)) + && (isIncluded(buf + table[u].pos, buf + elt.pos + 1, table[u].length)) ) { + size_t const addedLength = MAX( (int)elt.length - (int)table[u].length , 1 ); + table[u].pos = elt.pos; + table[u].savings += (U32)(elt.savings * addedLength / elt.length); + table[u].length = MIN(elt.length, table[u].length + 1); + return u; } } @@ -707,19 +706,19 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, goto _cleanup; } if (offcodeMax>OFFCODE_MAX) { eSize = ERROR(dictionary_wrong); goto _cleanup; } /* too large dictionary */ - for (u=0; u<256; u++) countLit[u]=1; /* any character must be described */ - for (u=0; u<=offcodeMax; u++) offcodeCount[u]=1; - for (u=0; u<=MaxML; u++) matchLengthCount[u]=1; - for (u=0; u<=MaxLL; u++) litLengthCount[u]=1; + for (u=0; u<256; u++) countLit[u] = 1; /* any character must be described */ + for (u=0; u<=offcodeMax; u++) offcodeCount[u] = 1; + for (u=0; u<=MaxML; u++) matchLengthCount[u] = 1; + for (u=0; u<=MaxLL; u++) litLengthCount[u] = 1; memset(repOffset, 0, sizeof(repOffset)); repOffset[1] = repOffset[4] = repOffset[8] = 1; memset(bestRepOffset, 0, sizeof(bestRepOffset)); - if (compressionLevel==0) compressionLevel=g_compressionLevel_default; + if (compressionLevel==0) compressionLevel = g_compressionLevel_default; params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize); { size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0); - if (ZSTD_isError(beginResult)) { + if (ZSTD_isError(beginResult)) { + DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced() failed : %s \n", ZSTD_getErrorName(beginResult)); eSize = ERROR(GENERIC); - DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced failed \n"); goto _cleanup; } } @@ -901,20 +900,11 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, ZDICT_params_t params) { - size_t hSize; int const compressionLevel = (params.compressionLevel <= 0) ? g_compressionLevel_default : params.compressionLevel; U32 const notificationLevel = params.notificationLevel; + size_t hSize = 8; - /* dictionary header */ - MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC); - { U64 const randomID = XXH64((char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, 0); - U32 const compliantID = (randomID % ((1U<<31)-32768)) + 32768; - U32 const dictID = params.dictID ? params.dictID : compliantID; - MEM_writeLE32((char*)dictBuffer+4, dictID); - } - hSize = 8; - - /* entropy tables */ + /* calculate entropy tables */ DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */ DISPLAYLEVEL(2, "statistics ... \n"); { size_t const eSize = ZDICT_analyzeEntropy((char*)dictBuffer+hSize, dictBufferCapacity-hSize, @@ -926,6 +916,13 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo hSize += eSize; } + /* add dictionary header (after entropy tables) */ + MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC); + { U64 const randomID = XXH64((char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, 0); + U32 const compliantID = (randomID % ((1U<<31)-32768)) + 32768; + U32 const dictID = params.dictID ? params.dictID : compliantID; + MEM_writeLE32((char*)dictBuffer+4, dictID); + } if (hSize + dictContentSize < dictBufferCapacity) memmove((char*)dictBuffer + hSize, (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize); From 582760818f2573106dee003d779cafd4f46a7420 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 26 Mar 2017 02:59:13 -0700 Subject: [PATCH 4/7] minor refactor add const changed if for easier to add new conditions --- lib/dictBuilder/zdict.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 0cd9cc88..2c87084a 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -389,7 +389,7 @@ static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, cons if (u==eltNbToSkip) continue; if ((table[u].pos > elt.pos) && (table[u].pos <= eltEnd)) { /* overlap, existing > new */ /* append */ - U32 addedLength = table[u].pos - elt.pos; + U32 const addedLength = table[u].pos - elt.pos; table[u].length += addedLength; table[u].pos = elt.pos; table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */ @@ -408,7 +408,7 @@ static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, cons if ((table[u].pos + table[u].length >= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */ /* append */ - int addedLength = (int)eltEnd - (table[u].pos + table[u].length); + int const addedLength = (int)eltEnd - (table[u].pos + table[u].length); table[u].savings += elt.length / 8; /* rough approx bonus */ if (addedLength > 0) { /* otherwise, elt fully included into existing */ table[u].length += addedLength; @@ -422,13 +422,14 @@ static U32 ZDICT_checkMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, cons return u; } - if ( (MEM_read64(buf + table[u].pos) == MEM_read64(buf + elt.pos + 1)) - && (isIncluded(buf + table[u].pos, buf + elt.pos + 1, table[u].length)) ) { - size_t const addedLength = MAX( (int)elt.length - (int)table[u].length , 1 ); - table[u].pos = elt.pos; - table[u].savings += (U32)(elt.savings * addedLength / elt.length); - table[u].length = MIN(elt.length, table[u].length + 1); - return u; + if (MEM_read64(buf + table[u].pos) == MEM_read64(buf + elt.pos + 1)) { + if (isIncluded(buf + table[u].pos, buf + elt.pos + 1, table[u].length)) { + size_t const addedLength = MAX( (int)elt.length - (int)table[u].length , 1 ); + table[u].pos = elt.pos; + table[u].savings += (U32)(elt.savings * addedLength / elt.length); + table[u].length = MIN(elt.length, table[u].length + 1); + return u; + } } } From 4cf0093571f2aece4ae6f5d1047d8633323aac95 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 26 Mar 2017 14:51:00 -0700 Subject: [PATCH 5/7] restored bonus rule --- lib/dictBuilder/zdict.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index cd2e3a2f..bee5e059 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -393,7 +393,7 @@ static U32 ZDICT_tryMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, const table[u].length += addedLength; table[u].pos = elt.pos; table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */ - // table[u].savings += elt.length / 8; /* rough approx bonus */ + table[u].savings += elt.length / 8; /* rough approx bonus */ elt = table[u]; /* sort : improve rank */ while ((u>1) && (table[u-1].savings < elt.savings)) From 894bf4971374be233df7e954aabb3b19116c1ac7 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Mon, 27 Mar 2017 12:19:30 -0700 Subject: [PATCH 6/7] Fix IS_CONSOLE returning 1 for NUL on windows --- programs/platform.h | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/programs/platform.h b/programs/platform.h index 89a9f6cd..819bec87 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -100,9 +100,18 @@ extern "C" { #if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__) # include /* isatty */ # define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) -#elif defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) +#elif defined(MSDOS) || defined(OS2) || defined(__CYGWIN__) # include /* _isatty */ # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#elif defined(WIN32) || defined(_WIN32) +# include /* _isatty */ +# include /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ +# include /* FILE */ +static inline int IS_CONSOLE(FILE* stdStream) +{ + DWORD dummy; + return _isatty(_fileno(stdStream)) && GetConsoleMode((HANDLE)_get_osfhandle(_fileno(stdStream)), &dummy); +} #else # define IS_CONSOLE(stdStream) 0 #endif From efdaf8bb7ca36c80be96226775927b69dfa0cc11 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Mon, 27 Mar 2017 12:26:40 -0700 Subject: [PATCH 7/7] Fix inline compile errors --- programs/platform.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/platform.h b/programs/platform.h index 819bec87..ee781951 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -107,7 +107,7 @@ extern "C" { # include /* _isatty */ # include /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ # include /* FILE */ -static inline int IS_CONSOLE(FILE* stdStream) +static __inline int IS_CONSOLE(FILE* stdStream) { DWORD dummy; return _isatty(_fileno(stdStream)) && GetConsoleMode((HANDLE)_get_osfhandle(_fileno(stdStream)), &dummy);