AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #631 from facebook/LegacyDictBuilder

Browse Source 
Legacy dict builder
This commit is contained in:
Yann Collet 2017-03-27 14:42:52 -07:00 committed by GitHub
commit 32ef985877
1 changed files with 55 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
lib/dictBuilder/zdict.c
View File

@ -361,21 +361,35 @@ static dictItem ZDICT_analyzePos(
}
/*! ZDICT_tryMerge() :
* check if dictItem can be merged, do it if possible
* @return : id of destination elt, 0 if not merged
*/
static U32 ZDICT_tryMerge(dictItem* table, dictItem elt, U32 eltNbToSkip)
static int isIncluded(const void* in, const void* container, size_t length)
{
const char* const ip = (const char*) in;
const char* const into = (const char*) container;
size_t u;
for (u=0; u<length; u++) { /* works because end of buffer is a noisy guard band */
if (ip[u] != into[u]) break;
}
return u==length;
}
/*! ZDICT_checkMerge
check if dictItem can be merged, do it if possible
@return : id of destination elt, 0 if not merged
*/
static U32 ZDICT_tryMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, const void* buffer)
{
const U32 tableSize = table->pos;
const U32 eltEnd = elt.pos + elt.length;
const char* const buf = (const char*) buffer;
/* tail overlap */
U32 u; for (u=1; u<tableSize; u++) {
if (u==eltNbToSkip) continue;
if ((table[u].pos > elt.pos) && (table[u].pos <= eltEnd)) { /* overlap, existing > new */
/* append */
U32 addedLength = table[u].pos - elt.pos;
U32 const addedLength = table[u].pos - elt.pos;
table[u].length += addedLength;
table[u].pos = elt.pos;
table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */
@ -391,9 +405,10 @@ static U32 ZDICT_tryMerge(dictItem* table, dictItem elt, U32 eltNbToSkip)
/* front overlap */
for (u=1; u<tableSize; u++) {
if (u==eltNbToSkip) continue;
if ((table[u].pos + table[u].length >= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */
/* append */
int addedLength = (int)eltEnd - (table[u].pos + table[u].length);
int const addedLength = (int)eltEnd - (table[u].pos + table[u].length);
table[u].savings += elt.length / 8; /* rough approx bonus */
if (addedLength > 0) { /* otherwise, elt fully included into existing */
table[u].length += addedLength;
@ -405,7 +420,18 @@ static U32 ZDICT_tryMerge(dictItem* table, dictItem elt, U32 eltNbToSkip)
table[u] = table[u-1], u--;
table[u] = elt;
return u;
} }
}
if (MEM_read64(buf + table[u].pos) == MEM_read64(buf + elt.pos + 1)) {
if (isIncluded(buf + table[u].pos, buf + elt.pos + 1, table[u].length)) {
size_t const addedLength = MAX( (int)elt.length - (int)table[u].length , 1 );
table[u].pos = elt.pos;
table[u].savings += (U32)(elt.savings * addedLength / elt.length);
table[u].length = MIN(elt.length, table[u].length + 1);
return u;
}
}
}
return 0;
}
@ -423,14 +449,14 @@ static void ZDICT_removeDictItem(dictItem* table, U32 id)
}
static void ZDICT_insertDictItem(dictItem* table, U32 maxSize, dictItem elt)
static void ZDICT_insertDictItem(dictItem* table, U32 maxSize, dictItem elt, const void* buffer)
{
/* merge if possible */
U32 mergeId = ZDICT_tryMerge(table, elt, 0);
if (mergeId) { /* recursive : re-merge the newly merged elt */
U32 mergeId = ZDICT_tryMerge(table, elt, 0, buffer);
if (mergeId) {
U32 newMerge = 1;
while (newMerge) {
newMerge = ZDICT_tryMerge(table, table[mergeId], mergeId); /* merge existing elt */
newMerge = ZDICT_tryMerge(table, table[mergeId], mergeId, buffer);
if (newMerge) ZDICT_removeDictItem(table, mergeId);
mergeId = newMerge;
}
@ -519,7 +545,7 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize,
if (doneMarks[cursor]) { cursor++; continue; }
solution = ZDICT_analyzePos(doneMarks, suffix, reverseSuffix[cursor], buffer, minRatio, notificationLevel);
if (solution.length==0) { cursor++; continue; }
ZDICT_insertDictItem(dictList, dictListSize, solution);
ZDICT_insertDictItem(dictList, dictListSize, solution, buffer);
cursor += solution.length;
DISPLAYUPDATE(2, "\r%4.2f %% \r", (double)cursor / bufferSize * 100);
} }
@ -681,19 +707,19 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
goto _cleanup;
}
if (offcodeMax>OFFCODE_MAX) { eSize = ERROR(dictionary_wrong); goto _cleanup; } /* too large dictionary */
for (u=0; u<256; u++) countLit[u]=1; /* any character must be described */
for (u=0; u<=offcodeMax; u++) offcodeCount[u]=1;
for (u=0; u<=MaxML; u++) matchLengthCount[u]=1;
for (u=0; u<=MaxLL; u++) litLengthCount[u]=1;
for (u=0; u<256; u++) countLit[u] = 1; /* any character must be described */
for (u=0; u<=offcodeMax; u++) offcodeCount[u] = 1;
for (u=0; u<=MaxML; u++) matchLengthCount[u] = 1;
for (u=0; u<=MaxLL; u++) litLengthCount[u] = 1;
memset(repOffset, 0, sizeof(repOffset));
repOffset[1] = repOffset[4] = repOffset[8] = 1;
memset(bestRepOffset, 0, sizeof(bestRepOffset));
if (compressionLevel==0) compressionLevel=g_compressionLevel_default;
if (compressionLevel==0) compressionLevel = g_compressionLevel_default;
params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize);
{ size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0);
if (ZSTD_isError(beginResult)) {
if (ZSTD_isError(beginResult)) {
DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced() failed : %s \n", ZSTD_getErrorName(beginResult));
eSize = ERROR(GENERIC);
DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced failed \n");
goto _cleanup;
} }
@ -874,20 +900,11 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
ZDICT_params_t params)
{
size_t hSize;
int const compressionLevel = (params.compressionLevel <= 0) ? g_compressionLevel_default : params.compressionLevel;
U32 const notificationLevel = params.notificationLevel;
size_t hSize = 8;
/* dictionary header */
MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC);
{ U64 const randomID = XXH64((char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, 0);
U32 const compliantID = (randomID % ((1U<<31)-32768)) + 32768;
U32 const dictID = params.dictID ? params.dictID : compliantID;
MEM_writeLE32((char*)dictBuffer+4, dictID);
}
hSize = 8;
/* entropy tables */
/* calculate entropy tables */
DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */
DISPLAYLEVEL(2, "statistics ... \n");
{ size_t const eSize = ZDICT_analyzeEntropy((char*)dictBuffer+hSize, dictBufferCapacity-hSize,
@ -899,6 +916,13 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo
hSize += eSize;
}
/* add dictionary header (after entropy tables) */
MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC);
{ U64 const randomID = XXH64((char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, 0);
U32 const compliantID = (randomID % ((1U<<31)-32768)) + 32768;
U32 const dictID = params.dictID ? params.dictID : compliantID;
MEM_writeLE32((char*)dictBuffer+4, dictID);
}
if (hSize + dictContentSize < dictBufferCapacity)
memmove((char*)dictBuffer + hSize, (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize);