AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev

Browse Source
This commit is contained in:
inikep 2016-10-05 12:17:51 +02:00
commit 6ac85a414a
2 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
NEWS
View File

@ -1,7 +1,9 @@
v1.1.0 v1.1.0
New : contrib/pzstd, parallel version of zstd, by Nick Terrell New : contrib/pzstd, parallel version of zstd, by Nick Terrell
added : NetBSD install target (#338) added : NetBSD install target (#338)
Improved : speed improvements for batches of small files. Improved : speed for batches of small files
Improved : speed of zlib wrapper, by Przemyslaw Skibinski
Changed : libzstd on Windows supports legacy formats, by Christophe Chevalier
Fixed : CLI -d output to stdout by default when input is stdin (#322) Fixed : CLI -d output to stdout by default when input is stdin (#322)
Fixed : CLI correctly detects console on Mac OS-X Fixed : CLI correctly detects console on Mac OS-X
Fixed : CLI supports recursive mode `-r` on Mac OS-X Fixed : CLI supports recursive mode `-r` on Mac OS-X

8
lib/dictBuilder/zdict.c
View File

@ -911,7 +911,7 @@ size_t ZDICT_trainFromBuffer_unsafe(
/* create dictionary */ /* create dictionary */
{ U32 dictContentSize = ZDICT_dictSize(dictList); { U32 dictContentSize = ZDICT_dictSize(dictList);
if (dictContentSize < targetDictSize/2) { if (dictContentSize < targetDictSize/3) {
DISPLAYLEVEL(2, "! warning : selected content significantly smaller than requested (%u < %u) \n", dictContentSize, (U32)maxDictSize); DISPLAYLEVEL(2, "! warning : selected content significantly smaller than requested (%u < %u) \n", dictContentSize, (U32)maxDictSize);
if (minRep > MINRATIO) { if (minRep > MINRATIO) {
DISPLAYLEVEL(2, "! consider increasing selectivity to produce larger dictionary (-s%u) \n", selectivity+1); DISPLAYLEVEL(2, "! consider increasing selectivity to produce larger dictionary (-s%u) \n", selectivity+1);
@ -921,12 +921,12 @@ size_t ZDICT_trainFromBuffer_unsafe(
DISPLAYLEVEL(2, "! consider increasing the number of samples (total size : %u MB)\n", (U32)(samplesBuffSize>>20)); DISPLAYLEVEL(2, "! consider increasing the number of samples (total size : %u MB)\n", (U32)(samplesBuffSize>>20));
} }
if ((dictContentSize > targetDictSize*2) && (nbSamples > 2*MINRATIO) && (selectivity>1)) { if ((dictContentSize > targetDictSize*3) && (nbSamples > 2*MINRATIO) && (selectivity>1)) {
U32 proposedSelectivity = selectivity-1; U32 proposedSelectivity = selectivity-1;
while ((nbSamples >> proposedSelectivity) <= MINRATIO) { proposedSelectivity--; } while ((nbSamples >> proposedSelectivity) <= MINRATIO) { proposedSelectivity--; }
DISPLAYLEVEL(2, "! note : calculated dictionary significantly larger than requested (%u > %u) \n", dictContentSize, (U32)maxDictSize); DISPLAYLEVEL(2, "! note : calculated dictionary significantly larger than requested (%u > %u) \n", dictContentSize, (U32)maxDictSize);
DISPLAYLEVEL(2, "! you may consider decreasing selectivity to produce denser dictionary (-s%u) \n", proposedSelectivity); DISPLAYLEVEL(2, "! consider increasing dictionary size, or produce denser dictionary (-s%u) \n", proposedSelectivity);
DISPLAYLEVEL(2, "! but test its efficiency on samples \n"); DISPLAYLEVEL(2, "! always test dictionary efficiency on samples \n");
} }
/* limit dictionary size */ /* limit dictionary size */