AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

paramgrill can select manually targetLength

Browse Source 
new compression levels
This commit is contained in:
Yann Collet 2016-02-11 06:23:24 +01:00
parent bd828d9cd1
commit 04b12d8bcb
3 changed files with 32 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
lib/zstd_compress.c
View File

@ -903,7 +903,7 @@ static size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLim
return (size_t)(pIn - pStart); return (size_t)(pIn - pStart);
} }
/** ZSTD_count_2segments /** ZSTD_count_2segments() :
* can count match length with ip & match in potentially 2 different segments. * can count match length with ip & match in potentially 2 different segments.
* convention : on reaching mEnd, match count continue starting from iStart * convention : on reaching mEnd, match count continue starting from iStart
*/ */
@ -1190,11 +1190,11 @@ void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx,
} }
/* ************************************* /*-*************************************
* Binary Tree search * Binary Tree search
***************************************/ ***************************************/
/** ZSTD_insertBt1() : add one or multiple positions to tree /** ZSTD_insertBt1() : add one or multiple positions to tree.
* ip : assumed <= iend-8 * ip : assumed <= iend-8 .
* @return : nb of positions added */ * @return : nb of positions added */
static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares, static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares,
U32 extDict) U32 extDict)
@ -1230,7 +1230,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
while (nbCompares-- && (matchIndex > windowLow)) { while (nbCompares-- && (matchIndex > windowLow)) {
U32* nextPtr = bt + 2*(matchIndex & btMask); U32* nextPtr = bt + 2*(matchIndex & btMask);
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
#if 1 /* note : can create issues when hlog small <= 11 */
const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */ const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */
if (matchIndex == predictedSmall) { if (matchIndex == predictedSmall) {
/* no need to check length, result known */ /* no need to check length, result known */
@ -1249,7 +1249,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
predictedLarge = predictPtr[0] + (predictPtr[0]>0); predictedLarge = predictPtr[0] + (predictPtr[0]>0);
continue; continue;
} }
#endif
if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { if ((!extDict) || (matchIndex+matchLength >= dictLimit)) {
match = base + matchIndex; match = base + matchIndex;
if (match[matchLength] == ip[matchLength]) if (match[matchLength] == ip[matchLength])
@ -1284,7 +1284,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
} } } }
*smallerPtr = *largerPtr = 0; *smallerPtr = *largerPtr = 0;
return (matchEndIdx > current + 8) ? matchEndIdx - current - 8 : 1; return (matchEndIdx > current + 8) ? (matchEndIdx - current) - 8 : 1;
} }
@ -2339,12 +2339,12 @@ static const ZSTD_parameters ZSTD_defaultParameters[4][ZSTD_MAX_CLEVEL+1] = {
{ 0, 0, 23, 24, 23, 4, 5, ZSTD_btlazy2 }, /* level 17 */ { 0, 0, 23, 24, 23, 4, 5, ZSTD_btlazy2 }, /* level 17 */
{ 0, 0, 25, 24, 23, 5, 5, ZSTD_btlazy2 }, /* level 18 */ { 0, 0, 25, 24, 23, 5, 5, ZSTD_btlazy2 }, /* level 18 */
{ 0, 0, 25, 26, 23, 5, 5, ZSTD_btlazy2 }, /* level 19 */ { 0, 0, 25, 26, 23, 5, 5, ZSTD_btlazy2 }, /* level 19 */
{ 0, 0, 26, 27, 25, 9, 5, ZSTD_btlazy2 }, /* level 20 */ { 0, 12, 22, 20, 21, 3, 5, ZSTD_opt }, /* level 20 */
{ 0, 0, 23, 21, 22, 5, 4, ZSTD_btlazy2 }, /* level 21 = 16 + L=4 */ // 41233150 btlazy1=41560211 norep1=42322286 { 0, 16, 23, 21, 22, 4, 4, ZSTD_opt }, /* level 21 */
{ 0, 12, 23, 21, 22, 5, 4, ZSTD_opt }, /* level 22 */ { 0, 32, 25, 25, 24, 5, 4, ZSTD_opt_bt }, /* level 22 */
{ 0, 32, 23, 21, 22, 5, 4, ZSTD_opt }, /* level 23 */ { 0, 64, 25, 26, 24, 6, 4, ZSTD_opt_bt }, /* level 23 */
{ 0, 32, 23, 21, 22, 5, 4, ZSTD_opt_bt }, /* level 24 = 16 + btopt */ { 0,128, 26, 26, 25, 8, 4, ZSTD_opt_bt }, /* level 24 */
{ 0, 64, 26, 27, 25, 10, 4, ZSTD_opt_bt }, /* level 25 = 20 + btopt */ { 0,256, 26, 27, 25, 10, 4, ZSTD_opt_bt }, /* level 25 */
}, },
{ /* for srcSize <= 256 KB */ { /* for srcSize <= 256 KB */
/* SL, W, C, H, S, L, strat */ /* SL, W, C, H, S, L, strat */

16
lib/zstd_opt.h
View File

@ -269,7 +269,7 @@ U32 ZSTD_BtGetAllMatches (
} }
FORCE_INLINE U32 ZSTD_BtGetAllMatches_selectMLS ( static U32 ZSTD_BtGetAllMatches_selectMLS (
ZSTD_CCtx* zc, /* Index table will be updated */ ZSTD_CCtx* zc, /* Index table will be updated */
const BYTE* ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit, const BYTE* ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit,
const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, U32 minml) const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, U32 minml)
@ -297,7 +297,7 @@ U32 ZSTD_BtGetAllMatches_extDict (
} }
FORCE_INLINE U32 ZSTD_BtGetAllMatches_selectMLS_extDict ( static U32 ZSTD_BtGetAllMatches_selectMLS_extDict (
ZSTD_CCtx* zc, /* Index table will be updated */ ZSTD_CCtx* zc, /* Index table will be updated */
const BYTE* ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit, const BYTE* ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit,
const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, U32 minml) const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, U32 minml)
@ -382,7 +382,7 @@ U32 ZSTD_HcGetAllMatches_generic (
} }
FORCE_INLINE U32 ZSTD_HcGetAllMatches_selectMLS ( static U32 ZSTD_HcGetAllMatches_selectMLS (
ZSTD_CCtx* zc, ZSTD_CCtx* zc,
const BYTE* ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit, const BYTE* ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit,
const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, U32 minml) const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, U32 minml)
@ -396,7 +396,7 @@ FORCE_INLINE U32 ZSTD_HcGetAllMatches_selectMLS (
} }
} }
FORCE_INLINE U32 ZSTD_HcGetAllMatches_selectMLS_extDict ( static U32 ZSTD_HcGetAllMatches_selectMLS_extDict (
ZSTD_CCtx* zc, ZSTD_CCtx* zc,
const BYTE* ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit, const BYTE* ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit,
const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, U32 minml) const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, U32 minml)
@ -769,7 +769,6 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set
} }
FORCE_INLINE FORCE_INLINE
void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
const void* src, size_t srcSize, const void* src, size_t srcSize,
@ -803,7 +802,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
const U32 sufficient_len = ctx->params.targetLength; const U32 sufficient_len = ctx->params.targetLength;
const U32 faster_get_matches = (ctx->params.strategy == ZSTD_opt); const U32 faster_get_matches = (ctx->params.strategy == ZSTD_opt);
/* init */ /* init */
ZSTD_resetSeqStore(seqStorePtr); ZSTD_resetSeqStore(seqStorePtr);
if ((ip - prefixStart) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE; if ((ip - prefixStart) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE;
@ -1055,14 +1053,12 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set
cur -= mlen; cur -= mlen;
} }
for (u = 0; u <= last_pos;) { for (u = 0; u <= last_pos; ) {
ZSTD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep, opt[u].rep2); ZSTD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep, opt[u].rep2);
u += opt[u].mlen; u += opt[u].mlen;
} }
cur = 0; for (cur=0; cur < last_pos; ) {
while (cur < last_pos) {
U32 litLength; U32 litLength;
ZSTD_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep, opt[cur].rep2); ZSTD_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep, opt[cur].rep2);
mlen = opt[cur].mlen; mlen = opt[cur].mlen;

22
programs/paramgrill.c
View File

@ -930,7 +930,7 @@ int optimizeForSize(char* inFileName)
} }
int usage(char* exename) static int usage(char* exename)
{ {
DISPLAY( "Usage :\n"); DISPLAY( "Usage :\n");
DISPLAY( " %s [arg] file\n", exename); DISPLAY( " %s [arg] file\n", exename);
@ -940,16 +940,17 @@ int usage(char* exename)
return 0; return 0;
} }
int usage_advanced(void) static int usage_advanced(void)
{ {
DISPLAY( "\nAdvanced options :\n"); DISPLAY( "\nAdvanced options :\n");
DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS); DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS);
DISPLAY( " -B# : cut input into blocks of size # (default : single block)\n"); DISPLAY( " -B# : cut input into blocks of size # (default : single block)\n");
DISPLAY( " -P# : generated sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100); DISPLAY( " -P# : generated sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100);
DISPLAY( " -S : Single run\n");
return 0; return 0;
} }
int badusage(char* exename) static int badusage(char* exename)
{ {
DISPLAY("Wrong parameters\n"); DISPLAY("Wrong parameters\n");
usage(exename); usage(exename);
@ -1064,13 +1065,16 @@ int main(int argc, char** argv)
while ((*argument>= '0') && (*argument<='9')) while ((*argument>= '0') && (*argument<='9'))
g_params.searchLength *= 10, g_params.searchLength += *argument++ - '0'; g_params.searchLength *= 10, g_params.searchLength += *argument++ - '0';
continue; continue;
case 't': /* strategy */ case 't': /* target length */
g_params.strategy = (ZSTD_strategy)0; g_params.targetLength = 0;
argument++; argument++;
while ((*argument>= '0') && (*argument<='9')) { while ((*argument>= '0') && (*argument<='9'))
g_params.strategy = (ZSTD_strategy)((U32)g_params.strategy *10); g_params.targetLength *= 10, g_params.targetLength += *argument++ - '0';
g_params.strategy = (ZSTD_strategy)((U32)g_params.strategy + *argument++ - '0'); continue;
} case 'S': /* strategy */
argument++;
while ((*argument>= '0') && (*argument<='9'))
g_params.strategy = (ZSTD_strategy)(*argument++ - '0');
continue; continue;
case 'L': case 'L':
{ {