AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

code cleaning

Browse Source
This commit is contained in:
inikep 2016-02-04 21:45:26 +01:00
parent 4fa4098628
commit b3df71e538
1 changed files with 27 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
lib/zstd_opt.c
View File

@ -31,7 +31,7 @@ typedef struct
#define ZSTD_LOG_TRY_PRICE(fmt, args...) ;//printf(fmt, ##args) #define ZSTD_LOG_TRY_PRICE(fmt, args...) ;//printf(fmt, ##args)
#define ZSTD_OPT_NUM (1<<12) #define ZSTD_OPT_NUM (1<<12)
#define ZSTD_FREQ_THRESHOLD (256)
const int tab32[32] = { const int tab32[32] = {
0, 9, 1, 10, 13, 21, 2, 29, 0, 9, 1, 10, 13, 21, 2, 29,
@ -52,19 +52,15 @@ int log2_32 (uint32_t value)
FORCE_INLINE size_t ZSTD_getLiteralPriceReal(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals) FORCE_INLINE size_t ZSTD_getLiteralPriceReal(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals)
{ {
size_t freq;
size_t price = 0; size_t price = 0;
size_t litBits, litLenBits;
// printf("litSum=%d litLengthSum=%d matchLengthSum=%d offCodeSum=%d\n", seqStorePtr->litSum, seqStorePtr->litLengthSum, seqStorePtr->matchLengthSum, seqStorePtr->offCodeSum);
/* literals */
litBits = 0;
if (litLength > 0) { if (litLength > 0) {
/* literals */
for (int i=litLength-1; i>=0; i--) for (int i=litLength-1; i>=0; i--)
// litBits += -log2((double)seqStorePtr->litFreq[literals[i]]/(double)seqStorePtr->litSum); price += log2_32(seqStorePtr->litSum) - log2_32(seqStorePtr->litFreq[literals[i]]);
litBits += log2_32(seqStorePtr->litSum) - log2_32(seqStorePtr->litFreq[literals[i]]);
/* literal Length */ /* literal Length */
size_t freq;
if (litLength >= MaxLL) { if (litLength >= MaxLL) {
freq = seqStorePtr->litLengthFreq[MaxLL]; freq = seqStorePtr->litLengthFreq[MaxLL];
if (litLength<255 + MaxLL) { if (litLength<255 + MaxLL) {
@ -73,53 +69,37 @@ FORCE_INLINE size_t ZSTD_getLiteralPriceReal(seqStore_t* seqStorePtr, size_t lit
price += 8; price += 8;
if (litLength < (1<<15)) price += 16; else price += 24; if (litLength < (1<<15)) price += 16; else price += 24;
} } } }
else freq = seqStorePtr->litLengthFreq[litLength]; else
// litLenBits = -log2((double)freq/(double)seqStorePtr->litLengthSum); freq = seqStorePtr->litLengthFreq[litLength];
litLenBits = log2_32(seqStorePtr->litLengthSum) - log2_32(freq); price += log2_32(seqStorePtr->litLengthSum) - log2_32(freq);
} }
else litLenBits = 0;
// freq = round(1.0f*(litBits + litLenBits + price)); return price + (price <= 0);
freq = litBits + litLenBits + price;
// printf("litLength=%d litBits=%.02f litLenBits=%.02f dumpsPrice=%d sum=%d\n", (int)litLength, litBits, litLenBits, (int)price, (int)freq);
// printf("old=%d new=%d\n", (int)((litLength<<3)+1), (int)freq/8);
if (freq <= 0) freq = 1;
return freq;
} }
FORCE_INLINE size_t ZSTD_getLiteralPrice(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals) FORCE_INLINE size_t ZSTD_getLiteralPrice(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals)
{ {
#if 1 if (seqStorePtr->litSum > ZSTD_FREQ_THRESHOLD)
return ZSTD_getLiteralPriceReal(seqStorePtr, litLength, literals); return ZSTD_getLiteralPriceReal(seqStorePtr, litLength, literals);
#else
size_t lit_cost = 1 + (litLength<<3); return 1 + (litLength<<3);
return lit_cost;
#endif
} }
FORCE_INLINE size_t ZSTD_getMatchPriceReal(seqStore_t* seqStorePtr, size_t offset, size_t matchLength) FORCE_INLINE size_t ZSTD_getMatchPriceReal(seqStore_t* seqStorePtr, size_t offset, size_t matchLength)
{ {
size_t freq;
size_t price = 0;
size_t offCodeBits, matchBits;
// printf("litSum=%d litLengthSum=%d matchLengthSum=%d offCodeSum=%d\n", seqStorePtr->litSum, seqStorePtr->litLengthSum, seqStorePtr->matchLengthSum, seqStorePtr->offCodeSum);
/* match offset */ /* match offset */
BYTE offCode = (BYTE)ZSTD_highbit(offset) + 1; BYTE offCode = (BYTE)ZSTD_highbit(offset) + 1;
if (offset==0) if (offset==0)
offCode = 0; offCode = 0;
// offCodeBits = -log2((double)seqStorePtr->offCodeFreq[offCode]/(double)seqStorePtr->offCodeSum); size_t price = log2_32(seqStorePtr->offCodeSum) - log2_32(seqStorePtr->offCodeFreq[offCode]);
offCodeBits = log2_32(seqStorePtr->offCodeSum) - log2_32(seqStorePtr->offCodeFreq[offCode]); price += offCode;
// printf("offCodeBits=%.02f matchBits=%.02f dumpsPrice=%d sum=%d\n", offCodeBits, matchBits, (int)price, (int)freq);
offCodeBits += offCode;
/* match Length */ /* match Length */
size_t freq;
if (matchLength >= MaxML) { if (matchLength >= MaxML) {
freq = seqStorePtr->matchLengthFreq[MaxML]; freq = seqStorePtr->matchLengthFreq[MaxML];
if (matchLength < 255+MaxML) { if (matchLength < 255+MaxML) {
@ -129,28 +109,18 @@ FORCE_INLINE size_t ZSTD_getMatchPriceReal(seqStore_t* seqStorePtr, size_t offse
if (matchLength < (1<<15)) price += 16; else price += 24; if (matchLength < (1<<15)) price += 16; else price += 24;
} } } }
else freq = seqStorePtr->matchLengthFreq[matchLength]; else freq = seqStorePtr->matchLengthFreq[matchLength];
// matchBits = -log2((double)freq/(double)seqStorePtr->matchLengthSum); price += log2_32(seqStorePtr->matchLengthSum) - log2_32(freq);
matchBits = log2_32(seqStorePtr->matchLengthSum) - log2_32(freq);
// freq = round(1.0f*(offCodeBits + matchBits + price)); return price;
freq = offCodeBits + matchBits + price;
// printf("offCodeBits=%.02f matchBits=%.02f dumpsPrice=%d sum=%d\n", offCodeBits, matchBits, (int)price, (int)freq);
return freq;
} }
FORCE_INLINE size_t ZSTD_getPrice(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) FORCE_INLINE size_t ZSTD_getPrice(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength)
{ {
#if 1 if (seqStorePtr->litSum > ZSTD_FREQ_THRESHOLD)
size_t lit_cost = ZSTD_getLiteralPriceReal(seqStorePtr, litLength, literals); return ZSTD_getLiteralPriceReal(seqStorePtr, litLength, literals) + ZSTD_getMatchPriceReal(seqStorePtr, offset, matchLength);
// size_t match_cost_old = ZSTD_highbit((U32)matchLength+1) + Offbits + ZSTD_highbit((U32)offset+1);
size_t match_cost = ZSTD_getMatchPriceReal(seqStorePtr, offset, matchLength); return (litLength<<3) + ZSTD_highbit((U32)matchLength+1) + Offbits + ZSTD_highbit((U32)offset+1);
return lit_cost + match_cost;
#else
size_t lit_cost = (litLength<<3);
size_t match_cost = ZSTD_highbit((U32)matchLength+1) + Offbits + ZSTD_highbit((U32)offset+1);
return lit_cost + match_cost;
#endif
} }
@ -232,11 +202,7 @@ size_t ZSTD_insertBtAndGetAllMatches (
U32 dummy32; /* to be nullified at the end */ U32 dummy32; /* to be nullified at the end */
size_t mnum = 0; size_t mnum = 0;
#if 1
bestLength = 0; bestLength = 0;
#else
bestLength--;
#endif
hashTable[h] = current; /* Update Hash Table */ hashTable[h] = current; /* Update Hash Table */
while (nbCompares-- && (matchIndex > windowLow)) { while (nbCompares-- && (matchIndex > windowLow)) {
@ -255,7 +221,6 @@ size_t ZSTD_insertBtAndGetAllMatches (
match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
} }
#if 1
if (matchLength > bestLength) { if (matchLength > bestLength) {
if (matchLength > matchEndIdx - matchIndex) if (matchLength > matchEndIdx - matchIndex)
matchEndIdx = matchIndex + (U32)matchLength; matchEndIdx = matchIndex + (U32)matchLength;
@ -272,24 +237,6 @@ size_t ZSTD_insertBtAndGetAllMatches (
if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ if (ip+matchLength == iend) /* equal : no way to know if inf or sup */
break; /* drop, to guarantee consistency (miss a little bit of compression) */ break; /* drop, to guarantee consistency (miss a little bit of compression) */
} }
#else
if (matchLength > matchEndIdx - matchIndex)
matchEndIdx = matchIndex + (U32)matchLength;
if (matchLength > bestLength) {
bestLength = matchLength;
matches[mnum].off = current - matchIndex;
matches[mnum].len = matchLength;
matches[mnum].back = 0;
mnum++;
if (matchLength > ZSTD_OPT_NUM) break;
}
if (ip+matchLength == iend) /* equal : no way to know if inf or sup */
break; /* drop, to guarantee consistency (miss a little bit of compression) */
#endif
if (match[matchLength] < ip[matchLength]) { if (match[matchLength] < ip[matchLength]) {
/* match is smaller than current */ /* match is smaller than current */
@ -447,7 +394,7 @@ void print_hex_text(uint8_t* buf, int bufsize, int endline)
/* ******************************* /* *******************************
* Optimal parser OLD * Old parser
*********************************/ *********************************/
FORCE_INLINE FORCE_INLINE
void ZSTD_compressBlock_opt2_generic(ZSTD_CCtx* ctx, void ZSTD_compressBlock_opt2_generic(ZSTD_CCtx* ctx,
@ -644,8 +591,8 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
last_pos = 0; last_pos = 0;
inr = ip; inr = ip;
opt[0].litlen = ip - anchor; opt[0].litlen = ip - anchor;
// opt[0].price = ZSTD_getLiteralPrice(seqStorePtr, opt[0].litlen, anchor);
/* check repCode */ /* check repCode */
if (MEM_read32(ip+1) == MEM_read32(ip+1 - rep_1)) { if (MEM_read32(ip+1) == MEM_read32(ip+1 - rep_1)) {
/* repcode : we take it */ /* repcode : we take it */
@ -674,10 +621,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
if (faster_get_matches && last_pos) if (faster_get_matches && last_pos)
match_num = 0; match_num = 0;
else else
{ match_num = getAllMatches(ctx, ip, ip, iend, maxSearches, mls, matches, best_mlen); /* first search (depth 0) */
/* first search (depth 0) */
match_num = getAllMatches(ctx, ip, ip, iend, maxSearches, mls, matches, best_mlen);
}
ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos); ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos);
if (!last_pos && !match_num) { ip++; continue; } if (!last_pos && !match_num) { ip++; continue; }
@ -747,7 +691,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
if (cur == last_pos) break; if (cur == last_pos) break;
if (inr > ilimit) if (inr > ilimit) // last match must start at a minimum distance of 8 from oend
continue; continue;
mlen = opt[cur].mlen; mlen = opt[cur].mlen;
@ -778,7 +722,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
#if 1
if (!opt[cur].off && opt[cur].mlen != 1) { if (!opt[cur].off && opt[cur].mlen != 1) {
mlen = ZSTD_count(inr, inr - opt[cur].rep2, iend); // check rep2 mlen = ZSTD_count(inr, inr - opt[cur].rep2, iend); // check rep2
ZSTD_LOG_PARSER("%d: try REP2 rep2=%d mlen=%d\n", (int)(inr-base), opt[cur].rep2, mlen); ZSTD_LOG_PARSER("%d: try REP2 rep2=%d mlen=%d\n", (int)(inr-base), opt[cur].rep2, mlen);
@ -787,9 +730,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
mlen = ZSTD_count(inr, inr - opt[cur].rep, iend); // check rep mlen = ZSTD_count(inr, inr - opt[cur].rep, iend); // check rep
ZSTD_LOG_PARSER("%d: try REP1 rep=%d mlen=%d\n", (int)(inr-base), opt[cur].rep, mlen); ZSTD_LOG_PARSER("%d: try REP1 rep=%d mlen=%d\n", (int)(inr-base), opt[cur].rep, mlen);
} }
#else
mlen = ZSTD_count(inr, inr - opt[cur].rep, iend); // check rep
#endif
best_mlen = 0; best_mlen = 0;
@ -892,11 +832,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
ZSTD_LOG_TRY_PRICE("%d: TRY8 price=%d opt[%d].price=%d\n", (int)(inr-base), price, cur2 + mlen, opt[cur2 + mlen].price); ZSTD_LOG_TRY_PRICE("%d: TRY8 price=%d opt[%d].price=%d\n", (int)(inr-base), price, cur2 + mlen, opt[cur2 + mlen].price);
if (cur2 + mlen > last_pos || (price < opt[cur2 + mlen].price)) if (cur2 + mlen > last_pos || (price < opt[cur2 + mlen].price))
{
SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price); SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price);
// opt[cur2 + mlen].rep = matches[i].off; // update reps
// opt[cur2 + mlen].rep2 = opt[cur2].rep;
}
mlen++; mlen++;
} }
@ -969,7 +905,7 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set
// printf("orig="); print_hex_text(ip, mlen, 0); // printf("orig="); print_hex_text(ip, mlen, 0);
// printf("match="); print_hex_text(ip-offset, mlen, 0); // printf("match="); print_hex_text(ip-offset, mlen, 0);
#if 1 #if 0 // for debugging
size_t ml2; size_t ml2;
if (offset) if (offset)
ml2 = ZSTD_count(ip, ip-offset, iend); ml2 = ZSTD_count(ip, ip-offset, iend);