diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index d378d7ab..967ecc06 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -627,10 +627,8 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc, /* CTable for Offset codes */ { /* create Offset codes */ size_t i; for (i=0; i> g_searchStrength) + 1; @@ -994,10 +992,11 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc, while ((ip>anchor) && (match>lowest) && (ip[-1] == match[-1])) { ip--; match--; mlCode++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; + + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_NUM - 1, mlCode); } /* match found */ - ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_NUM - 1, mlCode); ip += mlCode + MINMATCH; anchor = ip; @@ -1012,7 +1011,7 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* zc, size_t rlCode = ZSTD_count(ip+MINMATCH, ip+MINMATCH-offset_2, iend); size_t tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip-base); - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0 + ZSTD_REP_NUM - 1, rlCode); + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rlCode); ip += rlCode+MINMATCH; anchor = ip; continue; /* faster when present ... (?) */ @@ -1093,7 +1092,7 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx, const BYTE* repMatchEnd = repIndex < dictLimit ? dictEnd : iend; mlCode = ZSTD_count_2segments(ip+1+MINMATCH, repMatch+MINMATCH, iend, repMatchEnd, lowPrefixPtr); ip++; - offset = 0; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset, mlCode); } else { if ( (matchIndex < lowLimit) || (MEM_read32(match) != MEM_read32(ip)) ) @@ -1106,10 +1105,10 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx, offset = current - matchIndex; offset_2 = offset_1; offset_1 = offset; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_NUM - 1, mlCode); } } /* found a match : store it */ - ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_NUM - 1, mlCode); ip += mlCode + MINMATCH; anchor = ip; @@ -1127,7 +1126,7 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx, const BYTE* const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; size_t repLength2 = ZSTD_count_2segments(ip+MINMATCH, repMatch2+MINMATCH, iend, repEnd2, lowPrefixPtr); U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0 + ZSTD_REP_NUM - 1, repLength2); + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2); hashTable[ZSTD_hashPtr(ip, hBits, mls)] = current2; ip += repLength2+MINMATCH; anchor = ip; @@ -1578,13 +1577,19 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, const BYTE* start=ip+1; /* check repCode */ - for (int i=0; i<1; i++) { - if (MEM_read32(ip+1) == MEM_read32(ip+1 - rep[i])) { - /* repcode : we take it */ + for (int i=0; i gain1) + matchLength = mlRep, offset = i; } } @@ -1605,12 +1610,13 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, if (depth>=1) while (ip= ZSTD_REP_NUM) && (MEM_read32(ip) == MEM_read32(ip - rep[0]))) { - size_t mlRep = ZSTD_count(ip+MINMATCH, ip+MINMATCH-rep[0], iend) + MINMATCH; + for (int i=0; i= ZSTD_REP_NUM) && (MEM_read32(ip) == MEM_read32(ip - rep[i]))) { + size_t mlRep = ZSTD_count(ip+MINMATCH, ip+MINMATCH-rep[i], iend) + MINMATCH; int gain2 = (int)(mlRep * 3); int gain1 = (int)(matchLength*3 - ZSTD_highbit((U32)offset+1) + 1); if ((mlRep >= MINMATCH) && (gain2 > gain1)) - matchLength = mlRep, offset = 0 + (ZSTD_REP_NUM - 1), start = ip; + matchLength = mlRep, offset = i, start = ip; } { size_t offset2=999999; @@ -1625,12 +1631,13 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, /* let's find an even better one */ if ((depth==2) && (ip= ZSTD_REP_NUM) && (MEM_read32(ip) == MEM_read32(ip - rep[0]))) { - size_t ml2 = ZSTD_count(ip+MINMATCH, ip+MINMATCH-rep[0], iend) + MINMATCH; + for (int i=0; i= ZSTD_REP_NUM) && (MEM_read32(ip) == MEM_read32(ip - rep[i]))) { + size_t ml2 = ZSTD_count(ip+MINMATCH, ip+MINMATCH-rep[i], iend) + MINMATCH; int gain2 = (int)(ml2 * 4); int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 1); if ((ml2 >= MINMATCH) && (gain2 > gain1)) - matchLength = ml2, offset = 0 + (ZSTD_REP_NUM - 1), start = ip; + matchLength = ml2, offset = i, start = ip; } { size_t offset2=999999; @@ -1654,7 +1661,20 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, _storeSequence: { if (offset >= ZSTD_REP_NUM) { - rep[1] = rep[0]; rep[0] = offset - (ZSTD_REP_NUM - 1); + rep[3] = rep[2]; + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = offset - (ZSTD_REP_NUM - 1); + } else { + if (offset != 0) { + size_t temp = rep[offset]; + if (offset != 1) { + if (offset == 3) rep[3] = rep[2]; + rep[2] = rep[1]; + } + rep[1] = rep[0]; + rep[0] = temp; + } } size_t litLength = start - anchor; ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, matchLength-MINMATCH); @@ -1669,7 +1689,7 @@ _storeSequence: offset = rep[1]; rep[1] = rep[0]; rep[0] = offset; - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0 + ZSTD_REP_NUM - 1, matchLength); + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength); ip += matchLength+MINMATCH; anchor = ip; continue; /* faster when present ... (?) */ @@ -1736,6 +1756,8 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, U32 maxNbAttempts, U32 matchLengthSearch); searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS_extDict : ZSTD_HcFindBestMatch_extDict_selectMLS; + printf("ZSTD_compressBlock_lazy_extDict_generic reps not implemented!\n"); exit(1); + /* init */ U32 rep[ZSTD_REP_NUM]; for (int i=0; i= MINMATCH) && (gain2 > gain1)) - matchLength = repLength, offset = 0 + (ZSTD_REP_NUM - 1), start = ip; + matchLength = repLength, offset = 0, start = ip; } } /* search match, depth 1 */ @@ -1825,7 +1847,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, int gain2 = (int)(repLength * 4); int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 1); if ((repLength >= MINMATCH) && (gain2 > gain1)) - matchLength = repLength, offset = 0 + (ZSTD_REP_NUM - 1), start = ip; + matchLength = repLength, offset = 0, start = ip; } } /* search match, depth 2 */ @@ -1869,7 +1891,7 @@ _storeSequence: const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; matchLength = ZSTD_count_2segments(ip+MINMATCH, repMatch+MINMATCH, iend, repEnd, prefixStart) + MINMATCH; offset = rep[1]; rep[1] = rep[0]; rep[0] = offset; /* swap offset history */ - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0 + ZSTD_REP_NUM - 1, matchLength-MINMATCH); + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength-MINMATCH); ip += matchLength; anchor = ip; continue; /* faster when present ... (?) */ diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index fbe419de..4f7ba6d7 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -625,7 +625,6 @@ typedef struct { static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState, const U32 mls) { size_t litLength; - size_t prevOffset; size_t offset; size_t matchLength; const BYTE* dumps = seqState->dumps; @@ -633,7 +632,6 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState, const U32 mls) /* Literal length */ litLength = FSE_peakSymbol(&(seqState->stateLL)); - prevOffset = litLength ? seq->offset : seqState->prevOffset[0]; if (litLength == MaxLL) { U32 add = *dumps++; if (add < 255) litLength += add; @@ -657,43 +655,26 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState, const U32 mls) offset = offsetPrefix[offsetCode] + BIT_readBits(&(seqState->DStream), nbBits); if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); if (offsetCode==0) { - -/* - if (offset < LZMAX_NUM_REPS) - { - #if ZSTD_REP_NUM > 1 - uint32_t pos = offset; - offset = reps[pos]; - if (pos != 0) - { - if (pos != 1) - { - if (pos == 3) - reps[3] = reps[2]; - reps[2] = reps[1]; - } - reps[1] = reps[0]; - reps[0] = offset; - } - #else - offset = reps[0]; - #endif - } - else - { - #if ZSTD_REP_NUM > 1 - offset -= LZMAX_NUM_REPS - 1; - reps[3] = reps[2]; - reps[2] = reps[1]; - reps[1] = reps[0]; - #endif - reps[0] = offset; - } -*/ - offset = prevOffset; /* repcode, cmove */ + if (!litLength) { + offset = seqState->prevOffset[1]; + seqState->prevOffset[1] = seq->offset; /* cmove */ + } else + offset = seq->offset; } else { - if (!litLength) seqState->prevOffset[0] = seq->offset; /* cmove */ - offset -= ZSTD_REP_NUM - 1; + if (offset < ZSTD_REP_NUM) { /* offset = 1,2,3 */ + size_t temp = seqState->prevOffset[offset]; + if (offset != 1) { + if (offset == 3) seqState->prevOffset[3] = seqState->prevOffset[2]; + seqState->prevOffset[2] = seqState->prevOffset[1]; + } + seqState->prevOffset[1] = seq->offset; + offset = temp; + } else { + offset -= ZSTD_REP_NUM - 1; + seqState->prevOffset[3] = seqState->prevOffset[2]; + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seq->offset; /* cmove */ + } } FSE_decodeSymbol(&(seqState->stateOffb), &(seqState->DStream)); /* update */ // printf("offsetCode=%d nbBits=%d offset=%d\n", offsetCode, nbBits, (int)offset); fflush(stdout); diff --git a/lib/zstd_opt.h b/lib/zstd_opt.h index e3cab295..0648a2ae 100644 --- a/lib/zstd_opt.h +++ b/lib/zstd_opt.h @@ -574,7 +574,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, } best_mlen = mlen; - ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, 0, price, litlen); + ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, 0, price, litlen); do { if (cur + mlen > last_pos || price <= opt[cur + mlen].price) @@ -688,7 +688,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ #endif ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-minMatch); - ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset + ZSTD_REP_NUM - 1, mlen-minMatch); + ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset ? offset + ZSTD_REP_NUM - 1 : 0, mlen-minMatch); anchor = ip = ip + mlen; } /* for (cur=0; cur < last_pos; ) */ @@ -702,7 +702,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ rep[0] = best_off; ZSTD_LOG_ENCODE("%d/%d: ENCODE REP literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(anchor-base), (int)(iend-base), (int)(0), (int)best_mlen, (int)(0), (int)rep[0], (int)rep[1]); ZSTD_updatePrice(seqStorePtr, 0, anchor, 0, best_mlen); - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0 + ZSTD_REP_NUM - 1, best_mlen); + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, best_mlen); anchor += best_mlen+minMatch; continue; /* faster when present ... (?) */ } @@ -1036,7 +1036,7 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set #endif ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-minMatch); - ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset + ZSTD_REP_NUM - 1, mlen-minMatch); + ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset ? offset + ZSTD_REP_NUM - 1 : 0, mlen-minMatch); anchor = ip = ip + mlen; } @@ -1059,7 +1059,7 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set offset = rep[1]; rep[1] = rep[0]; rep[0] = offset; /* swap offset history */ ZSTD_LOG_ENCODE("%d/%d: ENCODE REP literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(anchor-base), (int)(iend-base), (int)(0), (int)best_mlen, (int)(0), (int)rep[0], (int)rep[1]); ZSTD_updatePrice(seqStorePtr, 0, anchor, 0, mlen-minMatch); - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0 + ZSTD_REP_NUM - 1, mlen-minMatch); + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, mlen-minMatch); anchor += mlen; } #else @@ -1077,7 +1077,7 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set offset = rep[1]; rep[1] = rep[0]; rep[0] = offset; /* swap offset history */ ZSTD_LOG_ENCODE("%d/%d: ENCODE REP literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(anchor-base), (int)(iend-base), (int)(0), (int)best_mlen, (int)(0), (int)rep[0], (int)rep[1]); ZSTD_updatePrice(seqStorePtr, 0, anchor, 0, mlen-minMatch); - ZSTD_storeSeq(seqStorePtr, 0, anchor, 0 + ZSTD_REP_NUM - 1, mlen-minMatch); + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, mlen-minMatch); anchor += mlen; continue; /* faster when present ... (?) */ }