AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

optimal parser: removed ZSTD_REP_INIT

Browse Source
This commit is contained in:
inikep 2016-07-27 11:07:13 +02:00
parent 2214e469ee
commit 003c7a8568
4 changed files with 25 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -107,7 +107,7 @@ armtest: clean
ppctest: clean ppctest: clean
$(MAKE) -C $(PRGDIR) datagen # use native, faster $(MAKE) -C $(PRGDIR) datagen # use native, faster
$(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-static" $(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static"
ppc64test: clean ppc64test: clean
$(MAKE) -C $(PRGDIR) datagen # use native, faster $(MAKE) -C $(PRGDIR) datagen # use native, faster

9
lib/common/zstd_internal.h
View File

@ -52,8 +52,9 @@
* Common constants * Common constants
***************************************/ ***************************************/
#define ZSTD_OPT_DEBUG 0 /* 3 = compression stats; 5 = check encoded sequences; 9 = full logs */ #define ZSTD_OPT_DEBUG 0 /* 3 = compression stats; 5 = check encoded sequences; 9 = full logs */
#include <stdio.h>
#if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>=9 #if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>=9
#include <stdio.h>
#include <stdlib.h>
#define ZSTD_LOG_PARSER(...) printf(__VA_ARGS__) #define ZSTD_LOG_PARSER(...) printf(__VA_ARGS__)
#define ZSTD_LOG_ENCODE(...) printf(__VA_ARGS__) #define ZSTD_LOG_ENCODE(...) printf(__VA_ARGS__)
#define ZSTD_LOG_BLOCK(...) printf(__VA_ARGS__) #define ZSTD_LOG_BLOCK(...) printf(__VA_ARGS__)
@ -66,8 +67,8 @@
#define ZSTD_OPT_NUM (1<<12) #define ZSTD_OPT_NUM (1<<12)
#define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7 */ #define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7 */
#define ZSTD_REP_NUM 3 #define ZSTD_REP_NUM 3 /* number of repcodes */
#define ZSTD_REP_INIT ZSTD_REP_NUM #define ZSTD_REP_CHECK (ZSTD_REP_NUM-0) /* number of repcodes to check by the optimal parser */
#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) #define ZSTD_REP_MOVE (ZSTD_REP_NUM-1)
static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
@ -169,7 +170,7 @@ typedef struct {
U32 off; U32 off;
U32 mlen; U32 mlen;
U32 litlen; U32 litlen;
U32 rep[ZSTD_REP_INIT]; U32 rep[ZSTD_REP_NUM];
} ZSTD_optimal_t; } ZSTD_optimal_t;
#if ZSTD_OPT_DEBUG == 3 #if ZSTD_OPT_DEBUG == 3

28
lib/compress/zstd_opt.h
View File

@ -464,13 +464,14 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
ZSTD_optimal_t* opt = seqStorePtr->priceTable; ZSTD_optimal_t* opt = seqStorePtr->priceTable;
ZSTD_match_t* matches = seqStorePtr->matchTable; ZSTD_match_t* matches = seqStorePtr->matchTable;
const BYTE* inr; const BYTE* inr;
U32 offset, rep[ZSTD_REP_INIT]; U32 offset, rep[ZSTD_REP_NUM];
/* init */ /* init */
ctx->nextToUpdate3 = ctx->nextToUpdate; ctx->nextToUpdate3 = ctx->nextToUpdate;
ZSTD_rescaleFreqs(seqStorePtr); ZSTD_rescaleFreqs(seqStorePtr);
ip += (ip==prefixStart); ip += (ip==prefixStart);
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=ctx->rep[i]; } { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=ctx->rep[i]; }
inr = ip;
ZSTD_LOG_BLOCK("%d: COMPBLOCK_OPT_GENERIC srcSz=%d maxSrch=%d mls=%d sufLen=%d\n", (int)(ip-base), (int)srcSize, maxSearches, mls, sufficient_len); ZSTD_LOG_BLOCK("%d: COMPBLOCK_OPT_GENERIC srcSz=%d maxSrch=%d mls=%d sufLen=%d\n", (int)(ip-base), (int)srcSize, maxSearches, mls, sufficient_len);
@ -484,7 +485,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
/* check repCode */ /* check repCode */
{ U32 i; { U32 i;
for (i=0; i<ZSTD_REP_NUM; i++) { for (i=0; i<ZSTD_REP_CHECK; i++) {
if ((rep[i]<(U32)(ip-prefixStart)) if ((rep[i]<(U32)(ip-prefixStart))
&& (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - rep[i], minMatch))) { && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - rep[i], minMatch))) {
mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-rep[i], iend) + minMatch; mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-rep[i], iend) + minMatch;
@ -531,7 +532,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
if (last_pos < minMatch) { ip++; continue; } if (last_pos < minMatch) { ip++; continue; }
/* initialize opt[0] */ /* initialize opt[0] */
{ U32 i ; for (i=0; i<ZSTD_REP_INIT; i++) opt[0].rep[i] = rep[i]; } { U32 i ; for (i=0; i<ZSTD_REP_NUM; i++) opt[0].rep[i] = rep[i]; }
opt[0].mlen = 1; opt[0].mlen = 1;
opt[0].litlen = litlen; opt[0].litlen = litlen;
@ -575,15 +576,15 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
best_mlen = minMatch; best_mlen = minMatch;
{ U32 i; { U32 i;
for (i=0; i<ZSTD_REP_NUM; i++) { for (i=0; i<ZSTD_REP_CHECK; i++) {
if ((opt[cur].rep[i]<(U32)(inr-prefixStart)) if ((opt[cur].rep[i]<(U32)(inr-prefixStart))
&& (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - opt[cur].rep[i], minMatch))) { /* check rep */ && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - opt[cur].rep[i], minMatch))) { /* check rep */
mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - opt[cur].rep[i], iend) + minMatch; mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - opt[cur].rep[i], iend) + minMatch;
ZSTD_LOG_PARSER("%d: Found REP %d/%d mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), i, ZSTD_REP_NUM, mlen, i, opt[cur].rep[i], cur, opt[cur].off); ZSTD_LOG_PARSER("%d: Found REP %d/%d mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), i, ZSTD_REP_NUM, mlen, i, opt[cur].rep[i], cur, opt[cur].off);
if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos);
best_mlen = mlen; best_off = i; last_pos = cur + 1; best_mlen = mlen; best_off = i; last_pos = cur + 1;
ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos);
goto _storeSequence; goto _storeSequence;
} }
@ -755,12 +756,13 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
const BYTE* inr; const BYTE* inr;
/* init */ /* init */
U32 offset, rep[ZSTD_REP_INIT]; U32 offset, rep[ZSTD_REP_NUM];
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=ctx->rep[i]; } { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=ctx->rep[i]; }
ctx->nextToUpdate3 = ctx->nextToUpdate; ctx->nextToUpdate3 = ctx->nextToUpdate;
ZSTD_rescaleFreqs(seqStorePtr); ZSTD_rescaleFreqs(seqStorePtr);
ip += (ip==prefixStart); ip += (ip==prefixStart);
inr = ip;
ZSTD_LOG_BLOCK("%d: COMPBLOCK_OPT_EXTDICT srcSz=%d maxSrch=%d mls=%d sufLen=%d\n", (int)(ip-base), (int)srcSize, maxSearches, mls, sufficient_len); ZSTD_LOG_BLOCK("%d: COMPBLOCK_OPT_EXTDICT srcSz=%d maxSrch=%d mls=%d sufLen=%d\n", (int)(ip-base), (int)srcSize, maxSearches, mls, sufficient_len);
@ -776,7 +778,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
/* check repCode */ /* check repCode */
{ U32 i; { U32 i;
for (i=0; i<ZSTD_REP_NUM; i++) { for (i=0; i<ZSTD_REP_CHECK; i++) {
const U32 repIndex = (U32)(current - rep[i]); const U32 repIndex = (U32)(current - rep[i]);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex; const BYTE* const repMatch = repBase + repIndex;
@ -807,7 +809,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos); ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos);
if (!last_pos && !match_num) { ip++; continue; } if (!last_pos && !match_num) { ip++; continue; }
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) opt[0].rep[i] = rep[i]; } { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) opt[0].rep[i] = rep[i]; }
opt[0].mlen = 1; opt[0].mlen = 1;
if (match_num && (matches[match_num-1].len > sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) { if (match_num && (matches[match_num-1].len > sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) {
@ -878,7 +880,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
best_mlen = 0; best_mlen = 0;
{ U32 i; { U32 i;
for (i=0; i<ZSTD_REP_NUM; i++) { for (i=0; i<ZSTD_REP_CHECK; i++) {
const U32 repIndex = (U32)(current+cur - opt[cur].rep[i]); const U32 repIndex = (U32)(current+cur - opt[cur].rep[i]);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex; const BYTE* const repMatch = repBase + repIndex;
@ -890,8 +892,8 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
ZSTD_LOG_PARSER("%d: Found REP %d/%d mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), i, ZSTD_REP_NUM, mlen, i, opt[cur].rep[i], cur, opt[cur].off); ZSTD_LOG_PARSER("%d: Found REP %d/%d mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), i, ZSTD_REP_NUM, mlen, i, opt[cur].rep[i], cur, opt[cur].off);
if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos);
best_mlen = mlen; best_off = i; last_pos = cur + 1; best_mlen = mlen; best_off = i; last_pos = cur + 1;
ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos);
goto _storeSequence; goto _storeSequence;
} }
@ -1033,7 +1035,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
} } /* for (cur=0; cur < last_pos; ) */ } } /* for (cur=0; cur < last_pos; ) */
/* Save reps for next block */ /* Save reps for next block */
ctx->savedRep[0] = rep[0]; ctx->savedRep[1] = rep[1]; ctx->savedRep[2] = rep[2]; { int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->savedRep[i] = rep[i]; }
/* Last Literals */ /* Last Literals */
{ size_t lastLLSize = iend - anchor; { size_t lastLLSize = iend - anchor;

8
lib/decompress/zstd_decompress.c
View File

@ -118,7 +118,7 @@ struct ZSTD_DCtx_s
const void* vBase; const void* vBase;
const void* dictEnd; const void* dictEnd;
size_t expected; size_t expected;
U32 rep[3]; U32 rep[ZSTD_REP_NUM];
ZSTD_frameParams fParams; ZSTD_frameParams fParams;
blockType_e bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ blockType_e bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
ZSTD_dStage stage; ZSTD_dStage stage;
@ -571,7 +571,7 @@ typedef struct {
FSE_DState_t stateLL; FSE_DState_t stateLL;
FSE_DState_t stateOffb; FSE_DState_t stateOffb;
FSE_DState_t stateML; FSE_DState_t stateML;
size_t prevOffset[ZSTD_REP_INIT]; size_t prevOffset[ZSTD_REP_NUM];
} seqState_t; } seqState_t;
@ -751,7 +751,7 @@ static size_t ZSTD_decompressSequences(
if (nbSeq) { if (nbSeq) {
seqState_t seqState; seqState_t seqState;
dctx->fseEntropy = 1; dctx->fseEntropy = 1;
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) seqState.prevOffset[i] = dctx->rep[i]; } { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->rep[i]; }
{ size_t const errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip); { size_t const errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip);
if (ERR_isError(errorCode)) return ERROR(corruption_detected); } if (ERR_isError(errorCode)) return ERROR(corruption_detected); }
FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL); FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL);
@ -769,7 +769,7 @@ static size_t ZSTD_decompressSequences(
/* check if reached exact end */ /* check if reached exact end */
if (nbSeq) return ERROR(corruption_detected); if (nbSeq) return ERROR(corruption_detected);
/* save reps for next block */ /* save reps for next block */
{ U32 i; for (i=0; i<ZSTD_REP_INIT; i++) dctx->rep[i] = (U32)(seqState.prevOffset[i]); } { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->rep[i] = (U32)(seqState.prevOffset[i]); }
} }
/* last literal segment */ /* last literal segment */