removed stats in debug mode
This commit is contained in:
parent
57ef4b1a0d
commit
8a36f8527c
@ -51,18 +51,9 @@
|
||||
/*-*************************************
|
||||
* Common constants
|
||||
***************************************/
|
||||
#define ZSTD_OPT_DEBUG 0 /* 3 = compression stats; 5 = check encoded sequences; 9 = full logs */
|
||||
#if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>=9
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#define ZSTD_LOG_PARSER(...) printf(__VA_ARGS__)
|
||||
#define ZSTD_LOG_ENCODE(...) printf(__VA_ARGS__)
|
||||
#define ZSTD_LOG_BLOCK(...) printf(__VA_ARGS__)
|
||||
#else
|
||||
#define ZSTD_LOG_PARSER(...)
|
||||
#define ZSTD_LOG_ENCODE(...)
|
||||
#define ZSTD_LOG_BLOCK(...)
|
||||
#endif
|
||||
#define ZSTD_LOG_PARSER(...)
|
||||
#define ZSTD_LOG_ENCODE(...)
|
||||
#define ZSTD_LOG_BLOCK(...)
|
||||
|
||||
#define ZSTD_OPT_NUM (1<<12)
|
||||
#define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7+ */
|
||||
@ -184,16 +175,6 @@ typedef struct {
|
||||
U32 rep[ZSTD_REP_NUM];
|
||||
} ZSTD_optimal_t;
|
||||
|
||||
#if ZSTD_OPT_DEBUG == 3
|
||||
#include ".debug/zstd_stats.h"
|
||||
#else
|
||||
struct ZSTD_stats_s { U32 unused; };
|
||||
MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; }
|
||||
MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; }
|
||||
MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; }
|
||||
MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }
|
||||
#endif /* #if ZSTD_OPT_DEBUG == 3 */
|
||||
|
||||
|
||||
typedef struct seqDef_s {
|
||||
U32 offset;
|
||||
|
@ -1,162 +0,0 @@
|
||||
/*
|
||||
zstd - standard compression library
|
||||
Header File for static linking only
|
||||
Copyright (C) 2014-2016, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd homepage : http://www.zstd.net
|
||||
*/
|
||||
#ifndef ZSTD_STATS_H
|
||||
#define ZSTD_STATS_H
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Types
|
||||
***************************************/
|
||||
struct ZSTD_stats_s {
|
||||
U32 priceOffset, priceOffCode, priceMatchLength, priceLiteral, priceLitLength;
|
||||
U32 totalMatchSum, totalLitSum, totalSeqSum, totalRepSum;
|
||||
U32 litSum, matchLengthSum, litLengthSum, offCodeSum;
|
||||
U32 matchLengthFreq[MaxML+1];
|
||||
U32 litLengthFreq[MaxLL+1];
|
||||
U32 litFreq[1<<Litbits];
|
||||
U32 offCodeFreq[MaxOff+1];
|
||||
};
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Stats functions
|
||||
***************************************/
|
||||
MEM_STATIC ZSTD_stats_t* ZSTD_statsAlloc() { return malloc(sizeof(ZSTD_stats_t)); }
|
||||
MEM_STATIC void ZSTD_statsFree(struct ZSTD_stats_s* stats) { free(stats); }
|
||||
|
||||
MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength)
|
||||
{
|
||||
stats->totalMatchSum += stats->totalSeqSum * ((searchLength == 3) ? 3 : 4);
|
||||
printf("\navgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d\n", (float)stats->totalMatchSum/stats->totalSeqSum, (float)stats->totalLitSum/stats->totalSeqSum, 100.0*stats->totalMatchSum/(stats->totalMatchSum+stats->totalLitSum), 100.0*stats->totalLitSum/(stats->totalMatchSum+stats->totalLitSum), stats->totalRepSum, stats->totalSeqSum);
|
||||
printf("SumBytes=%d Offset=%d OffCode=%d Match=%d Literal=%d LitLength=%d\n", (stats->priceOffset+stats->priceOffCode+stats->priceMatchLength+stats->priceLiteral+stats->priceLitLength)/8, stats->priceOffset/8, stats->priceOffCode/8, stats->priceMatchLength/8, stats->priceLiteral/8, stats->priceLitLength/8);
|
||||
}
|
||||
|
||||
|
||||
MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats)
|
||||
{
|
||||
stats->totalLitSum = stats->totalMatchSum = stats->totalSeqSum = stats->totalRepSum = 1;
|
||||
stats->priceOffset = stats->priceOffCode = stats->priceMatchLength = stats->priceLiteral = stats->priceLitLength = 0;
|
||||
}
|
||||
|
||||
|
||||
MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats)
|
||||
{
|
||||
unsigned u;
|
||||
|
||||
stats->litSum = (2<<Litbits);
|
||||
stats->litLengthSum = MaxLL+1;
|
||||
stats->matchLengthSum = MaxML+1;
|
||||
stats->offCodeSum = (MaxOff+1);
|
||||
|
||||
for (u=0; u<=MaxLit; u++)
|
||||
stats->litFreq[u] = 1;
|
||||
for (u=0; u<=MaxLL; u++)
|
||||
stats->litLengthFreq[u] = 1;
|
||||
for (u=0; u<=MaxML; u++)
|
||||
stats->matchLengthFreq[u] = 1;
|
||||
for (u=0; u<=MaxOff; u++)
|
||||
stats->offCodeFreq[u] = 1;
|
||||
}
|
||||
|
||||
|
||||
MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength)
|
||||
{
|
||||
U32 u;
|
||||
/* literals */
|
||||
stats->priceLiteral += litLength * ZSTD_highbit(stats->litSum+1);
|
||||
for (u=0; u < litLength; u++)
|
||||
stats->priceLiteral -= ZSTD_highbit(stats->litFreq[literals[u]]+1);
|
||||
stats->litSum += litLength;
|
||||
for (u=0; u < litLength; u++)
|
||||
stats->litFreq[literals[u]]++;
|
||||
|
||||
/* literal Length */
|
||||
{ static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7,
|
||||
8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 16, 17, 17, 18, 18, 19, 19,
|
||||
20, 20, 20, 20, 21, 21, 21, 21,
|
||||
22, 22, 22, 22, 22, 22, 22, 22,
|
||||
23, 23, 23, 23, 23, 23, 23, 23,
|
||||
24, 24, 24, 24, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24 };
|
||||
const BYTE LL_deltaCode = 19;
|
||||
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength];
|
||||
if (litLength) {
|
||||
stats->priceLitLength += LL_bits[llCode] + ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[llCode]+1);
|
||||
} else {
|
||||
stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[0]+1);
|
||||
}
|
||||
stats->litLengthFreq[llCode]++;
|
||||
stats->litLengthSum++;
|
||||
}
|
||||
|
||||
/* match offset */
|
||||
{ BYTE offCode = (BYTE)ZSTD_highbit(offset+1);
|
||||
stats->priceOffCode += ZSTD_highbit(stats->offCodeSum+1) - ZSTD_highbit(stats->offCodeFreq[offCode]+1);
|
||||
stats->priceOffset += offCode;
|
||||
stats->offCodeSum++;
|
||||
stats->offCodeFreq[offCode]++;
|
||||
}
|
||||
|
||||
/* match Length */
|
||||
{ static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
|
||||
32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
|
||||
38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39,
|
||||
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
|
||||
41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
|
||||
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
|
||||
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
|
||||
const BYTE ML_deltaCode = 36;
|
||||
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength];
|
||||
stats->priceMatchLength += ML_bits[mlCode] + ZSTD_highbit(stats->matchLengthSum+1) - ZSTD_highbit(stats->matchLengthFreq[mlCode]+1);
|
||||
stats->matchLengthFreq[mlCode]++;
|
||||
stats->matchLengthSum++;
|
||||
}
|
||||
|
||||
if (offset == 0) stats->totalRepSum++;
|
||||
stats->totalSeqSum++;
|
||||
stats->totalMatchSum += matchLength;
|
||||
stats->totalLitSum += litLength;
|
||||
}
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_STATIC_H */
|
@ -765,8 +765,6 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v
|
||||
printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n",
|
||||
pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
|
||||
#endif
|
||||
ZSTD_statsUpdatePrices(&seqStorePtr->stats, litLength, (const BYTE*)literals, offsetCode, matchCode); /* debug only */
|
||||
|
||||
/* copy Literals */
|
||||
ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
|
||||
seqStorePtr->lit += litLength;
|
||||
@ -1945,7 +1943,6 @@ _storeSequence:
|
||||
{ size_t const lastLLSize = iend - anchor;
|
||||
memcpy(seqStorePtr->lit, anchor, lastLLSize);
|
||||
seqStorePtr->lit += lastLLSize;
|
||||
ZSTD_statsUpdatePrices(&seqStorePtr->stats, lastLLSize, anchor, 0, 0);
|
||||
}
|
||||
}
|
||||
|
||||
@ -2236,7 +2233,6 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
|
||||
BYTE* op = ostart;
|
||||
U32 const maxDist = 1 << cctx->params.cParams.windowLog;
|
||||
ZSTD_stats_t* stats = &cctx->seqStore.stats;
|
||||
ZSTD_statsInit(stats); /* debug only */
|
||||
|
||||
if (cctx->params.fParams.checksumFlag)
|
||||
XXH64_update(&cctx->xxhState, src, srcSize);
|
||||
@ -2244,7 +2240,6 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
|
||||
while (remaining) {
|
||||
U32 const lastBlock = lastFrameChunk & (blockSize >= remaining);
|
||||
size_t cSize;
|
||||
ZSTD_statsResetFreqs(stats); /* debug only */
|
||||
|
||||
if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
|
||||
if (remaining < blockSize) blockSize = remaining;
|
||||
@ -2293,7 +2288,6 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
|
||||
}
|
||||
|
||||
if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending;
|
||||
ZSTD_statsPrint(stats, cctx->params.cParams.searchLength); /* debug only */
|
||||
return op-ostart;
|
||||
}
|
||||
|
||||
|
@ -295,23 +295,10 @@ static U32 ZSTD_insertBtAndGetAllMatches (
|
||||
if ((!extDict) || (matchIndex+matchLength >= dictLimit)) {
|
||||
match = base + matchIndex;
|
||||
if (match[matchLength] == ip[matchLength]) {
|
||||
#if ZSTD_OPT_DEBUG >= 5
|
||||
size_t ml;
|
||||
if (matchIndex < dictLimit)
|
||||
ml = ZSTD_count_2segments(ip, dictBase + matchIndex, iLimit, dictEnd, prefixStart);
|
||||
else
|
||||
ml = ZSTD_count(ip, match, ip+matchLength);
|
||||
if (ml < matchLength)
|
||||
printf("%d: ERROR_NOEXT: offset=%d matchLength=%d matchIndex=%d dictLimit=%d ml=%d\n", current, (int)(current - matchIndex), (int)matchLength, (int)matchIndex, (int)dictLimit, (int)ml), exit(0);
|
||||
#endif
|
||||
matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iLimit) +1;
|
||||
}
|
||||
} else {
|
||||
match = dictBase + matchIndex;
|
||||
#if ZSTD_OPT_DEBUG >= 5
|
||||
if (memcmp(match, ip, matchLength) != 0)
|
||||
printf("%d: ERROR_EXT: matchLength=%d ZSTD_count=%d\n", current, (int)matchLength, (int)ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart)), exit(0);
|
||||
#endif
|
||||
matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart);
|
||||
ZSTD_LOG_PARSER("%d: ZSTD_INSERTBTANDGETALLMATCHES=%d offset=%d dictBase=%p dictEnd=%p prefixStart=%p ip=%p match=%p\n", (int)current, (int)matchLength, (int)(current - matchIndex), dictBase, dictEnd, prefixStart, ip, match);
|
||||
if (matchIndex+matchLength >= dictLimit)
|
||||
@ -672,20 +659,6 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
|
||||
|
||||
ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]);
|
||||
|
||||
#if ZSTD_OPT_DEBUG >= 5
|
||||
U32 ml2;
|
||||
if (offset+1 > ZSTD_REP_MOVE_OPT)
|
||||
ml2 = (U32)ZSTD_count(ip, ip-(offset+1-ZSTD_REP_MOVE_OPT), iend);
|
||||
else
|
||||
ml2 = (U32)ZSTD_count(ip, ip-rep[0], iend);
|
||||
if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) {
|
||||
printf("%d: ERROR_NoExt iend=%d mlen=%d offset=%d ml2=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset, (int)ml2); exit(0); }
|
||||
if (ip < anchor) {
|
||||
printf("%d: ERROR_NoExt ip < anchor iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); }
|
||||
if (ip + mlen > iend) {
|
||||
printf("%d: ERROR_NoExt ip + mlen >= iend iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); }
|
||||
#endif
|
||||
|
||||
ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH);
|
||||
ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH);
|
||||
anchor = ip = ip + mlen;
|
||||
@ -990,26 +963,6 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
|
||||
|
||||
ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]);
|
||||
|
||||
#if ZSTD_OPT_DEBUG >= 5
|
||||
U32 ml2;
|
||||
if (offset+1 > ZSTD_REP_MOVE_OPT) {
|
||||
best_off = offset+1 - ZSTD_REP_MOVE_OPT;
|
||||
if (best_off > (size_t)(ip - prefixStart)) {
|
||||
const BYTE* match = dictEnd - (best_off - (ip - prefixStart));
|
||||
ml2 = ZSTD_count_2segments(ip, match, iend, dictEnd, prefixStart);
|
||||
ZSTD_LOG_PARSER("%d: ZSTD_count_2segments=%d offset=%d dictBase=%p dictEnd=%p prefixStart=%p ip=%p match=%p\n", (int)current, (int)ml2, (int)best_off, dictBase, dictEnd, prefixStart, ip, match);
|
||||
}
|
||||
else ml2 = (U32)ZSTD_count(ip, ip-best_off, iend);
|
||||
}
|
||||
else ml2 = (U32)ZSTD_count(ip, ip-rep[0], iend);
|
||||
if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) {
|
||||
printf("%d: ERROR_Ext iend=%d mlen=%d offset=%d ml2=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset, (int)ml2); exit(0); }
|
||||
if (ip < anchor) {
|
||||
printf("%d: ERROR_Ext ip < anchor iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); }
|
||||
if (ip + mlen > iend) {
|
||||
printf("%d: ERROR_Ext ip + mlen >= iend iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); }
|
||||
#endif
|
||||
|
||||
ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH);
|
||||
ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH);
|
||||
anchor = ip = ip + mlen;
|
||||
|
Loading…
Reference in New Issue
Block a user