AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

reduced table sizes for HUF_readDTableX4

Browse Source
This commit is contained in:
Yann Collet 2016-12-01 18:24:59 -08:00
parent a0d742b1e4
commit b89af20353
3 changed files with 20 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
lib/common/entropy_common.c
View File

@ -159,6 +159,7 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
/*! HUF_readStats() : /*! HUF_readStats() :
Read compact Huffman tree, saved by HUF_writeCTable(). Read compact Huffman tree, saved by HUF_writeCTable().
`huffWeight` is destination buffer. `huffWeight` is destination buffer.
`rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32.
@return : size read from `src` , or an error Code . @return : size read from `src` , or an error Code .
Note : Needed by HUF_readCTable() and HUF_readDTableX?() . Note : Needed by HUF_readCTable() and HUF_readDTableX?() .
*/ */
@ -187,17 +188,17 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
huffWeight[n+1] = ip[n/2] & 15; huffWeight[n+1] = ip[n/2] & 15;
} } } } } }
else { /* header compressed with FSE (normal case) */ else { /* header compressed with FSE (normal case) */
FSE_DTable fseWorkspace[FSE_DTABLE_SIZE_U32(5)]; /* 5 is max possible tableLog for HUF header */ FSE_DTable fseWorkspace[FSE_DTABLE_SIZE_U32(6)]; /* 6 is max possible tableLog for HUF header (maybe even 5, to be tested) */
if (iSize+1 > srcSize) return ERROR(srcSize_wrong); if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
oSize = FSE_decompress_wksp(huffWeight, hwSize-1, ip+1, iSize, fseWorkspace, 5); /* max (hwSize-1) values decoded, as last one is implied */ oSize = FSE_decompress_wksp(huffWeight, hwSize-1, ip+1, iSize, fseWorkspace, 6); /* max (hwSize-1) values decoded, as last one is implied */
if (FSE_isError(oSize)) return oSize; if (FSE_isError(oSize)) return oSize;
} }
/* collect weight stats */ /* collect weight stats */
memset(rankStats, 0, (HUF_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32)); memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32));
weightTotal = 0; weightTotal = 0;
{ U32 n; for (n=0; n<oSize; n++) { { U32 n; for (n=0; n<oSize; n++) {
if (huffWeight[n] >= HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); if (huffWeight[n] >= HUF_TABLELOG_MAX) return ERROR(corruption_detected);
rankStats[huffWeight[n]]++; rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1; weightTotal += (1 << huffWeight[n]) >> 1;
} } } }
@ -205,7 +206,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
/* get last non-null symbol weight (implied, total must be 2^n) */ /* get last non-null symbol weight (implied, total must be 2^n) */
{ U32 const tableLog = BIT_highbit32(weightTotal) + 1; { U32 const tableLog = BIT_highbit32(weightTotal) + 1;
if (tableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected);
*tableLogPtr = tableLog; *tableLogPtr = tableLog;
/* determine last weight */ /* determine last weight */
{ U32 const total = 1 << tableLog; { U32 const total = 1 << tableLog;

17
lib/common/huf.h
View File

@ -62,19 +62,19 @@ size_t HUF_compress(void* dst, size_t dstCapacity,
HUF_decompress() : HUF_decompress() :
Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
into already allocated buffer 'dst', of minimum size 'dstSize'. into already allocated buffer 'dst', of minimum size 'dstSize'.
`dstSize` : **must** be the ***exact*** size of original (uncompressed) data. `originalSize` : **must** be the ***exact*** size of original (uncompressed) data.
Note : in contrast with FSE, HUF_decompress can regenerate Note : in contrast with FSE, HUF_decompress can regenerate
RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
because it knows size to regenerate. because it knows size to regenerate.
@return : size of regenerated data (== dstSize), @return : size of regenerated data (== originalSize),
or an error code, which can be tested using HUF_isError() or an error code, which can be tested using HUF_isError()
*/ */
size_t HUF_decompress(void* dst, size_t dstSize, size_t HUF_decompress(void* dst, size_t originalSize,
const void* cSrc, size_t cSrcSize); const void* cSrc, size_t cSrcSize);
/* *** Tool functions *** */ /* *** Tool functions *** */
#define HUF_BLOCKSIZE_MAX (128 * 1024) #define HUF_BLOCKSIZE_MAX (128 * 1024) /*< maximum input size for a single block compressed with HUF_compress */
size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
/* Error Management */ /* Error Management */
@ -85,7 +85,8 @@ const char* HUF_getErrorName(size_t code); /**< provides error code string (us
/* *** Advanced function *** */ /* *** Advanced function *** */
/** HUF_compress2() : /** HUF_compress2() :
* Same as HUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog` */ * Same as HUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog` .
* `tableLog` must be `<= HUF_TABLELOG_MAX` . */
size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
/** HUF_compress4X_wksp() : /** HUF_compress4X_wksp() :
@ -101,7 +102,7 @@ size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t s
/* *** Constants *** */ /* *** Constants *** */
#define HUF_TABLELOG_ABSOLUTEMAX 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ #define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ #define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
#define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */ #define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */
#define HUF_SYMBOLVALUE_MAX 255 #define HUF_SYMBOLVALUE_MAX 255
@ -128,9 +129,9 @@ size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t s
typedef U32 HUF_DTable; typedef U32 HUF_DTable;
#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog))) #define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog)))
#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ #define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1)*0x1000001) } HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) }
#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \ #define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog)*0x1000001) } HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }
/* **************************************** /* ****************************************

10
lib/decompress/huf_decompress.c
View File

@ -398,14 +398,14 @@ static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 co
} } } }
} }
typedef U32 rankVal_t[HUF_TABLELOG_ABSOLUTEMAX][HUF_TABLELOG_ABSOLUTEMAX + 1]; typedef U32 rankVal_t[HUF_TABLELOG_MAX][HUF_TABLELOG_MAX + 1];
static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog, static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog,
const sortedSymbol_t* sortedList, const U32 sortedListSize, const sortedSymbol_t* sortedList, const U32 sortedListSize,
const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight,
const U32 nbBitsBaseline) const U32 nbBitsBaseline)
{ {
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; U32 rankVal[HUF_TABLELOG_MAX + 1];
const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */
const U32 minBits = nbBitsBaseline - maxWeight; const U32 minBits = nbBitsBaseline - maxWeight;
U32 s; U32 s;
@ -446,8 +446,8 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
{ {
BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; BYTE weightList[HUF_SYMBOLVALUE_MAX + 1];
sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1];
U32 rankStats[HUF_TABLELOG_ABSOLUTEMAX + 1] = { 0 }; U32 rankStats[HUF_TABLELOG_MAX + 1] = { 0 };
U32 rankStart0[HUF_TABLELOG_ABSOLUTEMAX + 2] = { 0 }; U32 rankStart0[HUF_TABLELOG_MAX + 2] = { 0 };
U32* const rankStart = rankStart0+1; U32* const rankStart = rankStart0+1;
rankVal_t rankVal; rankVal_t rankVal;
U32 tableLog, maxW, sizeOfSort, nbSymbols; U32 tableLog, maxW, sizeOfSort, nbSymbols;
@ -458,7 +458,7 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
HUF_DEltX4* const dt = (HUF_DEltX4*)dtPtr; HUF_DEltX4* const dt = (HUF_DEltX4*)dtPtr;
HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compilation fails here, assertion is false */ HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compilation fails here, assertion is false */
if (maxTableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge); if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
/* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);