AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/Cyan4973/dev' into dev

Browse Source 
# Conflicts:
#	lib/common/util.h
#	programs/paramgrill.c
#	visual/2013/fullbench/fullbench.vcxproj.filters
#	visual/2013/fuzzer/fuzzer.vcxproj.filters
This commit is contained in:
inikep 2016-05-10 13:20:01 +02:00
commit ed9a08538c
28 changed files with 392 additions and 1442 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.travis.yml
View File

@ -12,6 +12,7 @@ env:
- ZSTD_TRAVIS_CI_ENV=cmaketest - ZSTD_TRAVIS_CI_ENV=cmaketest
- ZSTD_TRAVIS_CI_ENV=clangtest - ZSTD_TRAVIS_CI_ENV=clangtest
- ZSTD_TRAVIS_CI_ENV=gpptest - ZSTD_TRAVIS_CI_ENV=gpptest
- ZSTD_TRAVIS_CI_ENV=gnu90test
- ZSTD_TRAVIS_CI_ENV=armtest-w-install - ZSTD_TRAVIS_CI_ENV=armtest-w-install
- ZSTD_TRAVIS_CI_ENV=test - ZSTD_TRAVIS_CI_ENV=test
- ZSTD_TRAVIS_CI_ENV="-C programs test32" - ZSTD_TRAVIS_CI_ENV="-C programs test32"

21
Makefile
View File

@ -54,6 +54,9 @@ all:
zstdprogram: zstdprogram:
$(MAKE) -C $(PRGDIR) $(MAKE) -C $(PRGDIR)
test:
$(MAKE) -C $(PRGDIR) $@
clean: clean:
@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID) @$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
@$(MAKE) -C $(PRGDIR) $@ > $(VOID) @$(MAKE) -C $(PRGDIR) $@ > $(VOID)
@ -75,9 +78,6 @@ uninstall:
travis-install: travis-install:
$(MAKE) install PREFIX=~/install_test_dir $(MAKE) install PREFIX=~/install_test_dir
test:
$(MAKE) -C $(PRGDIR) $@
cmaketest: cmaketest:
cd contrib/cmake ; cmake . ; $(MAKE) cd contrib/cmake ; cmake . ; $(MAKE)
@ -88,6 +88,21 @@ clangtest: clean
gpptest: clean gpptest: clean
$(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" $(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
gnu90test: clean
$(MAKE) all CFLAGS="-std=gnu90 -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef -Wdeclaration-after-statement -Werror"
c90test: clean
$(MAKE) all CFLAGS="-std=c90 -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef -Werror" # will fail, due to // and long long
bmix64test: clean
CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(PRGDIR) test
bmix32test: clean
CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(PRGDIR) test
bmi32test: clean
CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(PRGDIR) test
armtest: clean armtest: clean
$(MAKE) -C $(PRGDIR) datagen # use native, faster $(MAKE) -C $(PRGDIR) datagen # use native, faster
$(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static" $(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"

7
NEWS
View File

@ -1,3 +1,8 @@
v0.6.1
Fixed : Legacy codec v0.5 compatible with dictionary decompression
Fixed : Decoder corruption error (#173)
New : midipix support
v0.6.0 v0.6.0
Stronger high compression modes, thanks to Przemyslaw Skibinski Stronger high compression modes, thanks to Przemyslaw Skibinski
API : ZSTD_getFrameParams() provides size of decompressed content API : ZSTD_getFrameParams() provides size of decompressed content
@ -95,7 +100,7 @@ frame concatenation support
v0.1.1 v0.1.1
fix compression bug fix compression bug
detects write-flush errors detects write-flush errors
git@github.com:Cyan4973/zstd.git
v0.1.0 v0.1.0
first release first release

14
lib/common/bitstream.h
View File

@ -301,7 +301,12 @@ MEM_STATIC size_t BIT_getUpperBits(size_t bitD, U32 const start)
MEM_STATIC size_t BIT_getMiddleBits(size_t bitD, U32 const nbBits, U32 const start) MEM_STATIC size_t BIT_getMiddleBits(size_t bitD, U32 const nbBits, U32 const start)
{ {
#if defined(__BMI__) && defined(__GNUC__) /* experimental */ #if defined(__BMI__) && defined(__GNUC__) /* experimental */
return __builtin_ia32_bextr_u64(bitD, (nbBits<<8) | start ); # if defined(__x86_64__)
if (sizeof(bitD)==8)
return _bextr_u64(bitD, start, nbBits);
else
# endif
return _bextr_u32(bitD, start, nbBits);
#else #else
return (bitD >> start) & BIT_mask[nbBits]; return (bitD >> start) & BIT_mask[nbBits];
#endif #endif
@ -322,7 +327,12 @@ MEM_STATIC size_t BIT_getLowerBits(size_t bitD, U32 const nbBits)
MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
{ {
#if defined(__BMI__) && defined(__GNUC__) /* experimental */ #if defined(__BMI__) && defined(__GNUC__) /* experimental */
return __builtin_ia32_bextr_u64(bitD->bitContainer, (nbBits<<8) | (64 - bitD->bitsConsumed - nbBits) ); # if defined(__x86_64__)
if (sizeof(bitD->bitContainer)==8)
return _bextr_u64(bitD->bitContainer, 64 - bitD->bitsConsumed - nbBits, nbBits);
else
# endif
return _bextr_u32(bitD->bitContainer, 32 - bitD->bitsConsumed - nbBits, nbBits);
#else #else
U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask); return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);

69
lib/common/fse_decompress.c
View File

@ -111,17 +111,13 @@ void FSE_freeDTable (FSE_DTable* dt)
size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{ {
FSE_DTableHeader DTableH; FSE_DTableHeader DTableH;
void* const tdPtr = dt+1; /* because dt is unsigned, 32-bits aligned on 32-bits */ void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */
FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr);
const U32 tableSize = 1 << tableLog;
const U32 tableMask = tableSize-1;
const U32 step = FSE_TABLESTEP(tableSize);
U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1]; U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1];
U32 const maxSV1 = maxSymbolValue + 1;
U32 const tableSize = 1 << tableLog;
U32 highThreshold = tableSize-1; U32 highThreshold = tableSize-1;
S16 const largeLimit= (S16)(1 << (tableLog-1));
U32 noLarge = 1;
U32 s;
/* Sanity Checks */ /* Sanity Checks */
if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge);
@ -129,18 +125,24 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned
/* Init, lay down lowprob symbols */ /* Init, lay down lowprob symbols */
DTableH.tableLog = (U16)tableLog; DTableH.tableLog = (U16)tableLog;
for (s=0; s<=maxSymbolValue; s++) { DTableH.fastMode = 1;
if (normalizedCounter[s]==-1) { { S16 const largeLimit= (S16)(1 << (tableLog-1));
tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s; U32 s;
symbolNext[s] = 1; for (s=0; s<maxSV1; s++) {
} else { if (normalizedCounter[s]==-1) {
if (normalizedCounter[s] >= largeLimit) noLarge=0; tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
symbolNext[s] = normalizedCounter[s]; symbolNext[s] = 1;
} } } else {
if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
symbolNext[s] = normalizedCounter[s];
} } }
memcpy(dt, &DTableH, sizeof(DTableH));
/* Spread symbols */ /* Spread symbols */
{ U32 position = 0; { U32 const tableMask = tableSize-1;
for (s=0; s<=maxSymbolValue; s++) { U32 const step = FSE_TABLESTEP(tableSize);
U32 s, position = 0;
for (s=0; s<maxSV1; s++) {
int i; int i;
for (i=0; i<normalizedCounter[s]; i++) { for (i=0; i<normalizedCounter[s]; i++) {
tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s; tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
@ -154,15 +156,12 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned
/* Build Decoding table */ /* Build Decoding table */
{ U32 u; { U32 u;
for (u=0; u<tableSize; u++) { for (u=0; u<tableSize; u++) {
FSE_FUNCTION_TYPE symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol); FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
U16 nextState = symbolNext[symbol]++; U16 nextState = symbolNext[symbol]++;
tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32 ((U32)nextState) ); tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32 ((U32)nextState) );
tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize); tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
} } } }
DTableH.fastMode = (U16)noLarge;
memcpy(dt, &DTableH, sizeof(DTableH));
return 0; return 0;
} }
@ -199,7 +198,7 @@ size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr;
const unsigned tableSize = 1 << nbBits; const unsigned tableSize = 1 << nbBits;
const unsigned tableMask = tableSize - 1; const unsigned tableMask = tableSize - 1;
const unsigned maxSymbolValue = tableMask; const unsigned maxSV1 = tableMask+1;
unsigned s; unsigned s;
/* Sanity checks */ /* Sanity checks */
@ -208,7 +207,7 @@ size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
/* Build Decoding Table */ /* Build Decoding Table */
DTableH->tableLog = (U16)nbBits; DTableH->tableLog = (U16)nbBits;
DTableH->fastMode = 1; DTableH->fastMode = 1;
for (s=0; s<=maxSymbolValue; s++) { for (s=0; s<maxSV1; s++) {
dinfo[s].newState = 0; dinfo[s].newState = 0;
dinfo[s].symbol = (BYTE)s; dinfo[s].symbol = (BYTE)s;
dinfo[s].nbBits = (BYTE)nbBits; dinfo[s].nbBits = (BYTE)nbBits;
@ -230,11 +229,10 @@ FORCE_INLINE size_t FSE_decompress_usingDTable_generic(
BIT_DStream_t bitD; BIT_DStream_t bitD;
FSE_DState_t state1; FSE_DState_t state1;
FSE_DState_t state2; FSE_DState_t state2;
size_t errorCode;
/* Init */ /* Init */
errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); /* replaced last arg by maxCompressed Size */ { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); /* replaced last arg by maxCompressed Size */
if (FSE_isError(errorCode)) return errorCode; if (FSE_isError(errorCode)) return errorCode; }
FSE_initDState(&state1, &bitD, dt); FSE_initDState(&state1, &bitD, dt);
FSE_initDState(&state2, &bitD, dt); FSE_initDState(&state2, &bitD, dt);
@ -308,22 +306,21 @@ size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSr
DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */
unsigned tableLog; unsigned tableLog;
unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
size_t errorCode;
if (cSrcSize<2) return ERROR(srcSize_wrong); /* too small input size */ if (cSrcSize<2) return ERROR(srcSize_wrong); /* too small input size */
/* normal FSE decoding mode */ /* normal FSE decoding mode */
errorCode = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); { size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
if (FSE_isError(errorCode)) return errorCode; if (FSE_isError(NCountLength)) return NCountLength;
if (errorCode >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size */ if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size */
ip += errorCode; ip += NCountLength;
cSrcSize -= errorCode; cSrcSize -= NCountLength;
}
errorCode = FSE_buildDTable (dt, counting, maxSymbolValue, tableLog); { size_t const errorCode = FSE_buildDTable (dt, counting, maxSymbolValue, tableLog);
if (FSE_isError(errorCode)) return errorCode; if (FSE_isError(errorCode)) return errorCode; }
/* always return, even if it is an error code */ return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); /* always return, even if it is an error code */
return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt);
} }

39
lib/common/huf_static.h
View File

@ -76,7 +76,7 @@ extern "C" {
******************************************/ ******************************************/
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */ size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */
size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder */ size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */
/* **************************************** /* ****************************************
@ -122,7 +122,7 @@ size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, si
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbol decoder */ size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */
size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable); size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable);
size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
@ -157,7 +157,6 @@ MEM_STATIC size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
const void* src, size_t srcSize) const void* src, size_t srcSize)
{ {
U32 weightTotal; U32 weightTotal;
U32 tableLog;
const BYTE* ip = (const BYTE*) src; const BYTE* ip = (const BYTE*) src;
size_t iSize = ip[0]; size_t iSize = ip[0];
size_t oSize; size_t oSize;
@ -191,31 +190,31 @@ MEM_STATIC size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
/* collect weight stats */ /* collect weight stats */
memset(rankStats, 0, (HUF_ABSOLUTEMAX_TABLELOG + 1) * sizeof(U32)); memset(rankStats, 0, (HUF_ABSOLUTEMAX_TABLELOG + 1) * sizeof(U32));
weightTotal = 0; weightTotal = 0;
{ U32 n; for (n=0; n<oSize; n++) { { U32 n; for (n=0; n<oSize; n++) {
if (huffWeight[n] >= HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected); if (huffWeight[n] >= HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected);
rankStats[huffWeight[n]]++; rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1; weightTotal += (1 << huffWeight[n]) >> 1;
}} } }
/* get last non-null symbol weight (implied, total must be 2^n) */ /* get last non-null symbol weight (implied, total must be 2^n) */
tableLog = BIT_highbit32(weightTotal) + 1; { U32 const tableLog = BIT_highbit32(weightTotal) + 1;
if (tableLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected); if (tableLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected);
/* determine last weight */ *tableLogPtr = tableLog;
{ U32 const total = 1 << tableLog; /* determine last weight */
U32 const rest = total - weightTotal; { U32 const total = 1 << tableLog;
U32 const verif = 1 << BIT_highbit32(rest); U32 const rest = total - weightTotal;
U32 const lastWeight = BIT_highbit32(rest) + 1; U32 const verif = 1 << BIT_highbit32(rest);
if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ U32 const lastWeight = BIT_highbit32(rest) + 1;
huffWeight[oSize] = (BYTE)lastWeight; if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
rankStats[lastWeight]++; huffWeight[oSize] = (BYTE)lastWeight;
} rankStats[lastWeight]++;
} }
/* check tree construction validity */ /* check tree construction validity */
if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
/* results */ /* results */
*nbSymbolsPtr = (U32)(oSize+1); *nbSymbolsPtr = (U32)(oSize+1);
*tableLogPtr = tableLog;
return iSize+1; return iSize+1;
} }

2
lib/common/mem.h
View File

@ -63,7 +63,7 @@ extern "C" {
/*-************************************************************** /*-**************************************************************
* Basic Types * Basic Types
*****************************************************************/ *****************************************************************/
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) #if !defined (__VMS) && ( defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
# include <stdint.h> # include <stdint.h>
typedef uint8_t BYTE; typedef uint8_t BYTE;
typedef uint16_t U16; typedef uint16_t U16;

8
lib/common/zstd_internal.h
View File

@ -203,10 +203,10 @@ typedef struct {
#include ".debug/zstd_stats.h" #include ".debug/zstd_stats.h"
#else #else
typedef struct { U32 unused; } ZSTD_stats_t; typedef struct { U32 unused; } ZSTD_stats_t;
MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; }; MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; }
MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; }; MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; }
MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; }; MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; }
MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }; MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }
#endif #endif
typedef struct { typedef struct {

8
lib/common/zstd_static.h
View File

@ -103,7 +103,7 @@ ZSTDLIB_API unsigned ZSTD_maxCLevel (void);
/*! ZSTD_getCParams() : /*! ZSTD_getCParams() :
* @return ZSTD_compressionParameters structure for a selected compression level and srcSize. * @return ZSTD_compressionParameters structure for a selected compression level and srcSize.
* `srcSize` value is optional, select 0 if not known */ * `srcSize` value is optional, select 0 if not known */
ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize); ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize);
/*! ZSTD_checkParams() : /*! ZSTD_checkParams() :
* Ensure param values remain within authorized range */ * Ensure param values remain within authorized range */
@ -250,8 +250,8 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
*/ */
#define ZSTD_BLOCKSIZE_MAX (128 * 1024) /* define, for static allocation */ #define ZSTD_BLOCKSIZE_MAX (128 * 1024) /* define, for static allocation */
size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/*-************************************* /*-*************************************
@ -261,7 +261,7 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, cons
/*! ZSTD_getErrorCode() : /*! ZSTD_getErrorCode() :
convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
which can be used to compare directly with enum list published into "error_public.h" */ which can be used to compare directly with enum list published into "error_public.h" */
ZSTD_ErrorCode ZSTD_getError(size_t code); ZSTDLIB_API ZSTD_ErrorCode ZSTD_getError(size_t code);
#if defined (__cplusplus) #if defined (__cplusplus)

2
lib/compress/zstd_compress.c
View File

@ -852,7 +852,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B
static const BYTE* g_start = NULL; static const BYTE* g_start = NULL;
const U32 pos = (U32)(literals - g_start); const U32 pos = (U32)(literals - g_start);
if (g_start==NULL) g_start = literals; if (g_start==NULL) g_start = literals;
if ((pos > 5810300) && (pos < 5810500)) if ((pos > 2587900) && (pos < 2588050))
printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n", printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n",
pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
#endif #endif

171
lib/decompress/huf_decompress.c
View File

@ -114,7 +114,7 @@ size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
/* Prepare ranks */ /* Prepare ranks */
nextRankStart = 0; nextRankStart = 0;
for (n=1; n<=tableLog; n++) { for (n=1; n<tableLog+1; n++) {
U32 current = nextRankStart; U32 current = nextRankStart;
nextRankStart += (rankVal[n] << (n-1)); nextRankStart += (rankVal[n] << (n-1));
rankVal[n] = current; rankVal[n] = current;
@ -442,7 +442,7 @@ size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
/* Get start index of each weight */ /* Get start index of each weight */
{ U32 w, nextRankStart = 0; { U32 w, nextRankStart = 0;
for (w=1; w<=maxW; w++) { for (w=1; w<maxW+1; w++) {
U32 current = nextRankStart; U32 current = nextRankStart;
nextRankStart += rankStats[w]; nextRankStart += rankStats[w];
rankStart[w] = current; rankStart[w] = current;
@ -454,8 +454,8 @@ size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
/* sort symbols by weight */ /* sort symbols by weight */
{ U32 s; { U32 s;
for (s=0; s<nbSymbols; s++) { for (s=0; s<nbSymbols; s++) {
U32 w = weightList[s]; U32 const w = weightList[s];
U32 r = rankStart[w]++; U32 const r = rankStart[w]++;
sortedSymbol[r].symbol = (BYTE)s; sortedSymbol[r].symbol = (BYTE)s;
sortedSymbol[r].weight = (BYTE)w; sortedSymbol[r].weight = (BYTE)w;
} }
@ -463,21 +463,23 @@ size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
} }
/* Build rankVal */ /* Build rankVal */
{ const U32 minBits = tableLog+1 - maxW; { U32* const rankVal0 = rankVal[0];
U32 nextRankVal = 0; { int const rescale = (memLog-tableLog) - 1; /* tableLog <= memLog */
U32 w, consumed; U32 nextRankVal = 0;
const int rescale = (memLog-tableLog) - 1; /* tableLog <= memLog */ U32 w;
U32* rankVal0 = rankVal[0]; for (w=1; w<maxW+1; w++) {
for (w=1; w<=maxW; w++) { U32 current = nextRankVal;
U32 current = nextRankVal; nextRankVal += rankStats[w] << (w+rescale);
nextRankVal += rankStats[w] << (w+rescale); rankVal0[w] = current;
rankVal0[w] = current; } }
} { U32 const minBits = tableLog+1 - maxW;
for (consumed = minBits; consumed <= memLog - minBits; consumed++) { U32 consumed;
U32* rankValPtr = rankVal[consumed]; for (consumed = minBits; consumed < memLog - minBits + 1; consumed++) {
for (w = 1; w <= maxW; w++) { U32* const rankValPtr = rankVal[consumed];
rankValPtr[w] = rankVal0[w] >> consumed; U32 w;
} } } for (w = 1; w < maxW+1; w++) {
rankValPtr[w] = rankVal0[w] >> consumed;
} } } }
HUF_fillDTableX4(dt, memLog, HUF_fillDTableX4(dt, memLog,
sortedSymbol, sizeOfSort, sortedSymbol, sizeOfSort,
@ -788,7 +790,7 @@ size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
/* Get start index of each weight */ /* Get start index of each weight */
{ U32 w, nextRankStart = 0; { U32 w, nextRankStart = 0;
for (w=1; w<=maxW; w++) { for (w=1; w<maxW+1; w++) {
U32 current = nextRankStart; U32 current = nextRankStart;
nextRankStart += rankStats[w]; nextRankStart += rankStats[w];
rankStart[w] = current; rankStart[w] = current;
@ -814,14 +816,14 @@ size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
U32 w, consumed; U32 w, consumed;
const int rescale = (memLog-tableLog) - 1; /* tableLog <= memLog */ const int rescale = (memLog-tableLog) - 1; /* tableLog <= memLog */
U32* rankVal0 = rankVal[0]; U32* rankVal0 = rankVal[0];
for (w=1; w<=maxW; w++) { for (w=1; w<maxW+1; w++) {
U32 current = nextRankVal; U32 current = nextRankVal;
nextRankVal += rankStats[w] << (w+rescale); nextRankVal += rankStats[w] << (w+rescale);
rankVal0[w] = current; rankVal0[w] = current;
} }
for (consumed = minBits; consumed <= memLog - minBits; consumed++) { for (consumed = minBits; consumed <= memLog - minBits; consumed++) {
U32* rankValPtr = rankVal[consumed]; U32* rankValPtr = rankVal[consumed];
for (w = 1; w <= maxW; w++) { for (w = 1; w < maxW+1; w++) {
rankValPtr[w] = rankVal0[w] >> consumed; rankValPtr[w] = rankVal0[w] >> consumed;
} } } } } }
@ -847,17 +849,17 @@ size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
static U32 HUF_decodeSymbolX6(void* op, BIT_DStream_t* DStream, const HUF_DDescX6* dd, const HUF_DSeqX6* ds, const U32 dtLog) static U32 HUF_decodeSymbolX6(void* op, BIT_DStream_t* DStream, const HUF_DDescX6* dd, const HUF_DSeqX6* ds, const U32 dtLog)
{ {
const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
memcpy(op, ds+val, sizeof(HUF_DSeqX6)); memcpy(op, ds+val, sizeof(HUF_DSeqX6));
BIT_skipBits(DStream, dd[val].nbBits); BIT_skipBits(DStream, dd[val].nbBits);
return dd[val].nbBytes; return dd[val].nbBytes;
} }
static U32 HUF_decodeLastSymbolsX6(void* op, const U32 maxL, BIT_DStream_t* DStream, static U32 HUF_decodeLastSymbolsX6(void* op, U32 const maxL, BIT_DStream_t* DStream,
const HUF_DDescX6* dd, const HUF_DSeqX6* ds, const U32 dtLog) const HUF_DDescX6* dd, const HUF_DSeqX6* ds, const U32 dtLog)
{ {
const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
U32 length = dd[val].nbBytes; U32 const length = dd[val].nbBytes;
if (length <= maxL) { if (length <= maxL) {
memcpy(op, ds+val, length); memcpy(op, ds+val, length);
BIT_skipBits(DStream, dd[val].nbBits); BIT_skipBits(DStream, dd[val].nbBits);
@ -910,7 +912,6 @@ static inline size_t HUF_decodeStreamX6(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* c
return p-pStart; return p-pStart;
} }
size_t HUF_decompress1X6_usingDTable( size_t HUF_decompress1X6_usingDTable(
void* dst, size_t dstSize, void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize, const void* cSrc, size_t cSrcSize,
@ -919,17 +920,15 @@ size_t HUF_decompress1X6_usingDTable(
const BYTE* const istart = (const BYTE*) cSrc; const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst; BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize; BYTE* const oend = ostart + dstSize;
BIT_DStream_t bitD;
const U32 dtLog = DTable[0];
size_t errorCode;
/* Init */ /* Init */
BIT_DStream_t bitD; { size_t const errorCode = BIT_initDStream(&bitD, istart, cSrcSize);
errorCode = BIT_initDStream(&bitD, istart, cSrcSize); if (HUF_isError(errorCode)) return errorCode; }
if (HUF_isError(errorCode)) return errorCode;
/* finish bitStreams one by one */ /* finish bitStreams one by one */
HUF_decodeStreamX6(ostart, &bitD, oend, DTable, dtLog); { U32 const dtLog = DTable[0];
HUF_decodeStreamX6(ostart, &bitD, oend, DTable, dtLog); }
/* check */ /* check */
if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected);
@ -943,7 +942,7 @@ size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cS
HUF_CREATE_STATIC_DTABLEX6(DTable, HUF_MAX_TABLELOG); HUF_CREATE_STATIC_DTABLEX6(DTable, HUF_MAX_TABLELOG);
const BYTE* ip = (const BYTE*) cSrc; const BYTE* ip = (const BYTE*) cSrc;
size_t hSize = HUF_readDTableX6 (DTable, cSrc, cSrcSize); size_t const hSize = HUF_readDTableX6 (DTable, cSrc, cSrcSize);
if (HUF_isError(hSize)) return hSize; if (HUF_isError(hSize)) return hSize;
if (hSize >= cSrcSize) return ERROR(srcSize_wrong); if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
ip += hSize; ip += hSize;
@ -953,6 +952,24 @@ size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cS
} }
#define HUF_DECODE_ROUNDX6 \
HUF_DECODE_SYMBOLX6_2(op1, &bitD1); \
HUF_DECODE_SYMBOLX6_2(op2, &bitD2); \
HUF_DECODE_SYMBOLX6_2(op3, &bitD3); \
HUF_DECODE_SYMBOLX6_2(op4, &bitD4); \
HUF_DECODE_SYMBOLX6_1(op1, &bitD1); \
HUF_DECODE_SYMBOLX6_1(op2, &bitD2); \
HUF_DECODE_SYMBOLX6_1(op3, &bitD3); \
HUF_DECODE_SYMBOLX6_1(op4, &bitD4); \
HUF_DECODE_SYMBOLX6_2(op1, &bitD1); \
HUF_DECODE_SYMBOLX6_2(op2, &bitD2); \
HUF_DECODE_SYMBOLX6_2(op3, &bitD3); \
HUF_DECODE_SYMBOLX6_2(op4, &bitD4); \
HUF_DECODE_SYMBOLX6_0(op1, &bitD1); \
HUF_DECODE_SYMBOLX6_0(op2, &bitD2); \
HUF_DECODE_SYMBOLX6_0(op3, &bitD3); \
HUF_DECODE_SYMBOLX6_0(op4, &bitD4);
size_t HUF_decompress4X6_usingDTable( size_t HUF_decompress4X6_usingDTable(
void* dst, size_t dstSize, void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize, const void* cSrc, size_t cSrcSize,
@ -960,6 +977,7 @@ size_t HUF_decompress4X6_usingDTable(
{ {
/* Check */ /* Check */
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
if (dstSize < 64) return ERROR(dstSize_tooSmall); /* only work for dstSize >= 64 */
{ const BYTE* const istart = (const BYTE*) cSrc; { const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst; BYTE* const ostart = (BYTE*) dst;
@ -970,7 +988,6 @@ size_t HUF_decompress4X6_usingDTable(
const HUF_DDescX6* dd = (const HUF_DDescX6*)ddPtr; const HUF_DDescX6* dd = (const HUF_DDescX6*)ddPtr;
const void* const dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1)); const void* const dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1));
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)dsPtr; const HUF_DSeqX6* ds = (const HUF_DSeqX6*)dsPtr;
size_t errorCode;
/* Init */ /* Init */
BIT_DStream_t bitD1; BIT_DStream_t bitD1;
@ -997,43 +1014,41 @@ size_t HUF_decompress4X6_usingDTable(
length4 = cSrcSize - (length1 + length2 + length3 + 6); length4 = cSrcSize - (length1 + length2 + length3 + 6);
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
errorCode = BIT_initDStream(&bitD1, istart1, length1); { size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1);
if (HUF_isError(errorCode)) return errorCode; if (HUF_isError(errorCode)) return errorCode; }
errorCode = BIT_initDStream(&bitD2, istart2, length2); { size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2);
if (HUF_isError(errorCode)) return errorCode; if (HUF_isError(errorCode)) return errorCode; }
errorCode = BIT_initDStream(&bitD3, istart3, length3); { size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3);
if (HUF_isError(errorCode)) return errorCode; if (HUF_isError(errorCode)) return errorCode; }
errorCode = BIT_initDStream(&bitD4, istart4, length4); { size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4);
if (HUF_isError(errorCode)) return errorCode; if (HUF_isError(errorCode)) return errorCode; }
/* 16-64 symbols per loop (4-16 symbols per stream) */ /* 4-64 symbols per loop (1-16 symbols per stream) */
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
for ( ; (op3 <= opStart4) && (endSignal==BIT_DStream_unfinished) && (op4<=(oend-16)) ; ) { if (endSignal==BIT_DStream_unfinished) {
HUF_DECODE_SYMBOLX6_2(op1, &bitD1); HUF_DECODE_ROUNDX6;
HUF_DECODE_SYMBOLX6_2(op2, &bitD2); if (sizeof(bitD1.bitContainer)==4) { /* need to decode at least 4 bytes per stream */
HUF_DECODE_SYMBOLX6_2(op3, &bitD3); endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
HUF_DECODE_SYMBOLX6_2(op4, &bitD4); HUF_DECODE_ROUNDX6;
HUF_DECODE_SYMBOLX6_1(op1, &bitD1); }
HUF_DECODE_SYMBOLX6_1(op2, &bitD2); { U32 const saved2 = MEM_read32(opStart2); /* saved from overwrite */
HUF_DECODE_SYMBOLX6_1(op3, &bitD3); U32 const saved3 = MEM_read32(opStart3);
HUF_DECODE_SYMBOLX6_1(op4, &bitD4); U32 const saved4 = MEM_read32(opStart4);
HUF_DECODE_SYMBOLX6_2(op1, &bitD1); endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
HUF_DECODE_SYMBOLX6_2(op2, &bitD2); for ( ; (op3 <= opStart4) && (endSignal==BIT_DStream_unfinished) && (op4<=(oend-16)) ; ) {
HUF_DECODE_SYMBOLX6_2(op3, &bitD3); HUF_DECODE_ROUNDX6;
HUF_DECODE_SYMBOLX6_2(op4, &bitD4); endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
HUF_DECODE_SYMBOLX6_0(op1, &bitD1); }
HUF_DECODE_SYMBOLX6_0(op2, &bitD2); MEM_write32(opStart2, saved2);
HUF_DECODE_SYMBOLX6_0(op3, &bitD3); MEM_write32(opStart3, saved3);
HUF_DECODE_SYMBOLX6_0(op4, &bitD4); MEM_write32(opStart4, saved4);
} }
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
}
/* check corruption */ /* check corruption */
if (op1 > opStart2) return ERROR(corruption_detected); if (op1 > opStart2) return ERROR(corruption_detected);
if (op2 > opStart3) return ERROR(corruption_detected); if (op2 > opStart3) return ERROR(corruption_detected);
if (op3 > opStart4) return ERROR(corruption_detected); if (op3 > opStart4) return ERROR(corruption_detected);
/* note : op4 supposed already verified within main loop */ /* note : op4 already verified within main loop */
/* finish bitStreams one by one */ /* finish bitStreams one by one */
HUF_decodeStreamX6(op1, &bitD1, opStart2, DTable, dtLog); HUF_decodeStreamX6(op1, &bitD1, opStart2, DTable, dtLog);
@ -1097,12 +1112,7 @@ typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc,
size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{ {
static const decompressionAlgo decompress[3] = { HUF_decompress4X2, HUF_decompress4X4, HUF_decompress4X6 }; static const decompressionAlgo decompress[3] = { HUF_decompress4X2, HUF_decompress4X4, HUF_decompress4X6 };
/* estimate decompression time */ U32 Dtime[3]; /* decompression time estimation */
U32 Q;
const U32 D256 = (U32)(dstSize >> 8);
U32 Dtime[3];
U32 algoNb = 0;
int n;
/* validation checks */ /* validation checks */
if (dstSize == 0) return ERROR(dstSize_tooSmall); if (dstSize == 0) return ERROR(dstSize_tooSmall);
@ -1111,16 +1121,19 @@ size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcS
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
/* decoder timing evaluation */ /* decoder timing evaluation */
Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */ { U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */
for (n=0; n<3; n++) U32 const D256 = (U32)(dstSize >> 8);
Dtime[n] = algoTime[Q][n].tableTime + (algoTime[Q][n].decode256Time * D256); U32 n; for (n=0; n<3; n++)
Dtime[n] = algoTime[Q][n].tableTime + (algoTime[Q][n].decode256Time * D256);
}
Dtime[1] += Dtime[1] >> 4; Dtime[2] += Dtime[2] >> 3; /* advantage to algorithms using less memory, for cache eviction */ Dtime[1] += Dtime[1] >> 4; Dtime[2] += Dtime[2] >> 3; /* advantage to algorithms using less memory, for cache eviction */
if (Dtime[1] < Dtime[0]) algoNb = 1; { U32 algoNb = 0;
if (Dtime[2] < Dtime[algoNb]) algoNb = 2; if (Dtime[1] < Dtime[0]) algoNb = 1;
if (Dtime[2] < Dtime[algoNb]) algoNb = 2;
return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
}
//return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */ //return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */
//return HUF_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */ //return HUF_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */

74
lib/decompress/zstd_decompress.c
View File

@ -266,10 +266,9 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
* @return : size of the Frame Header */ * @return : size of the Frame Header */
static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
{ {
U32 fcsId;
if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
fcsId = (((const BYTE*)src)[4]) >> 6; { U32 const fcsId = (((const BYTE*)src)[4]) >> 6;
return ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId]; return ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId]; }
} }
@ -843,14 +842,14 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
} }
/*! ZSTD_decompress_continueDCtx() : /*! ZSTD_decompressFrame() :
* `dctx` must have been properly initialized */ * `dctx` must be properly initialized */
static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
const void* src, size_t srcSize) const void* src, size_t srcSize)
{ {
const BYTE* ip = (const BYTE*)src; const BYTE* ip = (const BYTE*)src;
const BYTE* iend = ip + srcSize; const BYTE* const iend = ip + srcSize;
BYTE* const ostart = (BYTE* const)dst; BYTE* const ostart = (BYTE* const)dst;
BYTE* op = ostart; BYTE* op = ostart;
BYTE* const oend = ostart + dstCapacity; BYTE* const oend = ostart + dstCapacity;
@ -859,12 +858,6 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
/* check */ /* check */
if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
{ const U32 magicNumber = MEM_readLE32(src);
if (ZSTD_isLegacy(magicNumber))
return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, magicNumber);
}
#endif
/* Frame Header */ /* Frame Header */
{ size_t const frameHeaderSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min); { size_t const frameHeaderSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min);
@ -929,6 +922,12 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
const void* src, size_t srcSize, const void* src, size_t srcSize,
const void* dict, size_t dictSize) const void* dict, size_t dictSize)
{ {
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
{ const U32 magicNumber = MEM_readLE32(src);
if (ZSTD_isLegacy(magicNumber))
return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize, magicNumber);
}
#endif
ZSTD_decompressBegin_usingDict(dctx, dict, dictSize); ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
ZSTD_checkContinuity(dctx, dst); ZSTD_checkContinuity(dctx, dst);
return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize); return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
@ -950,7 +949,7 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr
regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize);
ZSTD_freeDCtx(dctx); ZSTD_freeDCtx(dctx);
return regenSize; return regenSize;
#else #else /* stack mode */
ZSTD_DCtx dctx; ZSTD_DCtx dctx;
return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize); return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize);
#endif #endif
@ -1054,37 +1053,40 @@ static void ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSi
static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
{ {
size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, errorCode, litlengthHeaderSize; size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, litlengthHeaderSize;
short offcodeNCount[MaxOff+1];
U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
short matchlengthNCount[MaxML+1];
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
short litlengthNCount[MaxLL+1];
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize); hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize);
if (HUF_isError(hSize)) return ERROR(dictionary_corrupted); if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
dict = (const char*)dict + hSize; dict = (const char*)dict + hSize;
dictSize -= hSize; dictSize -= hSize;
offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize); { short offcodeNCount[MaxOff+1];
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
errorCode = FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog); offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
dict = (const char*)dict + offcodeHeaderSize; { size_t const errorCode = FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
dictSize -= offcodeHeaderSize; if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
dict = (const char*)dict + offcodeHeaderSize;
dictSize -= offcodeHeaderSize;
}
matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize); { short matchlengthNCount[MaxML+1];
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
errorCode = FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
dict = (const char*)dict + matchlengthHeaderSize; { size_t const errorCode = FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
dictSize -= matchlengthHeaderSize; if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
dict = (const char*)dict + matchlengthHeaderSize;
dictSize -= matchlengthHeaderSize;
}
litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize); { short litlengthNCount[MaxLL+1];
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
errorCode = FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog); litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
}
dctx->flagRepeatTable = 1; dctx->flagRepeatTable = 1;
return hSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize; return hSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize;

28
lib/legacy/zstd_legacy.h
View File

@ -1,7 +1,7 @@
/* /*
zstd_legacy - decoder for legacy format zstd_legacy - decoder for legacy format
Header File Header File
Copyright (C) 2015, Yann Collet. Copyright (C) 2015-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@ -48,16 +48,20 @@ extern "C" {
#include "zstd_v04.h" #include "zstd_v04.h"
#include "zstd_v05.h" #include "zstd_v05.h"
/** ZSTD_isLegacy() :
@return : > 0 if supported by legacy decoder. 0 otherwise.
return value is the version.
*/
MEM_STATIC unsigned ZSTD_isLegacy (U32 magicNumberLE) MEM_STATIC unsigned ZSTD_isLegacy (U32 magicNumberLE)
{ {
switch(magicNumberLE) switch(magicNumberLE)
{ {
case ZSTDv01_magicNumberLE : case ZSTDv01_magicNumberLE:return 1;
case ZSTDv02_magicNumber : case ZSTDv02_magicNumber : return 2;
case ZSTDv03_magicNumber : case ZSTDv03_magicNumber : return 3;
case ZSTDv04_magicNumber : case ZSTDv04_magicNumber : return 4;
case ZSTDv05_MAGICNUMBER : case ZSTDv05_MAGICNUMBER : return 5;
return 1;
default : return 0; default : return 0;
} }
} }
@ -66,6 +70,7 @@ MEM_STATIC unsigned ZSTD_isLegacy (U32 magicNumberLE)
MEM_STATIC size_t ZSTD_decompressLegacy( MEM_STATIC size_t ZSTD_decompressLegacy(
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
const void* src, size_t compressedSize, const void* src, size_t compressedSize,
const void* dict,size_t dictSize,
U32 magicNumberLE) U32 magicNumberLE)
{ {
switch(magicNumberLE) switch(magicNumberLE)
@ -79,7 +84,14 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
case ZSTDv04_magicNumber : case ZSTDv04_magicNumber :
return ZSTDv04_decompress(dst, dstCapacity, src, compressedSize); return ZSTDv04_decompress(dst, dstCapacity, src, compressedSize);
case ZSTDv05_MAGICNUMBER : case ZSTDv05_MAGICNUMBER :
return ZSTDv05_decompress(dst, dstCapacity, src, compressedSize); {
size_t result;
ZSTDv05_DCtx* zd = ZSTDv05_createDCtx();
if (zd==NULL) return ERROR(memory_allocation);
result = ZSTDv05_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize);
ZSTDv05_freeDCtx(zd);
return result;
}
default : default :
return ERROR(prefix_unknown); return ERROR(prefix_unknown);
} }

4
lib/legacy/zstd_v01.c
View File

@ -407,7 +407,7 @@ FORCE_INLINE unsigned FSE_highbit32 (register U32 val)
static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; } static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; }
#define FSE_DECODE_TYPE FSE_TYPE_NAME(FSE_decode_t, FSE_FUNCTION_EXTENSION) #define FSE_DECODE_TYPE FSE_decode_t
typedef struct { typedef struct {
@ -415,7 +415,7 @@ typedef struct {
U16 fastMode; U16 fastMode;
} FSE_DTableHeader; /* sizeof U32 */ } FSE_DTableHeader; /* sizeof U32 */
static size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION) static size_t FSE_buildDTable
(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{ {
void* ptr = dt; void* ptr = dt;

4
lib/legacy/zstd_v02.c
View File

@ -1266,11 +1266,11 @@ typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
/* Function templates */ /* Function templates */
#define FSE_DECODE_TYPE FSE_TYPE_NAME(FSE_decode_t, FSE_FUNCTION_EXTENSION) #define FSE_DECODE_TYPE FSE_decode_t
static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; } static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; }
static size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION) static size_t FSE_buildDTable
(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{ {
void* ptr = dt+1; void* ptr = dt+1;

364
lib/legacy/zstd_v03.c
View File

@ -991,7 +991,6 @@ extern "C" {
******************************************/ ******************************************/
static size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ static size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
static size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */ static size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */
static size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder */
#if defined (__cplusplus) #if defined (__cplusplus)
@ -1266,11 +1265,11 @@ typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
/* Function templates */ /* Function templates */
#define FSE_DECODE_TYPE FSE_TYPE_NAME(FSE_decode_t, FSE_FUNCTION_EXTENSION) #define FSE_DECODE_TYPE FSE_decode_t
static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; } static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; }
static size_t FSE_FUNCTION_NAME(FSE_buildDTable, FSE_FUNCTION_EXTENSION) static size_t FSE_buildDTable
(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{ {
void* ptr = dt+1; void* ptr = dt+1;
@ -2371,362 +2370,6 @@ static size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, si
} }
/**********************************/
/* quad-symbol decoding */
/**********************************/
typedef struct { BYTE nbBits; BYTE nbBytes; } HUF_DDescX6;
typedef union { BYTE byte[4]; U32 sequence; } HUF_DSeqX6;
/* recursive, up to level 3; may benefit from <template>-like strategy to nest each level inline */
static void HUF_fillDTableX6LevelN(HUF_DDescX6* DDescription, HUF_DSeqX6* DSequence, int sizeLog,
const rankVal_t rankValOrigin, const U32 consumed, const int minWeight, const U32 maxWeight,
const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, const U32* rankStart,
const U32 nbBitsBaseline, HUF_DSeqX6 baseSeq, HUF_DDescX6 DDesc)
{
const int scaleLog = nbBitsBaseline - sizeLog; /* note : targetLog >= (nbBitsBaseline-1), hence scaleLog <= 1 */
const int minBits = nbBitsBaseline - maxWeight;
const U32 level = DDesc.nbBytes;
U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1];
U32 symbolStartPos, s;
/* local rankVal, will be modified */
memcpy(rankVal, rankValOrigin[consumed], sizeof(rankVal));
/* fill skipped values */
if (minWeight>1)
{
U32 i;
const U32 skipSize = rankVal[minWeight];
for (i = 0; i < skipSize; i++)
{
DSequence[i] = baseSeq;
DDescription[i] = DDesc;
}
}
/* fill DTable */
DDesc.nbBytes++;
symbolStartPos = rankStart[minWeight];
for (s=symbolStartPos; s<sortedListSize; s++)
{
const BYTE symbol = sortedSymbols[s].symbol;
const U32 weight = sortedSymbols[s].weight; /* >= 1 (sorted) */
const int nbBits = nbBitsBaseline - weight; /* >= 1 (by construction) */
const int totalBits = consumed+nbBits;
const U32 start = rankVal[weight];
const U32 length = 1 << (sizeLog-nbBits);
baseSeq.byte[level] = symbol;
DDesc.nbBits = (BYTE)totalBits;
if ((level<3) && (sizeLog-totalBits >= minBits)) /* enough room for another symbol */
{
int nextMinWeight = totalBits + scaleLog;
if (nextMinWeight < 1) nextMinWeight = 1;
HUF_fillDTableX6LevelN(DDescription+start, DSequence+start, sizeLog-nbBits,
rankValOrigin, totalBits, nextMinWeight, maxWeight,
sortedSymbols, sortedListSize, rankStart,
nbBitsBaseline, baseSeq, DDesc); /* recursive (max : level 3) */
}
else
{
U32 i;
const U32 end = start + length;
for (i = start; i < end; i++)
{
DDescription[i] = DDesc;
DSequence[i] = baseSeq;
}
}
rankVal[weight] += length;
}
}
/* note : same preparation as X4 */
static size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
{
BYTE weightList[HUF_MAX_SYMBOL_VALUE + 1];
sortedSymbol_t sortedSymbol[HUF_MAX_SYMBOL_VALUE + 1];
U32 rankStats[HUF_ABSOLUTEMAX_TABLELOG + 1] = { 0 };
U32 rankStart0[HUF_ABSOLUTEMAX_TABLELOG + 2] = { 0 };
U32* const rankStart = rankStart0+1;
U32 tableLog, maxW, sizeOfSort, nbSymbols;
rankVal_t rankVal;
const U32 memLog = DTable[0];
const BYTE* ip = (const BYTE*) src;
size_t iSize = ip[0];
if (memLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
//memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
iSize = HUF_readStats(weightList, HUF_MAX_SYMBOL_VALUE + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
if (HUF_isError(iSize)) return iSize;
/* check result */
if (tableLog > memLog) return ERROR(tableLog_tooLarge); /* DTable is too small */
/* find maxWeight */
for (maxW = tableLog; rankStats[maxW]==0; maxW--)
{ if (!maxW) return ERROR(GENERIC); } /* necessarily finds a solution before maxW==0 */
/* Get start index of each weight */
{
U32 w, nextRankStart = 0;
for (w=1; w<=maxW; w++)
{
U32 current = nextRankStart;
nextRankStart += rankStats[w];
rankStart[w] = current;
}
rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/
sizeOfSort = nextRankStart;
}
/* sort symbols by weight */
{
U32 s;
for (s=0; s<nbSymbols; s++)
{
U32 w = weightList[s];
U32 r = rankStart[w]++;
sortedSymbol[r].symbol = (BYTE)s;
sortedSymbol[r].weight = (BYTE)w;
}
rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
}
/* Build rankVal */
{
const U32 minBits = tableLog+1 - maxW;
U32 nextRankVal = 0;
U32 w, consumed;
const int rescale = (memLog-tableLog) - 1; /* tableLog <= memLog */
U32* rankVal0 = rankVal[0];
for (w=1; w<=maxW; w++)
{
U32 current = nextRankVal;
nextRankVal += rankStats[w] << (w+rescale);
rankVal0[w] = current;
}
for (consumed = minBits; consumed <= memLog - minBits; consumed++)
{
U32* rankValPtr = rankVal[consumed];
for (w = 1; w <= maxW; w++)
{
rankValPtr[w] = rankVal0[w] >> consumed;
}
}
}
/* fill tables */
{
void* ddPtr = DTable+1;
HUF_DDescX6* DDescription = (HUF_DDescX6*)(ddPtr);
void* dsPtr = DTable + 1 + ((size_t)1<<(memLog-1));
HUF_DSeqX6* DSequence = (HUF_DSeqX6*)(dsPtr);
HUF_DSeqX6 DSeq;
HUF_DDescX6 DDesc;
DSeq.sequence = 0;
DDesc.nbBits = 0;
DDesc.nbBytes = 0;
HUF_fillDTableX6LevelN(DDescription, DSequence, memLog,
(const U32 (*)[HUF_ABSOLUTEMAX_TABLELOG + 1])rankVal, 0, 1, maxW,
sortedSymbol, sizeOfSort, rankStart0,
tableLog+1, DSeq, DDesc);
}
return iSize;
}
static U32 HUF_decodeSymbolX6(void* op, BIT_DStream_t* DStream, const HUF_DDescX6* dd, const HUF_DSeqX6* ds, const U32 dtLog)
{
const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
memcpy(op, ds+val, sizeof(HUF_DSeqX6));
BIT_skipBits(DStream, dd[val].nbBits);
return dd[val].nbBytes;
}
static U32 HUF_decodeLastSymbolsX6(void* op, const U32 maxL, BIT_DStream_t* DStream,
const HUF_DDescX6* dd, const HUF_DSeqX6* ds, const U32 dtLog)
{
const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
U32 length = dd[val].nbBytes;
if (length <= maxL)
{
memcpy(op, ds+val, length);
BIT_skipBits(DStream, dd[val].nbBits);
return length;
}
memcpy(op, ds+val, maxL);
if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8))
{
BIT_skipBits(DStream, dd[val].nbBits);
if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8))
DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
}
return maxL;
}
#define HUF_DECODE_SYMBOLX6_0(ptr, DStreamPtr) \
ptr += HUF_decodeSymbolX6(ptr, DStreamPtr, dd, ds, dtLog)
#define HUF_DECODE_SYMBOLX6_1(ptr, DStreamPtr) \
if (MEM_64bits() || (HUF_MAX_TABLELOG<=12)) \
HUF_DECODE_SYMBOLX6_0(ptr, DStreamPtr)
#define HUF_DECODE_SYMBOLX6_2(ptr, DStreamPtr) \
if (MEM_64bits()) \
HUF_DECODE_SYMBOLX6_0(ptr, DStreamPtr)
static inline size_t HUF_decodeStreamX6(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const U32* DTable, const U32 dtLog)
{
const void* ddPtr = DTable+1;
const HUF_DDescX6* dd = (const HUF_DDescX6*)(ddPtr);
const void* dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1));
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(dsPtr);
BYTE* const pStart = p;
/* up to 16 symbols at a time */
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-16))
{
HUF_DECODE_SYMBOLX6_2(p, bitDPtr);
HUF_DECODE_SYMBOLX6_1(p, bitDPtr);
HUF_DECODE_SYMBOLX6_2(p, bitDPtr);
HUF_DECODE_SYMBOLX6_0(p, bitDPtr);
}
/* closer to the end, up to 4 symbols at a time */
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-4))
HUF_DECODE_SYMBOLX6_0(p, bitDPtr);
while (p <= pEnd-4)
HUF_DECODE_SYMBOLX6_0(p, bitDPtr); /* no need to reload : reached the end of DStream */
while (p < pEnd)
p += HUF_decodeLastSymbolsX6(p, (U32)(pEnd-p), bitDPtr, dd, ds, dtLog);
return p-pStart;
}
static size_t HUF_decompress4X6_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const U32* DTable)
{
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
{
const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const U32 dtLog = DTable[0];
const void* ddPtr = DTable+1;
const HUF_DDescX6* dd = (const HUF_DDescX6*)(ddPtr);
const void* dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1));
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)(dsPtr);
size_t errorCode;
/* Init */
BIT_DStream_t bitD1;
BIT_DStream_t bitD2;
BIT_DStream_t bitD3;
BIT_DStream_t bitD4;
const size_t length1 = MEM_readLE16(istart);
const size_t length2 = MEM_readLE16(istart+2);
const size_t length3 = MEM_readLE16(istart+4);
size_t length4;
const BYTE* const istart1 = istart + 6; /* jumpTable */
const BYTE* const istart2 = istart1 + length1;
const BYTE* const istart3 = istart2 + length2;
const BYTE* const istart4 = istart3 + length3;
const size_t segmentSize = (dstSize+3) / 4;
BYTE* const opStart2 = ostart + segmentSize;
BYTE* const opStart3 = opStart2 + segmentSize;
BYTE* const opStart4 = opStart3 + segmentSize;
BYTE* op1 = ostart;
BYTE* op2 = opStart2;
BYTE* op3 = opStart3;
BYTE* op4 = opStart4;
U32 endSignal;
length4 = cSrcSize - (length1 + length2 + length3 + 6);
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
errorCode = BIT_initDStream(&bitD1, istart1, length1);
if (HUF_isError(errorCode)) return errorCode;
errorCode = BIT_initDStream(&bitD2, istart2, length2);
if (HUF_isError(errorCode)) return errorCode;
errorCode = BIT_initDStream(&bitD3, istart3, length3);
if (HUF_isError(errorCode)) return errorCode;
errorCode = BIT_initDStream(&bitD4, istart4, length4);
if (HUF_isError(errorCode)) return errorCode;
/* 16-64 symbols per loop (4-16 symbols per stream) */
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
for ( ; (op3 <= opStart4) && (endSignal==BIT_DStream_unfinished) && (op4<=(oend-16)) ; )
{
HUF_DECODE_SYMBOLX6_2(op1, &bitD1);
HUF_DECODE_SYMBOLX6_2(op2, &bitD2);
HUF_DECODE_SYMBOLX6_2(op3, &bitD3);
HUF_DECODE_SYMBOLX6_2(op4, &bitD4);
HUF_DECODE_SYMBOLX6_1(op1, &bitD1);
HUF_DECODE_SYMBOLX6_1(op2, &bitD2);
HUF_DECODE_SYMBOLX6_1(op3, &bitD3);
HUF_DECODE_SYMBOLX6_1(op4, &bitD4);
HUF_DECODE_SYMBOLX6_2(op1, &bitD1);
HUF_DECODE_SYMBOLX6_2(op2, &bitD2);
HUF_DECODE_SYMBOLX6_2(op3, &bitD3);
HUF_DECODE_SYMBOLX6_2(op4, &bitD4);
HUF_DECODE_SYMBOLX6_0(op1, &bitD1);
HUF_DECODE_SYMBOLX6_0(op2, &bitD2);
HUF_DECODE_SYMBOLX6_0(op3, &bitD3);
HUF_DECODE_SYMBOLX6_0(op4, &bitD4);
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
}
/* check corruption */
if (op1 > opStart2) return ERROR(corruption_detected);
if (op2 > opStart3) return ERROR(corruption_detected);
if (op3 > opStart4) return ERROR(corruption_detected);
/* note : op4 supposed already verified within main loop */
/* finish bitStreams one by one */
HUF_decodeStreamX6(op1, &bitD1, opStart2, DTable, dtLog);
HUF_decodeStreamX6(op2, &bitD2, opStart3, DTable, dtLog);
HUF_decodeStreamX6(op3, &bitD3, opStart4, DTable, dtLog);
HUF_decodeStreamX6(op4, &bitD4, oend, DTable, dtLog);
/* check */
endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
if (!endSignal) return ERROR(corruption_detected);
/* decoded size */
return dstSize;
}
}
static size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
HUF_CREATE_STATIC_DTABLEX6(DTable, HUF_MAX_TABLELOG);
const BYTE* ip = (const BYTE*) cSrc;
size_t hSize = HUF_readDTableX6 (DTable, cSrc, cSrcSize);
if (HUF_isError(hSize)) return hSize;
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
ip += hSize;
cSrcSize -= hSize;
return HUF_decompress4X6_usingDTable (dst, dstSize, ip, cSrcSize, DTable);
}
/**********************************/ /**********************************/
/* Generic decompression selector */ /* Generic decompression selector */
/**********************************/ /**********************************/
@ -2757,7 +2400,7 @@ typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc,
static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{ {
static const decompressionAlgo decompress[3] = { HUF_decompress4X2, HUF_decompress4X4, HUF_decompress4X6 }; static const decompressionAlgo decompress[3] = { HUF_decompress4X2, HUF_decompress4X4, NULL };
/* estimate decompression time */ /* estimate decompression time */
U32 Q; U32 Q;
const U32 D256 = (U32)(dstSize >> 8); const U32 D256 = (U32)(dstSize >> 8);
@ -2779,7 +2422,6 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
Dtime[1] += Dtime[1] >> 4; Dtime[2] += Dtime[2] >> 3; /* advantage to algorithms using less memory, for cache eviction */ Dtime[1] += Dtime[1] >> 4; Dtime[2] += Dtime[2] >> 3; /* advantage to algorithms using less memory, for cache eviction */
if (Dtime[1] < Dtime[0]) algoNb = 1; if (Dtime[1] < Dtime[0]) algoNb = 1;
if (Dtime[2] < Dtime[algoNb]) algoNb = 2;
return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);

356
lib/legacy/zstd_v04.c
View File

@ -2023,7 +2023,6 @@ extern "C" {
******************************************/ ******************************************/
static size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ static size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
static size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */ static size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */
static size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder */
/* **************************************** /* ****************************************
@ -2038,11 +2037,9 @@ HUF_decompress() does the following:
*/ */
static size_t HUF_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize); static size_t HUF_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize);
static size_t HUF_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize); static size_t HUF_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize);
static size_t HUF_readDTableX6 (unsigned* DTable, const void* src, size_t srcSize);
static size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable); static size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable);
static size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); static size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
static size_t HUF_decompress4X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
#if defined (__cplusplus) #if defined (__cplusplus)
@ -2793,356 +2790,6 @@ static size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, si
} }
/**********************************/
/* quad-symbol decoding */
/**********************************/
typedef struct { BYTE nbBits; BYTE nbBytes; } HUF_DDescX6;
typedef union { BYTE byte[4]; U32 sequence; } HUF_DSeqX6;
/* recursive, up to level 3; may benefit from <template>-like strategy to nest each level inline */
static void HUF_fillDTableX6LevelN(HUF_DDescX6* DDescription, HUF_DSeqX6* DSequence, int sizeLog,
const rankVal_t rankValOrigin, const U32 consumed, const int minWeight, const U32 maxWeight,
const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, const U32* rankStart,
const U32 nbBitsBaseline, HUF_DSeqX6 baseSeq, HUF_DDescX6 DDesc)
{
const int scaleLog = nbBitsBaseline - sizeLog; /* note : targetLog >= (nbBitsBaseline-1), hence scaleLog <= 1 */
const int minBits = nbBitsBaseline - maxWeight;
const U32 level = DDesc.nbBytes;
U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1];
U32 symbolStartPos, s;
/* local rankVal, will be modified */
memcpy(rankVal, rankValOrigin[consumed], sizeof(rankVal));
/* fill skipped values */
if (minWeight>1)
{
U32 i;
const U32 skipSize = rankVal[minWeight];
for (i = 0; i < skipSize; i++)
{
DSequence[i] = baseSeq;
DDescription[i] = DDesc;
}
}
/* fill DTable */
DDesc.nbBytes++;
symbolStartPos = rankStart[minWeight];
for (s=symbolStartPos; s<sortedListSize; s++)
{
const BYTE symbol = sortedSymbols[s].symbol;
const U32 weight = sortedSymbols[s].weight; /* >= 1 (sorted) */
const int nbBits = nbBitsBaseline - weight; /* >= 1 (by construction) */
const int totalBits = consumed+nbBits;
const U32 start = rankVal[weight];
const U32 length = 1 << (sizeLog-nbBits);
baseSeq.byte[level] = symbol;
DDesc.nbBits = (BYTE)totalBits;
if ((level<3) && (sizeLog-totalBits >= minBits)) /* enough room for another symbol */
{
int nextMinWeight = totalBits + scaleLog;
if (nextMinWeight < 1) nextMinWeight = 1;
HUF_fillDTableX6LevelN(DDescription+start, DSequence+start, sizeLog-nbBits,
rankValOrigin, totalBits, nextMinWeight, maxWeight,
sortedSymbols, sortedListSize, rankStart,
nbBitsBaseline, baseSeq, DDesc); /* recursive (max : level 3) */
}
else
{
U32 i;
const U32 end = start + length;
for (i = start; i < end; i++)
{
DDescription[i] = DDesc;
DSequence[i] = baseSeq;
}
}
rankVal[weight] += length;
}
}
/* note : same preparation as X4 */
static size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
{
BYTE weightList[HUF_MAX_SYMBOL_VALUE + 1];
sortedSymbol_t sortedSymbol[HUF_MAX_SYMBOL_VALUE + 1];
U32 rankStats[HUF_ABSOLUTEMAX_TABLELOG + 1] = { 0 };
U32 rankStart0[HUF_ABSOLUTEMAX_TABLELOG + 2] = { 0 };
U32* const rankStart = rankStart0+1;
U32 tableLog, maxW, sizeOfSort, nbSymbols;
rankVal_t rankVal;
const U32 memLog = DTable[0];
size_t iSize;
if (memLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
//memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
iSize = HUF_readStats(weightList, HUF_MAX_SYMBOL_VALUE + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
if (HUF_isError(iSize)) return iSize;
/* check result */
if (tableLog > memLog) return ERROR(tableLog_tooLarge); /* DTable is too small */
/* find maxWeight */
for (maxW = tableLog; rankStats[maxW]==0; maxW--)
{ if (!maxW) return ERROR(GENERIC); } /* necessarily finds a solution before maxW==0 */
/* Get start index of each weight */
{
U32 w, nextRankStart = 0;
for (w=1; w<=maxW; w++)
{
U32 current = nextRankStart;
nextRankStart += rankStats[w];
rankStart[w] = current;
}
rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/
sizeOfSort = nextRankStart;
}
/* sort symbols by weight */
{
U32 s;
for (s=0; s<nbSymbols; s++)
{
U32 w = weightList[s];
U32 r = rankStart[w]++;
sortedSymbol[r].symbol = (BYTE)s;
sortedSymbol[r].weight = (BYTE)w;
}
rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
}
/* Build rankVal */
{
const U32 minBits = tableLog+1 - maxW;
U32 nextRankVal = 0;
U32 w, consumed;
const int rescale = (memLog-tableLog) - 1; /* tableLog <= memLog */
U32* rankVal0 = rankVal[0];
for (w=1; w<=maxW; w++)
{
U32 current = nextRankVal;
nextRankVal += rankStats[w] << (w+rescale);
rankVal0[w] = current;
}
for (consumed = minBits; consumed <= memLog - minBits; consumed++)
{
U32* rankValPtr = rankVal[consumed];
for (w = 1; w <= maxW; w++)
{
rankValPtr[w] = rankVal0[w] >> consumed;
}
}
}
/* fill tables */
{
void* ddPtr = DTable+1;
HUF_DDescX6* DDescription = (HUF_DDescX6*)ddPtr;
void* dsPtr = DTable + 1 + ((size_t)1<<(memLog-1));
HUF_DSeqX6* DSequence = (HUF_DSeqX6*)dsPtr;
HUF_DSeqX6 DSeq;
HUF_DDescX6 DDesc;
DSeq.sequence = 0;
DDesc.nbBits = 0;
DDesc.nbBytes = 0;
HUF_fillDTableX6LevelN(DDescription, DSequence, memLog,
(const U32 (*)[HUF_ABSOLUTEMAX_TABLELOG + 1])rankVal, 0, 1, maxW,
sortedSymbol, sizeOfSort, rankStart0,
tableLog+1, DSeq, DDesc);
}
return iSize;
}
static U32 HUF_decodeSymbolX6(void* op, BIT_DStream_t* DStream, const HUF_DDescX6* dd, const HUF_DSeqX6* ds, const U32 dtLog)
{
const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
memcpy(op, ds+val, sizeof(HUF_DSeqX6));
BIT_skipBits(DStream, dd[val].nbBits);
return dd[val].nbBytes;
}
static U32 HUF_decodeLastSymbolsX6(void* op, const U32 maxL, BIT_DStream_t* DStream,
const HUF_DDescX6* dd, const HUF_DSeqX6* ds, const U32 dtLog)
{
const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
U32 length = dd[val].nbBytes;
if (length <= maxL)
{
memcpy(op, ds+val, length);
BIT_skipBits(DStream, dd[val].nbBits);
return length;
}
memcpy(op, ds+val, maxL);
if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8))
{
BIT_skipBits(DStream, dd[val].nbBits);
if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8))
DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
}
return maxL;
}
#define HUF_DECODE_SYMBOLX6_0(ptr, DStreamPtr) \
ptr += HUF_decodeSymbolX6(ptr, DStreamPtr, dd, ds, dtLog)
#define HUF_DECODE_SYMBOLX6_1(ptr, DStreamPtr) \
if (MEM_64bits() || (HUF_MAX_TABLELOG<=12)) \
HUF_DECODE_SYMBOLX6_0(ptr, DStreamPtr)
#define HUF_DECODE_SYMBOLX6_2(ptr, DStreamPtr) \
if (MEM_64bits()) \
HUF_DECODE_SYMBOLX6_0(ptr, DStreamPtr)
static inline size_t HUF_decodeStreamX6(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const U32* DTable, const U32 dtLog)
{
const void* const ddPtr = DTable+1;
const HUF_DDescX6* dd = (const HUF_DDescX6*)ddPtr;
const void* const dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1));
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)dsPtr;
BYTE* const pStart = p;
/* up to 16 symbols at a time */
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-16))
{
HUF_DECODE_SYMBOLX6_2(p, bitDPtr);
HUF_DECODE_SYMBOLX6_1(p, bitDPtr);
HUF_DECODE_SYMBOLX6_2(p, bitDPtr);
HUF_DECODE_SYMBOLX6_0(p, bitDPtr);
}
/* closer to the end, up to 4 symbols at a time */
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-4))
HUF_DECODE_SYMBOLX6_0(p, bitDPtr);
while ((BIT_reloadDStream(bitDPtr) <= BIT_DStream_endOfBuffer) && (p < pEnd))
p += HUF_decodeLastSymbolsX6(p, (U32)(pEnd-p), bitDPtr, dd, ds, dtLog);
return p-pStart;
}
static size_t HUF_decompress4X6_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const U32* DTable)
{
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
{
const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const U32 dtLog = DTable[0];
const void* const ddPtr = DTable+1;
const HUF_DDescX6* dd = (const HUF_DDescX6*)ddPtr;
const void* const dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1));
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)dsPtr;
size_t errorCode;
/* Init */
BIT_DStream_t bitD1;
BIT_DStream_t bitD2;
BIT_DStream_t bitD3;
BIT_DStream_t bitD4;
const size_t length1 = MEM_readLE16(istart);
const size_t length2 = MEM_readLE16(istart+2);
const size_t length3 = MEM_readLE16(istart+4);
size_t length4;
const BYTE* const istart1 = istart + 6; /* jumpTable */
const BYTE* const istart2 = istart1 + length1;
const BYTE* const istart3 = istart2 + length2;
const BYTE* const istart4 = istart3 + length3;
const size_t segmentSize = (dstSize+3) / 4;
BYTE* const opStart2 = ostart + segmentSize;
BYTE* const opStart3 = opStart2 + segmentSize;
BYTE* const opStart4 = opStart3 + segmentSize;
BYTE* op1 = ostart;
BYTE* op2 = opStart2;
BYTE* op3 = opStart3;
BYTE* op4 = opStart4;
U32 endSignal;
length4 = cSrcSize - (length1 + length2 + length3 + 6);
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
errorCode = BIT_initDStream(&bitD1, istart1, length1);
if (HUF_isError(errorCode)) return errorCode;
errorCode = BIT_initDStream(&bitD2, istart2, length2);
if (HUF_isError(errorCode)) return errorCode;
errorCode = BIT_initDStream(&bitD3, istart3, length3);
if (HUF_isError(errorCode)) return errorCode;
errorCode = BIT_initDStream(&bitD4, istart4, length4);
if (HUF_isError(errorCode)) return errorCode;
/* 16-64 symbols per loop (4-16 symbols per stream) */
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
for ( ; (op3 <= opStart4) && (endSignal==BIT_DStream_unfinished) && (op4<=(oend-16)) ; )
{
HUF_DECODE_SYMBOLX6_2(op1, &bitD1);
HUF_DECODE_SYMBOLX6_2(op2, &bitD2);
HUF_DECODE_SYMBOLX6_2(op3, &bitD3);
HUF_DECODE_SYMBOLX6_2(op4, &bitD4);
HUF_DECODE_SYMBOLX6_1(op1, &bitD1);
HUF_DECODE_SYMBOLX6_1(op2, &bitD2);
HUF_DECODE_SYMBOLX6_1(op3, &bitD3);
HUF_DECODE_SYMBOLX6_1(op4, &bitD4);
HUF_DECODE_SYMBOLX6_2(op1, &bitD1);
HUF_DECODE_SYMBOLX6_2(op2, &bitD2);
HUF_DECODE_SYMBOLX6_2(op3, &bitD3);
HUF_DECODE_SYMBOLX6_2(op4, &bitD4);
HUF_DECODE_SYMBOLX6_0(op1, &bitD1);
HUF_DECODE_SYMBOLX6_0(op2, &bitD2);
HUF_DECODE_SYMBOLX6_0(op3, &bitD3);
HUF_DECODE_SYMBOLX6_0(op4, &bitD4);
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
}
/* check corruption */
if (op1 > opStart2) return ERROR(corruption_detected);
if (op2 > opStart3) return ERROR(corruption_detected);
if (op3 > opStart4) return ERROR(corruption_detected);
/* note : op4 supposed already verified within main loop */
/* finish bitStreams one by one */
HUF_decodeStreamX6(op1, &bitD1, opStart2, DTable, dtLog);
HUF_decodeStreamX6(op2, &bitD2, opStart3, DTable, dtLog);
HUF_decodeStreamX6(op3, &bitD3, opStart4, DTable, dtLog);
HUF_decodeStreamX6(op4, &bitD4, oend, DTable, dtLog);
/* check */
endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
if (!endSignal) return ERROR(corruption_detected);
/* decoded size */
return dstSize;
}
}
static size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
HUF_CREATE_STATIC_DTABLEX6(DTable, HUF_MAX_TABLELOG);
const BYTE* ip = (const BYTE*) cSrc;
size_t hSize = HUF_readDTableX6 (DTable, cSrc, cSrcSize);
if (HUF_isError(hSize)) return hSize;
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
ip += hSize;
cSrcSize -= hSize;
return HUF_decompress4X6_usingDTable (dst, dstSize, ip, cSrcSize, DTable);
}
/**********************************/ /**********************************/
/* Generic decompression selector */ /* Generic decompression selector */
/**********************************/ /**********************************/
@ -3173,7 +2820,7 @@ typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc,
static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{ {
static const decompressionAlgo decompress[3] = { HUF_decompress4X2, HUF_decompress4X4, HUF_decompress4X6 }; static const decompressionAlgo decompress[3] = { HUF_decompress4X2, HUF_decompress4X4, NULL };
/* estimate decompression time */ /* estimate decompression time */
U32 Q; U32 Q;
const U32 D256 = (U32)(dstSize >> 8); const U32 D256 = (U32)(dstSize >> 8);
@ -3195,7 +2842,6 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_
Dtime[1] += Dtime[1] >> 4; Dtime[2] += Dtime[2] >> 3; /* advantage to algorithms using less memory, for cache eviction */ Dtime[1] += Dtime[1] >> 4; Dtime[2] += Dtime[2] >> 3; /* advantage to algorithms using less memory, for cache eviction */
if (Dtime[1] < Dtime[0]) algoNb = 1; if (Dtime[1] < Dtime[0]) algoNb = 1;
if (Dtime[2] < Dtime[algoNb]) algoNb = 2;
return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);

381
lib/legacy/zstd_v05.c
View File

@ -2023,7 +2023,6 @@ extern "C" {
******************************************/ ******************************************/
size_t HUFv05_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ size_t HUFv05_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
size_t HUFv05_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */ size_t HUFv05_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */
size_t HUFv05_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder */
/* **************************************** /* ****************************************
@ -2037,22 +2036,18 @@ HUFv05_decompress() does the following:
*/ */
size_t HUFv05_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize); size_t HUFv05_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize);
size_t HUFv05_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize); size_t HUFv05_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize);
size_t HUFv05_readDTableX6 (unsigned* DTable, const void* src, size_t srcSize);
size_t HUFv05_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable); size_t HUFv05_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable);
size_t HUFv05_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); size_t HUFv05_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
size_t HUFv05_decompress4X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
/* single stream variants */ /* single stream variants */
size_t HUFv05_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ size_t HUFv05_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
size_t HUFv05_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ size_t HUFv05_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
size_t HUFv05_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbol decoder */
size_t HUFv05_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable); size_t HUFv05_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable);
size_t HUFv05_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); size_t HUFv05_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
size_t HUFv05_decompress1X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable);
@ -2850,376 +2845,6 @@ size_t HUFv05_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t
} }
/* ********************************/
/* quad-symbol decoding */
/* ********************************/
typedef struct { BYTE nbBits; BYTE nbBytes; } HUFv05_DDescX6;
typedef union { BYTE byte[4]; U32 sequence; } HUFv05_DSeqX6;
/* recursive, up to level 3; may benefit from <template>-like strategy to nest each level inline */
static void HUFv05_fillDTableX6LevelN(HUFv05_DDescX6* DDescription, HUFv05_DSeqX6* DSequence, int sizeLog,
const rankVal_t rankValOrigin, const U32 consumed, const int minWeight, const U32 maxWeight,
const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, const U32* rankStart,
const U32 nbBitsBaseline, HUFv05_DSeqX6 baseSeq, HUFv05_DDescX6 DDesc)
{
const int scaleLog = nbBitsBaseline - sizeLog; /* note : targetLog >= (nbBitsBaseline-1), hence scaleLog <= 1 */
const int minBits = nbBitsBaseline - maxWeight;
const U32 level = DDesc.nbBytes;
U32 rankVal[HUFv05_ABSOLUTEMAX_TABLELOG + 1];
U32 symbolStartPos, s;
/* local rankVal, will be modified */
memcpy(rankVal, rankValOrigin[consumed], sizeof(rankVal));
/* fill skipped values */
if (minWeight>1) {
U32 i;
const U32 skipSize = rankVal[minWeight];
for (i = 0; i < skipSize; i++) {
DSequence[i] = baseSeq;
DDescription[i] = DDesc;
} }
/* fill DTable */
DDesc.nbBytes++;
symbolStartPos = rankStart[minWeight];
for (s=symbolStartPos; s<sortedListSize; s++) {
const BYTE symbol = sortedSymbols[s].symbol;
const U32 weight = sortedSymbols[s].weight; /* >= 1 (sorted) */
const int nbBits = nbBitsBaseline - weight; /* >= 1 (by construction) */
const int totalBits = consumed+nbBits;
const U32 start = rankVal[weight];
const U32 length = 1 << (sizeLog-nbBits);
baseSeq.byte[level] = symbol;
DDesc.nbBits = (BYTE)totalBits;
if ((level<3) && (sizeLog-totalBits >= minBits)) { /* enough room for another symbol */
int nextMinWeight = totalBits + scaleLog;
if (nextMinWeight < 1) nextMinWeight = 1;
HUFv05_fillDTableX6LevelN(DDescription+start, DSequence+start, sizeLog-nbBits,
rankValOrigin, totalBits, nextMinWeight, maxWeight,
sortedSymbols, sortedListSize, rankStart,
nbBitsBaseline, baseSeq, DDesc); /* recursive (max : level 3) */
} else {
U32 i;
const U32 end = start + length;
for (i = start; i < end; i++) {
DDescription[i] = DDesc;
DSequence[i] = baseSeq;
} }
rankVal[weight] += length;
}
}
/* note : same preparation as X4 */
size_t HUFv05_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
{
BYTE weightList[HUFv05_MAX_SYMBOL_VALUE + 1];
sortedSymbol_t sortedSymbol[HUFv05_MAX_SYMBOL_VALUE + 1];
U32 rankStats[HUFv05_ABSOLUTEMAX_TABLELOG + 1] = { 0 };
U32 rankStart0[HUFv05_ABSOLUTEMAX_TABLELOG + 2] = { 0 };
U32* const rankStart = rankStart0+1;
U32 tableLog, maxW, sizeOfSort, nbSymbols;
rankVal_t rankVal;
const U32 memLog = DTable[0];
size_t iSize;
if (memLog > HUFv05_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
//memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
iSize = HUFv05_readStats(weightList, HUFv05_MAX_SYMBOL_VALUE + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
if (HUFv05_isError(iSize)) return iSize;
/* check result */
if (tableLog > memLog) return ERROR(tableLog_tooLarge); /* DTable is too small */
/* find maxWeight */
for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */
/* Get start index of each weight */
{
U32 w, nextRankStart = 0;
for (w=1; w<=maxW; w++) {
U32 current = nextRankStart;
nextRankStart += rankStats[w];
rankStart[w] = current;
}
rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/
sizeOfSort = nextRankStart;
}
/* sort symbols by weight */
{
U32 s;
for (s=0; s<nbSymbols; s++) {
U32 w = weightList[s];
U32 r = rankStart[w]++;
sortedSymbol[r].symbol = (BYTE)s;
sortedSymbol[r].weight = (BYTE)w;
}
rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
}
/* Build rankVal */
{
const U32 minBits = tableLog+1 - maxW;
U32 nextRankVal = 0;
U32 w, consumed;
const int rescale = (memLog-tableLog) - 1; /* tableLog <= memLog */
U32* rankVal0 = rankVal[0];
for (w=1; w<=maxW; w++) {
U32 current = nextRankVal;
nextRankVal += rankStats[w] << (w+rescale);
rankVal0[w] = current;
}
for (consumed = minBits; consumed <= memLog - minBits; consumed++) {
U32* rankValPtr = rankVal[consumed];
for (w = 1; w <= maxW; w++) {
rankValPtr[w] = rankVal0[w] >> consumed;
} } }
/* fill tables */
{
void* ddPtr = DTable+1;
HUFv05_DDescX6* DDescription = (HUFv05_DDescX6*)ddPtr;
void* dsPtr = DTable + 1 + ((size_t)1<<(memLog-1));
HUFv05_DSeqX6* DSequence = (HUFv05_DSeqX6*)dsPtr;
HUFv05_DSeqX6 DSeq;
HUFv05_DDescX6 DDesc;
DSeq.sequence = 0;
DDesc.nbBits = 0;
DDesc.nbBytes = 0;
HUFv05_fillDTableX6LevelN(DDescription, DSequence, memLog,
(const U32 (*)[HUFv05_ABSOLUTEMAX_TABLELOG + 1])rankVal, 0, 1, maxW,
sortedSymbol, sizeOfSort, rankStart0,
tableLog+1, DSeq, DDesc);
}
return iSize;
}
static U32 HUFv05_decodeSymbolX6(void* op, BITv05_DStream_t* DStream, const HUFv05_DDescX6* dd, const HUFv05_DSeqX6* ds, const U32 dtLog)
{
const size_t val = BITv05_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
memcpy(op, ds+val, sizeof(HUFv05_DSeqX6));
BITv05_skipBits(DStream, dd[val].nbBits);
return dd[val].nbBytes;
}
static U32 HUFv05_decodeLastSymbolsX6(void* op, const U32 maxL, BITv05_DStream_t* DStream,
const HUFv05_DDescX6* dd, const HUFv05_DSeqX6* ds, const U32 dtLog)
{
const size_t val = BITv05_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
U32 length = dd[val].nbBytes;
if (length <= maxL) {
memcpy(op, ds+val, length);
BITv05_skipBits(DStream, dd[val].nbBits);
return length;
}
memcpy(op, ds+val, maxL);
if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) {
BITv05_skipBits(DStream, dd[val].nbBits);
if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8))
DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
}
return maxL;
}
#define HUFv05_DECODE_SYMBOLX6_0(ptr, DStreamPtr) \
ptr += HUFv05_decodeSymbolX6(ptr, DStreamPtr, dd, ds, dtLog)
#define HUFv05_DECODE_SYMBOLX6_1(ptr, DStreamPtr) \
if (MEM_64bits() || (HUFv05_MAX_TABLELOG<=12)) \
HUFv05_DECODE_SYMBOLX6_0(ptr, DStreamPtr)
#define HUFv05_DECODE_SYMBOLX6_2(ptr, DStreamPtr) \
if (MEM_64bits()) \
HUFv05_DECODE_SYMBOLX6_0(ptr, DStreamPtr)
static inline size_t HUFv05_decodeStreamX6(BYTE* p, BITv05_DStream_t* bitDPtr, BYTE* const pEnd, const U32* DTable, const U32 dtLog)
{
const void* const ddPtr = DTable+1;
const HUFv05_DDescX6* dd = (const HUFv05_DDescX6*)ddPtr;
const void* const dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1));
const HUFv05_DSeqX6* ds = (const HUFv05_DSeqX6*)dsPtr;
BYTE* const pStart = p;
/* up to 16 symbols at a time */
while ((BITv05_reloadDStream(bitDPtr) == BITv05_DStream_unfinished) && (p <= pEnd-16)) {
HUFv05_DECODE_SYMBOLX6_2(p, bitDPtr);
HUFv05_DECODE_SYMBOLX6_1(p, bitDPtr);
HUFv05_DECODE_SYMBOLX6_2(p, bitDPtr);
HUFv05_DECODE_SYMBOLX6_0(p, bitDPtr);
}
/* closer to the end, up to 4 symbols at a time */
while ((BITv05_reloadDStream(bitDPtr) == BITv05_DStream_unfinished) && (p <= pEnd-4))
HUFv05_DECODE_SYMBOLX6_0(p, bitDPtr);
while ((BITv05_reloadDStream(bitDPtr) <= BITv05_DStream_endOfBuffer) && (p < pEnd))
p += HUFv05_decodeLastSymbolsX6(p, (U32)(pEnd-p), bitDPtr, dd, ds, dtLog);
return p-pStart;
}
size_t HUFv05_decompress1X6_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const U32* DTable)
{
const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const U32 dtLog = DTable[0];
size_t errorCode;
/* Init */
BITv05_DStream_t bitD;
errorCode = BITv05_initDStream(&bitD, istart, cSrcSize);
if (HUFv05_isError(errorCode)) return errorCode;
/* finish bitStreams one by one */
HUFv05_decodeStreamX6(ostart, &bitD, oend, DTable, dtLog);
/* check */
if (!BITv05_endOfDStream(&bitD)) return ERROR(corruption_detected);
/* decoded size */
return dstSize;
}
size_t HUFv05_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
HUFv05_CREATE_STATIC_DTABLEX6(DTable, HUFv05_MAX_TABLELOG);
const BYTE* ip = (const BYTE*) cSrc;
size_t hSize = HUFv05_readDTableX6 (DTable, cSrc, cSrcSize);
if (HUFv05_isError(hSize)) return hSize;
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
ip += hSize;
cSrcSize -= hSize;
return HUFv05_decompress1X6_usingDTable (dst, dstSize, ip, cSrcSize, DTable);
}
size_t HUFv05_decompress4X6_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const U32* DTable)
{
const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const U32 dtLog = DTable[0];
const void* const ddPtr = DTable+1;
const HUFv05_DDescX6* dd = (const HUFv05_DDescX6*)ddPtr;
const void* const dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1));
const HUFv05_DSeqX6* ds = (const HUFv05_DSeqX6*)dsPtr;
size_t errorCode;
/* Init */
BITv05_DStream_t bitD1;
BITv05_DStream_t bitD2;
BITv05_DStream_t bitD3;
BITv05_DStream_t bitD4;
const size_t length1 = MEM_readLE16(istart);
const size_t length2 = MEM_readLE16(istart+2);
const size_t length3 = MEM_readLE16(istart+4);
size_t length4;
const BYTE* const istart1 = istart + 6; /* jumpTable */
const BYTE* const istart2 = istart1 + length1;
const BYTE* const istart3 = istart2 + length2;
const BYTE* const istart4 = istart3 + length3;
const size_t segmentSize = (dstSize+3) / 4;
BYTE* const opStart2 = ostart + segmentSize;
BYTE* const opStart3 = opStart2 + segmentSize;
BYTE* const opStart4 = opStart3 + segmentSize;
BYTE* op1 = ostart;
BYTE* op2 = opStart2;
BYTE* op3 = opStart3;
BYTE* op4 = opStart4;
U32 endSignal;
/* Check */
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
length4 = cSrcSize - (length1 + length2 + length3 + 6);
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
errorCode = BITv05_initDStream(&bitD1, istart1, length1);
if (HUFv05_isError(errorCode)) return errorCode;
errorCode = BITv05_initDStream(&bitD2, istart2, length2);
if (HUFv05_isError(errorCode)) return errorCode;
errorCode = BITv05_initDStream(&bitD3, istart3, length3);
if (HUFv05_isError(errorCode)) return errorCode;
errorCode = BITv05_initDStream(&bitD4, istart4, length4);
if (HUFv05_isError(errorCode)) return errorCode;
/* 16-64 symbols per loop (4-16 symbols per stream) */
endSignal = BITv05_reloadDStream(&bitD1) | BITv05_reloadDStream(&bitD2) | BITv05_reloadDStream(&bitD3) | BITv05_reloadDStream(&bitD4);
for ( ; (op3 <= opStart4) && (endSignal==BITv05_DStream_unfinished) && (op4<=(oend-16)) ; ) {
HUFv05_DECODE_SYMBOLX6_2(op1, &bitD1);
HUFv05_DECODE_SYMBOLX6_2(op2, &bitD2);
HUFv05_DECODE_SYMBOLX6_2(op3, &bitD3);
HUFv05_DECODE_SYMBOLX6_2(op4, &bitD4);
HUFv05_DECODE_SYMBOLX6_1(op1, &bitD1);
HUFv05_DECODE_SYMBOLX6_1(op2, &bitD2);
HUFv05_DECODE_SYMBOLX6_1(op3, &bitD3);
HUFv05_DECODE_SYMBOLX6_1(op4, &bitD4);
HUFv05_DECODE_SYMBOLX6_2(op1, &bitD1);
HUFv05_DECODE_SYMBOLX6_2(op2, &bitD2);
HUFv05_DECODE_SYMBOLX6_2(op3, &bitD3);
HUFv05_DECODE_SYMBOLX6_2(op4, &bitD4);
HUFv05_DECODE_SYMBOLX6_0(op1, &bitD1);
HUFv05_DECODE_SYMBOLX6_0(op2, &bitD2);
HUFv05_DECODE_SYMBOLX6_0(op3, &bitD3);
HUFv05_DECODE_SYMBOLX6_0(op4, &bitD4);
endSignal = BITv05_reloadDStream(&bitD1) | BITv05_reloadDStream(&bitD2) | BITv05_reloadDStream(&bitD3) | BITv05_reloadDStream(&bitD4);
}
/* check corruption */
if (op1 > opStart2) return ERROR(corruption_detected);
if (op2 > opStart3) return ERROR(corruption_detected);
if (op3 > opStart4) return ERROR(corruption_detected);
/* note : op4 supposed already verified within main loop */
/* finish bitStreams one by one */
HUFv05_decodeStreamX6(op1, &bitD1, opStart2, DTable, dtLog);
HUFv05_decodeStreamX6(op2, &bitD2, opStart3, DTable, dtLog);
HUFv05_decodeStreamX6(op3, &bitD3, opStart4, DTable, dtLog);
HUFv05_decodeStreamX6(op4, &bitD4, oend, DTable, dtLog);
/* check */
endSignal = BITv05_endOfDStream(&bitD1) & BITv05_endOfDStream(&bitD2) & BITv05_endOfDStream(&bitD3) & BITv05_endOfDStream(&bitD4);
if (!endSignal) return ERROR(corruption_detected);
/* decoded size */
return dstSize;
}
size_t HUFv05_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
HUFv05_CREATE_STATIC_DTABLEX6(DTable, HUFv05_MAX_TABLELOG);
const BYTE* ip = (const BYTE*) cSrc;
size_t hSize = HUFv05_readDTableX6 (DTable, cSrc, cSrcSize);
if (HUFv05_isError(hSize)) return hSize;
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
ip += hSize;
cSrcSize -= hSize;
return HUFv05_decompress4X6_usingDTable (dst, dstSize, ip, cSrcSize, DTable);
}
/* ********************************/ /* ********************************/
/* Generic decompression selector */ /* Generic decompression selector */
/* ********************************/ /* ********************************/
@ -3250,7 +2875,7 @@ typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc,
size_t HUFv05_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) size_t HUFv05_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{ {
static const decompressionAlgo decompress[3] = { HUFv05_decompress4X2, HUFv05_decompress4X4, HUFv05_decompress4X6 }; static const decompressionAlgo decompress[3] = { HUFv05_decompress4X2, HUFv05_decompress4X4, NULL };
/* estimate decompression time */ /* estimate decompression time */
U32 Q; U32 Q;
const U32 D256 = (U32)(dstSize >> 8); const U32 D256 = (U32)(dstSize >> 8);
@ -3272,7 +2897,6 @@ size_t HUFv05_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS
Dtime[1] += Dtime[1] >> 4; Dtime[2] += Dtime[2] >> 3; /* advantage to algorithms using less memory, for cache eviction */ Dtime[1] += Dtime[1] >> 4; Dtime[2] += Dtime[2] >> 3; /* advantage to algorithms using less memory, for cache eviction */
if (Dtime[1] < Dtime[0]) algoNb = 1; if (Dtime[1] < Dtime[0]) algoNb = 1;
if (Dtime[2] < Dtime[algoNb]) algoNb = 2;
return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
@ -4706,8 +4330,7 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst
*srcSizePtr = ip-istart; *srcSizePtr = ip-istart;
*maxDstSizePtr = op-ostart; *maxDstSizePtr = op-ostart;
{ { size_t nextSrcSizeHint = ZSTDv05_nextSrcSizeToDecompress(zbc->zc);
size_t nextSrcSizeHint = ZSTDv05_nextSrcSizeToDecompress(zbc->zc);
if (nextSrcSizeHint > ZBUFFv05_blockHeaderSize) nextSrcSizeHint+= ZBUFFv05_blockHeaderSize; /* get next block header too */ if (nextSrcSizeHint > ZBUFFv05_blockHeaderSize) nextSrcSizeHint+= ZBUFFv05_blockHeaderSize; /* get next block header too */
nextSrcSizeHint -= zbc->inPos; /* already loaded*/ nextSrcSizeHint -= zbc->inPos; /* already loaded*/
return nextSrcSizeHint; return nextSrcSizeHint;

27
lib/legacy/zstd_v05.h
View File

@ -69,7 +69,7 @@ const char* ZSTDv05_getErrorName(size_t code); /*!< provides readable string
/** Decompression context */ /** Decompression context */
typedef struct ZSTDv05_DCtx_s ZSTDv05_DCtx; typedef struct ZSTDv05_DCtx_s ZSTDv05_DCtx;
ZSTDv05_DCtx* ZSTDv05_createDCtx(void); ZSTDv05_DCtx* ZSTDv05_createDCtx(void);
size_t ZSTDv05_freeDCtx(ZSTDv05_DCtx* dctx); /*!< @return : errorCode */ size_t ZSTDv05_freeDCtx(ZSTDv05_DCtx* dctx); /*!< @return : errorCode */
/** ZSTDv05_decompressDCtx() : /** ZSTDv05_decompressDCtx() :
* Same as ZSTDv05_decompress(), but requires an already allocated ZSTDv05_DCtx (see ZSTDv05_createDCtx()) */ * Same as ZSTDv05_decompress(), but requires an already allocated ZSTDv05_DCtx (see ZSTDv05_createDCtx()) */
@ -77,25 +77,32 @@ size_t ZSTDv05_decompressDCtx(ZSTDv05_DCtx* ctx, void* dst, size_t dstCapacity,
/*-*********************** /*-***********************
* Dictionary API * Simple Dictionary API
*************************/ *************************/
/*! ZSTDv05_decompress_usingDict() : /*! ZSTDv05_decompress_usingDict() :
* Decompression using a pre-defined Dictionary content (see dictBuilder). * Decompression using a pre-defined Dictionary content (see dictBuilder).
* Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted. * Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted.
* Note : dict can be NULL, in which case, it's equivalent to ZSTDv05_decompressDCtx() */ * Note : dict can be NULL, in which case, it's equivalent to ZSTDv05_decompressDCtx() */
size_t ZSTDv05_decompress_usingDict(ZSTDv05_DCtx* dctx, size_t ZSTDv05_decompress_usingDict(ZSTDv05_DCtx* dctx,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
const void* src, size_t srcSize, const void* src, size_t srcSize,
const void* dict,size_t dictSize); const void* dict,size_t dictSize);
/*-************************
* Advanced Streaming API
***************************/
size_t ZSTDv05_decompressBegin_usingDict(ZSTDv05_DCtx* dctx, const void* dict, size_t dictSize);
void ZSTDv05_copyDCtx(ZSTDv05_DCtx* dstDCtx, const ZSTDv05_DCtx* srcDCtx);
size_t ZSTDv05_nextSrcSizeToDecompress(ZSTDv05_DCtx* dctx);
size_t ZSTDv05_decompressContinue(ZSTDv05_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/*-***********************
* ZBUFF API
*************************/
typedef struct ZBUFFv05_DCtx_s ZBUFFv05_DCtx; typedef struct ZBUFFv05_DCtx_s ZBUFFv05_DCtx;
ZBUFFv05_DCtx* ZBUFFv05_createDCtx(void); ZBUFFv05_DCtx* ZBUFFv05_createDCtx(void);
size_t ZBUFFv05_freeDCtx(ZBUFFv05_DCtx* dctx); size_t ZBUFFv05_freeDCtx(ZBUFFv05_DCtx* dctx);
size_t ZBUFFv05_decompressInit(ZBUFFv05_DCtx* dctx); size_t ZBUFFv05_decompressInit(ZBUFFv05_DCtx* dctx);
size_t ZBUFFv05_decompressInitDictionary(ZBUFFv05_DCtx* dctx, const void* dict, size_t dictSize); size_t ZBUFFv05_decompressInitDictionary(ZBUFFv05_DCtx* dctx, const void* dict, size_t dictSize);

2
programs/bench.c
View File

@ -423,7 +423,7 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
totalSize += (size_t)fileSize; totalSize += (size_t)fileSize;
fclose(f); fclose(f);
} }
if (totalSize == 0) EXM_THROW(12, "no data to bench"); if (totalSize == 0) EXM_THROW(12, "no data to bench");
} }

40
programs/datagen.c
View File

@ -29,25 +29,7 @@
#include <stdlib.h> /* malloc */ #include <stdlib.h> /* malloc */
#include <stdio.h> /* FILE, fwrite */ #include <stdio.h> /* FILE, fwrite */
#include <string.h> /* memcpy */ #include <string.h> /* memcpy */
#include "mem.h"
/*-************************************
* Basic Types
**************************************/
#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
#endif
/*-************************************ /*-************************************
@ -63,7 +45,7 @@
/*-************************************ /*-************************************
* Constants * Macros
**************************************/ **************************************/
#define KB *(1 <<10) #define KB *(1 <<10)
@ -98,15 +80,10 @@ static unsigned int RDG_rand(U32* src)
static void RDG_fillLiteralDistrib(litDistribTable lt, double ld) static void RDG_fillLiteralDistrib(litDistribTable lt, double ld)
{ {
U32 i = 0; U32 i = 0;
BYTE character = '0'; BYTE character = (ld<=0.0) ? 0 : '0';
BYTE firstChar = '('; BYTE const firstChar = (ld<=0.0) ? 0 : '(';
BYTE lastChar = '}'; BYTE const lastChar = (ld<=0.0) ?255: '}';
if (ld<=0.0) {
character = 0;
firstChar = 0;
lastChar =255;
}
while (i<LTSIZE) { while (i<LTSIZE) {
U32 weight = (U32)((double)(LTSIZE - i) * ld) + 1; U32 weight = (U32)((double)(LTSIZE - i) * ld) + 1;
U32 end; U32 end;
@ -161,7 +138,7 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match
/* Copy (within 32K) */ /* Copy (within 32K) */
size_t match; size_t match;
size_t d; size_t d;
int length = RDG_RANDLENGTH + 4; size_t const length = RDG_RANDLENGTH + 4;
U32 offset = RDG_RAND15BITS + 1; U32 offset = RDG_RAND15BITS + 1;
U32 repeatOffset = (RDG_rand(seed) & 15) == 2; U32 repeatOffset = (RDG_rand(seed) & 15) == 2;
if (repeatOffset) offset = prevOffset; if (repeatOffset) offset = prevOffset;
@ -173,9 +150,8 @@ void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double match
while (pos < d) buffPtr[pos++] = buffPtr[match++]; /* correctly manages overlaps */ while (pos < d) buffPtr[pos++] = buffPtr[match++]; /* correctly manages overlaps */
} else { } else {
/* Literal (noise) */ /* Literal (noise) */
size_t d; size_t const length = RDG_RANDLENGTH;
size_t length = RDG_RANDLENGTH; size_t d = pos + length;
d = pos + length;
if (d > buffSize) d = buffSize; if (d > buffSize) d = buffSize;
while (pos < d) buffPtr[pos++] = RDG_genChar(seed, lt); while (pos < d) buffPtr[pos++] = RDG_genChar(seed, lt);
} } } }

28
programs/dibio.c
View File

@ -58,6 +58,19 @@ static const size_t maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static unsigned g_displayLevel = 0; /* 0 : no display; 1: errors; 2: default; 4: full information */ static unsigned g_displayLevel = 0; /* 0 : no display; 1: errors; 2: default; 4: full information */
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
if ((DIB_GetMilliSpan(g_time) > refreshRate) || (g_displayLevel>=4)) \
{ g_time = clock(); DISPLAY(__VA_ARGS__); \
if (g_displayLevel>=4) fflush(stdout); } }
static const unsigned refreshRate = 150;
static clock_t g_time = 0;
static unsigned DIB_GetMilliSpan(clock_t nPrevious)
{
clock_t const nCurrent = clock();
return (unsigned)(((nCurrent - nPrevious) * 1000) / CLOCKS_PER_SEC);
}
/*-************************************* /*-*************************************
* Exceptions * Exceptions
@ -100,7 +113,7 @@ static void DiB_loadFiles(void* buffer, size_t bufferSize,
unsigned long long fileSize = UTIL_getFileSize(fileNamesTable[n]); unsigned long long fileSize = UTIL_getFileSize(fileNamesTable[n]);
FILE* f = fopen(fileNamesTable[n], "rb"); FILE* f = fopen(fileNamesTable[n], "rb");
if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]); if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]); DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
if (fileSize > bufferSize-pos) fileSize = 0; /* stop there, not enough memory to load all files */ if (fileSize > bufferSize-pos) fileSize = 0; /* stop there, not enough memory to load all files */
readSize = fread(buff+pos, 1, (size_t)fileSize, f); readSize = fread(buff+pos, 1, (size_t)fileSize, f);
if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]); if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
@ -148,17 +161,14 @@ static void DiB_fillNoise(void* buffer, size_t length)
static void DiB_saveDict(const char* dictFileName, static void DiB_saveDict(const char* dictFileName,
const void* buff, size_t buffSize) const void* buff, size_t buffSize)
{ {
FILE* f; FILE* f = fopen(dictFileName, "wb");
size_t n;
f = fopen(dictFileName, "wb");
if (f==NULL) EXM_THROW(3, "cannot open %s ", dictFileName); if (f==NULL) EXM_THROW(3, "cannot open %s ", dictFileName);
n = fwrite(buff, 1, buffSize, f); { size_t const n = fwrite(buff, 1, buffSize, f);
if (n!=buffSize) EXM_THROW(4, "%s : write error", dictFileName) if (n!=buffSize) EXM_THROW(4, "%s : write error", dictFileName) }
n = (size_t)fclose(f); { size_t const n = (size_t)fclose(f);
if (n!=0) EXM_THROW(5, "%s : flush error", dictFileName) if (n!=0) EXM_THROW(5, "%s : flush error", dictFileName) }
} }

131
programs/fileio.c
View File

@ -116,6 +116,11 @@ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
static const unsigned refreshRate = 150; static const unsigned refreshRate = 150;
static clock_t g_time = 0; static clock_t g_time = 0;
static unsigned FIO_GetMilliSpan(clock_t nPrevious)
{
clock_t const nCurrent = clock();
return (unsigned)(((nCurrent - nPrevious) * 1000) / CLOCKS_PER_SEC);
}
/*-************************************* /*-*************************************
@ -147,14 +152,6 @@ void FIO_setMaxWLog(unsigned maxWLog) { g_maxWLog = maxWLog; }
/*-************************************* /*-*************************************
* Functions * Functions
***************************************/ ***************************************/
static unsigned FIO_GetMilliSpan(clock_t nPrevious)
{
clock_t nCurrent = clock();
unsigned nSpan = (unsigned)(((nCurrent - nPrevious) * 1000) / CLOCKS_PER_SEC);
return nSpan;
}
static FILE* FIO_openSrcFile(const char* srcFileName) static FILE* FIO_openSrcFile(const char* srcFileName)
{ {
FILE* f; FILE* f;
@ -297,21 +294,20 @@ static int FIO_compressFilename_internal(cRess_t ress,
const char* dstFileName, const char* srcFileName, const char* dstFileName, const char* srcFileName,
int cLevel) int cLevel)
{ {
FILE* srcFile = ress.srcFile; FILE* const srcFile = ress.srcFile;
FILE* dstFile = ress.dstFile; FILE* const dstFile = ress.dstFile;
U64 readsize = 0; U64 readsize = 0;
U64 compressedfilesize = 0; U64 compressedfilesize = 0;
size_t dictSize = ress.dictBufferSize;
size_t sizeCheck;
ZSTD_parameters params;
U64 const fileSize = UTIL_getFileSize(srcFileName); U64 const fileSize = UTIL_getFileSize(srcFileName);
/* init */ /* init */
params.cParams = ZSTD_getCParams(cLevel, fileSize, dictSize); { ZSTD_parameters params;
params.fParams.contentSizeFlag = 1; params.cParams = ZSTD_getCParams(cLevel, fileSize, ress.dictBufferSize);
if (g_maxWLog) if (params.cParams.windowLog > g_maxWLog) params.cParams.windowLog = g_maxWLog; params.fParams.contentSizeFlag = 1;
{ size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize); if (g_maxWLog) if (params.cParams.windowLog > g_maxWLog) params.cParams.windowLog = g_maxWLog;
if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode)); } { size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode)); }
}
/* Main compression loop */ /* Main compression loop */
readsize = 0; readsize = 0;
@ -325,16 +321,15 @@ static int FIO_compressFilename_internal(cRess_t ress,
{ /* Compress using buffered streaming */ { /* Compress using buffered streaming */
size_t usedInSize = inSize; size_t usedInSize = inSize;
size_t cSize = ress.dstBufferSize; size_t cSize = ress.dstBufferSize;
size_t result = ZBUFF_compressContinue(ress.ctx, ress.dstBuffer, &cSize, ress.srcBuffer, &usedInSize); { size_t const result = ZBUFF_compressContinue(ress.ctx, ress.dstBuffer, &cSize, ress.srcBuffer, &usedInSize);
if (ZBUFF_isError(result)) if (ZBUFF_isError(result)) EXM_THROW(23, "Compression error : %s ", ZBUFF_getErrorName(result)); }
EXM_THROW(23, "Compression error : %s ", ZBUFF_getErrorName(result));
if (inSize != usedInSize) if (inSize != usedInSize)
/* inBuff should be entirely consumed since buffer sizes are recommended ones */ /* inBuff should be entirely consumed since buffer sizes are recommended ones */
EXM_THROW(24, "Compression error : input block not fully consumed"); EXM_THROW(24, "Compression error : input block not fully consumed");
/* Write cBlock */ /* Write cBlock */
sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile); { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile);
if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName); }
compressedfilesize += cSize; compressedfilesize += cSize;
} }
DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(readsize>>20), (double)compressedfilesize/readsize*100); DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", (U32)(readsize>>20), (double)compressedfilesize/readsize*100);
@ -345,8 +340,8 @@ static int FIO_compressFilename_internal(cRess_t ress,
size_t const result = ZBUFF_compressEnd(ress.ctx, ress.dstBuffer, &cSize); size_t const result = ZBUFF_compressEnd(ress.ctx, ress.dstBuffer, &cSize);
if (result!=0) EXM_THROW(26, "Compression error : cannot create frame end"); if (result!=0) EXM_THROW(26, "Compression error : cannot create frame end");
sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile); { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile);
if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); }
compressedfilesize += cSize; compressedfilesize += cSize;
} }
@ -376,7 +371,6 @@ static int FIO_compressFilename_srcFile(cRess_t ress,
result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel); result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel);
/* clean */
fclose(ress.srcFile); fclose(ress.srcFile);
return result; return result;
} }
@ -409,7 +403,7 @@ static int FIO_compressFilename_extRess(cRess_t ress,
int FIO_compressFilename(const char* dstFileName, const char* srcFileName, int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
const char* dictFileName, int compressionLevel) const char* dictFileName, int compressionLevel)
{ {
clock_t start, end; clock_t start;
cRess_t ress; cRess_t ress;
int issueWithSrcFile = 0; int issueWithSrcFile = 0;
@ -417,19 +411,13 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
start = clock(); start = clock();
ress = FIO_createCResources(dictFileName); ress = FIO_createCResources(dictFileName);
/* Compress File */
issueWithSrcFile += FIO_compressFilename_extRess(ress, dstFileName, srcFileName, compressionLevel); issueWithSrcFile += FIO_compressFilename_extRess(ress, dstFileName, srcFileName, compressionLevel);
/* Free resources */
FIO_freeCResources(ress); FIO_freeCResources(ress);
/* Final Status */ { double seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
end = clock();
{
double seconds = (double)(end - start) / CLOCKS_PER_SEC;
DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds); DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds);
} }
return issueWithSrcFile; return issueWithSrcFile;
} }
@ -438,11 +426,10 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
const char* suffix, const char* suffix,
const char* dictFileName, int compressionLevel) const char* dictFileName, int compressionLevel)
{ {
unsigned u;
int missed_files = 0; int missed_files = 0;
char* dstFileName = (char*)malloc(FNSPACE); char* dstFileName = (char*)malloc(FNSPACE);
size_t dfnSize = FNSPACE; size_t dfnSize = FNSPACE;
const size_t suffixSize = suffix ? strlen(suffix) : 0; size_t const suffixSize = suffix ? strlen(suffix) : 0;
cRess_t ress; cRess_t ress;
/* init */ /* init */
@ -450,12 +437,14 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
/* loop on each file */ /* loop on each file */
if (!strcmp(suffix, stdoutmark)) { if (!strcmp(suffix, stdoutmark)) {
unsigned u;
ress.dstFile = stdout; ress.dstFile = stdout;
for (u=0; u<nbFiles; u++) for (u=0; u<nbFiles; u++)
missed_files += FIO_compressFilename_srcFile(ress, stdoutmark, missed_files += FIO_compressFilename_srcFile(ress, stdoutmark,
inFileNamesTable[u], compressionLevel); inFileNamesTable[u], compressionLevel);
if (fclose(ress.dstFile)) EXM_THROW(29, "Write error : cannot properly close %s", stdoutmark); if (fclose(ress.dstFile)) EXM_THROW(29, "Write error : cannot properly close %s", stdoutmark);
} else { } else {
unsigned u;
for (u=0; u<nbFiles; u++) { for (u=0; u<nbFiles; u++) {
size_t ifnSize = strlen(inFileNamesTable[u]); size_t ifnSize = strlen(inFileNamesTable[u]);
if (dfnSize <= ifnSize+suffixSize+1) { free(dstFileName); dfnSize = ifnSize + 20; dstFileName = (char*)malloc(dfnSize); } if (dfnSize <= ifnSize+suffixSize+1) { free(dstFileName); dfnSize = ifnSize + 20; dstFileName = (char*)malloc(dfnSize); }
@ -472,7 +461,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
return missed_files; return missed_files;
} }
#endif // #ifndef ZSTD_NOCOMPRESS #endif /* #ifndef ZSTD_NOCOMPRESS */
@ -515,7 +504,7 @@ static dRess_t FIO_createDResources(const char* dictFileName)
static void FIO_freeDResources(dRess_t ress) static void FIO_freeDResources(dRess_t ress)
{ {
size_t errorCode = ZBUFF_freeDCtx(ress.dctx); size_t const errorCode = ZBUFF_freeDCtx(ress.dctx);
if (ZBUFF_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFF_getErrorName(errorCode)); if (ZBUFF_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFF_getErrorName(errorCode));
free(ress.srcBuffer); free(ress.srcBuffer);
free(ress.dstBuffer); free(ress.dstBuffer);
@ -536,16 +525,15 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
/* Complete Header loading */ /* Complete Header loading */
{ size_t const toLoad = ZSTD_frameHeaderSize_max - alreadyLoaded; /* assumption : alreadyLoaded <= ZSTD_frameHeaderSize_max */ { size_t const toLoad = ZSTD_frameHeaderSize_max - alreadyLoaded; /* assumption : alreadyLoaded <= ZSTD_frameHeaderSize_max */
size_t const checkSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput); size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput); /* can be <= toLoad (null string) */
if (checkSize != toLoad) EXM_THROW(32, "Read error"); /* assumption : srcSize >= ZSTD_frameHeaderSize_max */ readSize = alreadyLoaded + loadedSize;
} }
readSize = ZSTD_frameHeaderSize_max;
/* Main decompression Loop */ /* Main decompression Loop */
while (1) { while (1) {
/* Decode */ /* Decode */
size_t inSize=readSize, decodedSize=ress.dstBufferSize; size_t inSize=readSize, decodedSize=ress.dstBufferSize;
size_t toRead = ZBUFF_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize); size_t const toRead = ZBUFF_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead)); if (ZBUFF_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFF_getErrorName(toRead));
readSize -= inSize; readSize -= inSize;
@ -577,8 +565,8 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
{ {
unsigned long long filesize = 0; unsigned long long filesize = 0;
FILE* dstFile = ress.dstFile; FILE* const dstFile = ress.dstFile;
FILE* srcFile = FIO_openSrcFile(srcFileName); FILE* const srcFile = FIO_openSrcFile(srcFileName);
if (srcFile==0) return 1; if (srcFile==0) return 1;
/* for each frame */ /* for each frame */
@ -588,16 +576,17 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
size_t const sizeCheck = fread(ress.srcBuffer, (size_t)1, toRead, srcFile); size_t const sizeCheck = fread(ress.srcBuffer, (size_t)1, toRead, srcFile);
if (sizeCheck==0) break; /* no more input */ if (sizeCheck==0) break; /* no more input */
if (sizeCheck != toRead) EXM_THROW(31, "zstd: %s read error : cannot read header", srcFileName); if (sizeCheck != toRead) EXM_THROW(31, "zstd: %s read error : cannot read header", srcFileName);
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) { U32 const magic = MEM_readLE32(ress.srcBuffer);
if (ZSTD_isLegacy(MEM_readLE32(ress.srcBuffer))) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
filesize += FIO_decompressLegacyFrame(dstFile, srcFile, MEM_readLE32(ress.srcBuffer)); if (ZSTD_isLegacy(magic)) {
continue; filesize += FIO_decompressLegacyFrame(dstFile, srcFile, ress.dictBuffer, ress.dictBufferSize, magic);
} continue;
#endif /* ZSTD_LEGACY_SUPPORT */ }
if (MEM_readLE32(ress.srcBuffer) != ZSTD_MAGICNUMBER) { #endif
DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName); if (magic != ZSTD_MAGICNUMBER) {
return 1; DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName);
} return 1;
} }
filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead); filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead);
} }
@ -624,9 +613,7 @@ static int FIO_decompressFile_extRess(dRess_t ress,
if (ress.dstFile==0) return 1; if (ress.dstFile==0) return 1;
result = FIO_decompressSrcFile(ress, srcFileName); result = FIO_decompressSrcFile(ress, srcFileName);
if (result != 0) { if (result != 0) remove(dstFileName);
remove(dstFileName);
}
if (fclose(ress.dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName); if (fclose(ress.dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName);
return result; return result;
@ -651,28 +638,27 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
const char* suffix, const char* suffix,
const char* dictFileName) const char* dictFileName)
{ {
unsigned u;
int skippedFiles = 0; int skippedFiles = 0;
int missingFiles = 0; int missingFiles = 0;
char* dstFileName = (char*)malloc(FNSPACE); dRess_t ress = FIO_createDResources(dictFileName);
size_t dfnSize = FNSPACE;
const size_t suffixSize = suffix ? strlen(suffix) : 0;
dRess_t ress;
if (dstFileName==NULL) EXM_THROW(70, "not enough memory for dstFileName");
ress = FIO_createDResources(dictFileName);
if (!strcmp(suffix, stdoutmark) || !strcmp(suffix, nulmark)) { if (!strcmp(suffix, stdoutmark) || !strcmp(suffix, nulmark)) {
unsigned u;
ress.dstFile = FIO_openDstFile(suffix); ress.dstFile = FIO_openDstFile(suffix);
if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", suffix); if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", suffix);
for (u=0; u<nbFiles; u++) for (u=0; u<nbFiles; u++)
missingFiles += FIO_decompressSrcFile(ress, srcNamesTable[u]); missingFiles += FIO_decompressSrcFile(ress, srcNamesTable[u]);
if (fclose(ress.dstFile)) EXM_THROW(39, "Write error : cannot properly close %s", stdoutmark); if (fclose(ress.dstFile)) EXM_THROW(39, "Write error : cannot properly close %s", stdoutmark);
} else { } else {
size_t const suffixSize = suffix ? strlen(suffix) : 0;
size_t dfnSize = FNSPACE;
unsigned u;
char* dstFileName = (char*)malloc(FNSPACE);
if (dstFileName==NULL) EXM_THROW(70, "not enough memory for dstFileName");
for (u=0; u<nbFiles; u++) { /* create dstFileName */ for (u=0; u<nbFiles; u++) { /* create dstFileName */
const char* srcFileName = srcNamesTable[u]; const char* const srcFileName = srcNamesTable[u];
size_t sfnSize = strlen(srcFileName); size_t const sfnSize = strlen(srcFileName);
const char* suffixPtr = srcFileName + sfnSize - suffixSize; const char* const suffixPtr = srcFileName + sfnSize - suffixSize;
if (dfnSize+suffixSize <= sfnSize+1) { if (dfnSize+suffixSize <= sfnSize+1) {
free(dstFileName); free(dstFileName);
dfnSize = sfnSize + 20; dfnSize = sfnSize + 20;
@ -688,11 +674,12 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
dstFileName[sfnSize-suffixSize] = '\0'; dstFileName[sfnSize-suffixSize] = '\0';
missingFiles += FIO_decompressFile_extRess(ress, dstFileName, srcFileName); missingFiles += FIO_decompressFile_extRess(ress, dstFileName, srcFileName);
} } }
free(dstFileName);
}
FIO_freeDResources(ress); FIO_freeDResources(ress);
free(dstFileName);
return missingFiles + skippedFiles; return missingFiles + skippedFiles;
} }
#endif // #ifndef ZSTD_NODECOMPRESS #endif /* #ifndef ZSTD_NODECOMPRESS */

36
programs/legacy/fileio_legacy.c
View File

@ -38,8 +38,6 @@
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#endif #endif
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
#define _FILE_OFFSET_BITS 64 /* Large file support on 32-bits unix */ #define _FILE_OFFSET_BITS 64 /* Large file support on 32-bits unix */
#define _POSIX_SOURCE 1 /* enable fileno() within <stdio.h> on unix */ #define _POSIX_SOURCE 1 /* enable fileno() within <stdio.h> on unix */
@ -92,8 +90,8 @@
#define BIT6 0x40 #define BIT6 0x40
#define BIT7 0x80 #define BIT7 0x80
#define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */ #define FIO_FRAMEHEADERSIZE 5 /* as a define, because needed to allocated table on stack */
#define FSE_CHECKSUM_SEED 0 #define FSE_CHECKSUM_SEED 0
#define CACHELINE 64 #define CACHELINE 64
@ -149,7 +147,7 @@ static unsigned FIO_GetMilliSpan(clock_t nPrevious)
unsigned long long FIOv01_decompressFrame(FILE* foutput, FILE* finput) unsigned long long FIOv01_decompressFrame(FILE* foutput, FILE* finput)
{ {
size_t outBuffSize = 512 KB; size_t const outBuffSize = 512 KB;
BYTE* outBuff = (BYTE*)malloc(outBuffSize); BYTE* outBuff = (BYTE*)malloc(outBuffSize);
size_t inBuffSize = 128 KB + 8; size_t inBuffSize = 128 KB + 8;
BYTE inBuff[128 KB + 8]; BYTE inBuff[128 KB + 8];
@ -208,7 +206,7 @@ unsigned long long FIOv01_decompressFrame(FILE* foutput, FILE* finput)
unsigned long long FIOv02_decompressFrame(FILE* foutput, FILE* finput) unsigned long long FIOv02_decompressFrame(FILE* foutput, FILE* finput)
{ {
size_t outBuffSize = 512 KB; size_t const outBuffSize = 512 KB;
BYTE* outBuff = (BYTE*)malloc(outBuffSize); BYTE* outBuff = (BYTE*)malloc(outBuffSize);
size_t inBuffSize = 128 KB + 8; size_t inBuffSize = 128 KB + 8;
BYTE inBuff[128 KB + 8]; BYTE inBuff[128 KB + 8];
@ -266,7 +264,7 @@ unsigned long long FIOv02_decompressFrame(FILE* foutput, FILE* finput)
unsigned long long FIOv03_decompressFrame(FILE* foutput, FILE* finput) unsigned long long FIOv03_decompressFrame(FILE* foutput, FILE* finput)
{ {
size_t outBuffSize = 512 KB; size_t const outBuffSize = 512 KB;
BYTE* outBuff = (BYTE*)malloc(outBuffSize); BYTE* outBuff = (BYTE*)malloc(outBuffSize);
size_t inBuffSize = 128 KB + 8; size_t inBuffSize = 128 KB + 8;
BYTE inBuff[128 KB + 8]; BYTE inBuff[128 KB + 8];
@ -407,7 +405,7 @@ typedef struct {
size_t srcBufferSize; size_t srcBufferSize;
void* dstBuffer; void* dstBuffer;
size_t dstBufferSize; size_t dstBufferSize;
void* dictBuffer; const void* dictBuffer;
size_t dictBufferSize; size_t dictBufferSize;
ZBUFFv05_DCtx* dctx; ZBUFFv05_DCtx* dctx;
} dRessv05_t; } dRessv05_t;
@ -437,7 +435,6 @@ static void FIOv05_freeDResources(dRessv05_t ress)
if (ZBUFFv05_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFFv05_getErrorName(errorCode)); if (ZBUFFv05_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFFv05_getErrorName(errorCode));
free(ress.srcBuffer); free(ress.srcBuffer);
free(ress.dstBuffer); free(ress.dstBuffer);
free(ress.dictBuffer);
} }
@ -445,22 +442,21 @@ unsigned long long FIOv05_decompressFrame(dRessv05_t ress,
FILE* foutput, FILE* finput) FILE* foutput, FILE* finput)
{ {
U64 frameSize = 0; U64 frameSize = 0;
size_t readSize = 4; size_t readSize = 4;
MEM_writeLE32(ress.srcBuffer, ZSTDv05_MAGICNUMBER); MEM_writeLE32(ress.srcBuffer, ZSTDv05_MAGICNUMBER);
ZBUFFv05_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize); ZBUFFv05_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
while (1) { while (1) {
/* Decode */ /* Decode */
size_t sizeCheck;
size_t inSize=readSize, decodedSize=ress.dstBufferSize; size_t inSize=readSize, decodedSize=ress.dstBufferSize;
size_t toRead = ZBUFFv05_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize); size_t toRead = ZBUFFv05_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
if (ZBUFFv05_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFFv05_getErrorName(toRead)); if (ZBUFFv05_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFFv05_getErrorName(toRead));
readSize -= inSize; readSize -= inSize;
/* Write block */ /* Write block */
sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput); { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file"); if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file"); }
frameSize += decodedSize; frameSize += decodedSize;
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) ); DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(frameSize>>20) );
@ -479,7 +475,9 @@ unsigned long long FIOv05_decompressFrame(dRessv05_t ress,
/*===== General legacy dispatcher =====*/ /*===== General legacy dispatcher =====*/
unsigned long long FIO_decompressLegacyFrame(FILE* foutput, FILE* finput, U32 magicNumberLE) unsigned long long FIO_decompressLegacyFrame(FILE* foutput, FILE* finput,
const void* dictBuffer, size_t dictSize,
U32 magicNumberLE)
{ {
switch(magicNumberLE) switch(magicNumberLE)
{ {
@ -497,10 +495,12 @@ unsigned long long FIO_decompressLegacyFrame(FILE* foutput, FILE* finput, U32 ma
} }
case ZSTDv05_MAGICNUMBER : case ZSTDv05_MAGICNUMBER :
{ dRessv05_t r = FIOv05_createDResources(); { dRessv05_t r = FIOv05_createDResources();
unsigned long long const s = FIOv05_decompressFrame(r, foutput, finput); r.dictBuffer = dictBuffer;
FIOv05_freeDResources(r); r.dictBufferSize = dictSize;
return s; { unsigned long long const s = FIOv05_decompressFrame(r, foutput, finput);
} FIOv05_freeDResources(r);
return s;
} }
default : default :
return ERROR(prefix_unknown); return ERROR(prefix_unknown);
} }

4
programs/legacy/fileio_legacy.h
View File

@ -38,7 +38,9 @@ void FIO_legacy_setNotificationLevel(unsigned level);
/* ************************************* /* *************************************
* Stream/File functions * Stream/File functions
***************************************/ ***************************************/
unsigned long long FIO_decompressLegacyFrame(FILE* foutput, FILE* finput, U32 magicNumberLE); unsigned long long FIO_decompressLegacyFrame(FILE* foutput, FILE* finput,
const void* dictBuffer, size_t dictSize,
U32 magicNumberLE);
#if defined (__cplusplus) #if defined (__cplusplus)

7
programs/playTests.sh
View File

@ -29,8 +29,11 @@ $ZSTD -f tmp # trivial compression case, creates tmp
$ZSTD -df tmp.zst # trivial decompression case (overwrites tmp) $ZSTD -df tmp.zst # trivial decompression case (overwrites tmp)
echo "test : too large compression level (must fail)" echo "test : too large compression level (must fail)"
$ZSTD -99 tmp && die "too large compression level undetected" $ZSTD -99 tmp && die "too large compression level undetected"
$ZSTD tmp -c > tmpCompressed # compression using stdout echo "test : compress to stdout"
$ZSTD tmp --stdout > tmpCompressed # compressoin using stdout, long format $ZSTD tmp -c > tmpCompressed
$ZSTD tmp --stdout > tmpCompressed # long command format
echo "test : null-length file roundtrip"
echo -n '' | $ZSTD - --stdout | $ZSTD -d --stdout
echo "test : decompress file with wrong suffix (must fail)" echo "test : decompress file with wrong suffix (must fail)"
$ZSTD -d tmpCompressed && die "wrong suffix error not detected!" $ZSTD -d tmpCompressed && die "wrong suffix error not detected!"
$ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout $ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout

2
programs/xxhash.c
View File

@ -123,7 +123,7 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp
***************************************/ ***************************************/
#ifndef MEM_MODULE #ifndef MEM_MODULE
# define MEM_MODULE # define MEM_MODULE
# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ # if !defined (__VMS) && ( defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ )
# include <stdint.h> # include <stdint.h>
typedef uint8_t BYTE; typedef uint8_t BYTE;
typedef uint16_t U16; typedef uint16_t U16;

4
programs/zstdcli.c
View File

@ -75,7 +75,7 @@
# define ZSTD_VERSION "v" ZSTD_VERSION_STRING # define ZSTD_VERSION "v" ZSTD_VERSION_STRING
#endif #endif
#define AUTHOR "Yann Collet" #define AUTHOR "Yann Collet"
#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), ZSTD_VERSION, AUTHOR #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR
#define ZSTD_EXTENSION ".zst" #define ZSTD_EXTENSION ".zst"
#define ZSTD_CAT "zstdcat" #define ZSTD_CAT "zstdcat"
@ -432,7 +432,7 @@ int main(int argCount, const char** argv)
operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel); operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel);
else else
operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName, cLevel); operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName, cLevel);
} else } else
#endif #endif
{ /* decompression */ { /* decompression */
#ifndef ZSTD_NODECOMPRESS #ifndef ZSTD_NODECOMPRESS