AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added : Frame concatenation ability

Browse Source
This commit is contained in:
Yann Collet 2015-09-10 23:26:09 +01:00
parent 5abd8203cb
commit be50aaa0ee
5 changed files with 166 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
lib/fse.c
View File

@ -914,8 +914,8 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
else else
{ {
bitCount -= (int)(8 * (iend - 4 - ip)); bitCount -= (int)(8 * (iend - 4 - ip));
ip = iend - 4; ip = iend - 4;
} }
bitStream = FSE_readLE32(ip) >> (bitCount & 31); bitStream = FSE_readLE32(ip) >> (bitCount & 31);
} }
} }
@ -967,20 +967,20 @@ void FSE_freeCTable (FSE_CTable* ct)
/* provides the minimum logSize to safely represent a distribution */ /* provides the minimum logSize to safely represent a distribution */
static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
{ {
U32 minBitsSrc = FSE_highbit32((U32)(srcSize - 1)) + 1; U32 minBitsSrc = FSE_highbit32((U32)(srcSize - 1)) + 1;
U32 minBitsSymbols = FSE_highbit32(maxSymbolValue) + 2; U32 minBitsSymbols = FSE_highbit32(maxSymbolValue) + 2;
U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
return minBits; return minBits;
} }
unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
{ {
U32 maxBitsSrc = FSE_highbit32((U32)(srcSize - 1)) - 2; U32 maxBitsSrc = FSE_highbit32((U32)(srcSize - 1)) - 2;
U32 tableLog = maxTableLog; U32 tableLog = maxTableLog;
U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */
if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */
if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG; if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG;
if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG; if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG;
return tableLog; return tableLog;
@ -1569,8 +1569,8 @@ size_t FSE_readBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >=
unsigned FSE_reloadDStream(FSE_DStream_t* bitD) unsigned FSE_reloadDStream(FSE_DStream_t* bitD)
{ {
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */ if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
return FSE_DStream_tooFar; return FSE_DStream_tooFar;
if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer))
{ {
@ -1834,7 +1834,7 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* tree, U32
if (maxSymbolValue > (241-128)) return (size_t)-FSE_ERROR_GENERIC; /* not implemented (not possible with current format) */ if (maxSymbolValue > (241-128)) return (size_t)-FSE_ERROR_GENERIC; /* not implemented (not possible with current format) */
if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* not enough space within dst buffer */ if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* not enough space within dst buffer */
op[0] = (BYTE)(128 /*special case*/ + 0 /* Not Compressible */ + (maxSymbolValue-1)); op[0] = (BYTE)(128 /*special case*/ + 0 /* Not Compressible */ + (maxSymbolValue-1));
huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */ huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */
for (n=0; n<maxSymbolValue; n+=2) for (n=0; n<maxSymbolValue; n+=2)
op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]); op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
return ((maxSymbolValue+1)/2) + 1; return ((maxSymbolValue+1)/2) + 1;
@ -1878,7 +1878,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
U32 rankLast[HUF_MAX_TABLELOG]; U32 rankLast[HUF_MAX_TABLELOG];
U32 currentNbBits = maxNbBits; U32 currentNbBits = maxNbBits;
int pos; int pos;
memset(rankLast, 0xF0, sizeof(rankLast)); memset(rankLast, 0xF0, sizeof(rankLast));
for (pos=n ; pos >= 0; pos--) for (pos=n ; pos >= 0; pos--)
{ {
if (huffNode[pos].nbBits >= currentNbBits) continue; if (huffNode[pos].nbBits >= currentNbBits) continue;
@ -1917,20 +1917,20 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
} }
} }
while (totalCost < 0) /* Sometimes, cost correction overshoot */ while (totalCost < 0) /* Sometimes, cost correction overshoot */
{ {
if (rankLast[1] == noOne) /* special case, no weight 1, let's find it back at n */ if (rankLast[1] == noOne) /* special case, no weight 1, let's find it back at n */
{ {
while (huffNode[n].nbBits == maxNbBits) n--; while (huffNode[n].nbBits == maxNbBits) n--;
huffNode[n+1].nbBits--; huffNode[n+1].nbBits--;
rankLast[1] = n+1; rankLast[1] = n+1;
totalCost++; totalCost++;
continue; continue;
} }
huffNode[ rankLast[1] + 1 ].nbBits--; huffNode[ rankLast[1] + 1 ].nbBits--;
rankLast[1]++; rankLast[1]++;
totalCost ++; totalCost ++;
} }
} }
} }
@ -1981,7 +1981,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
/* safety checks */ /* safety checks */
if (maxNbBits == 0) maxNbBits = HUF_DEFAULT_TABLELOG; if (maxNbBits == 0) maxNbBits = HUF_DEFAULT_TABLELOG;
if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE) return (size_t)-FSE_ERROR_GENERIC; if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE) return (size_t)-FSE_ERROR_GENERIC;
memset(huffNode0, 0, sizeof(huffNode0)); memset(huffNode0, 0, sizeof(huffNode0));
// sort, decreasing order // sort, decreasing order
HUF_sort(huffNode, count, maxSymbolValue); HUF_sort(huffNode, count, maxSymbolValue);
@ -2066,7 +2066,7 @@ size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size
FSE_CStream_t bitC; FSE_CStream_t bitC;
/* init */ /* init */
if (dstSize < 8) return 0; if (dstSize < 8) return 0;
op += 6; /* jump Table -- could be optimized by delta / deviation */ op += 6; /* jump Table -- could be optimized by delta / deviation */
errorCode = FSE_initCStream(&bitC, op, oend-op); errorCode = FSE_initCStream(&bitC, op, oend-op);
if (FSE_isError(errorCode)) return 0; if (FSE_isError(errorCode)) return 0;

103
lib/zstd.c
View File

@ -542,11 +542,11 @@ static size_t ZSTD_compressLiterals (void* dst, size_t dstSize,
const size_t minGain = ZSTD_minGain(srcSize); const size_t minGain = ZSTD_minGain(srcSize);
BYTE* const ostart = (BYTE*)dst; BYTE* const ostart = (BYTE*)dst;
size_t hsize; size_t hsize;
static const size_t LHSIZE = 5; static const size_t LHSIZE = 5;
if (dstSize < LHSIZE+1) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* not enough space for compression */ if (dstSize < LHSIZE+1) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* not enough space for compression */
hsize = HUF_compress(ostart+LHSIZE, dstSize-LHSIZE, src, srcSize); hsize = HUF_compress(ostart+LHSIZE, dstSize-LHSIZE, src, srcSize);
if (hsize<2) return hsize; /* special cases */ if (hsize<2) return hsize; /* special cases */
if (hsize >= srcSize - minGain) return 0; if (hsize >= srcSize - minGain) return 0;
@ -619,8 +619,8 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
} }
/* Sequences Header */ /* Sequences Header */
if ((oend-op) < 2+3+6) /* nbSeq + dumpsLength + 3*rleCTable*/ if ((oend-op) < 2+3+6) /* nbSeq + dumpsLength + 3*rleCTable*/
return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
ZSTD_writeLE16(op, (U16)nbSeq); op+=2; ZSTD_writeLE16(op, (U16)nbSeq); op+=2;
seqHead = op; seqHead = op;
@ -640,7 +640,7 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
op[2] = (BYTE)(dumpsLength); op[2] = (BYTE)(dumpsLength);
op += 3; op += 3;
} }
if ((size_t)(oend-op) < dumpsLength+6) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; if ((size_t)(oend-op) < dumpsLength+6) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
memcpy(op, seqStorePtr->dumpsStart, dumpsLength); memcpy(op, seqStorePtr->dumpsStart, dumpsLength);
op += dumpsLength; op += dumpsLength;
} }
@ -661,11 +661,11 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
} }
else else
{ {
size_t NCountSize; size_t NCountSize;
tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max); tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max);
FSE_normalizeCount(norm, tableLog, count, nbSeq, max); FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC; if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
op += NCountSize; op += NCountSize;
FSE_buildCTable(CTable_LitLength, norm, max, tableLog); FSE_buildCTable(CTable_LitLength, norm, max, tableLog);
LLtype = bt_compressed; LLtype = bt_compressed;
@ -696,11 +696,11 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
} }
else else
{ {
size_t NCountSize; size_t NCountSize;
tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max); tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max);
FSE_normalizeCount(norm, tableLog, count, nbSeq, max); FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC; if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
op += NCountSize; op += NCountSize;
FSE_buildCTable(CTable_OffsetBits, norm, max, tableLog); FSE_buildCTable(CTable_OffsetBits, norm, max, tableLog);
Offtype = bt_compressed; Offtype = bt_compressed;
@ -722,11 +722,11 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
} }
else else
{ {
size_t NCountSize; size_t NCountSize;
tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max); tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max);
FSE_normalizeCount(norm, tableLog, count, nbSeq, max); FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC; if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
op += NCountSize; op += NCountSize;
FSE_buildCTable(CTable_MatchLength, norm, max, tableLog); FSE_buildCTable(CTable_MatchLength, norm, max, tableLog);
MLtype = bt_compressed; MLtype = bt_compressed;
@ -1028,7 +1028,7 @@ size_t ZSTD_compressContinue(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize, con
size_t blockSize = BLOCKSIZE; size_t blockSize = BLOCKSIZE;
if (blockSize > srcSize) blockSize = srcSize; if (blockSize > srcSize) blockSize = srcSize;
if (maxDstSize < 2*ZSTD_blockHeaderSize+1) /* one RLE block + endMark */ if (maxDstSize < 2*ZSTD_blockHeaderSize+1) /* one RLE block + endMark */
return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
/* update hash table */ /* update hash table */
@ -1250,8 +1250,8 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
U32 LLlog, Offlog, MLlog; U32 LLlog, Offlog, MLlog;
size_t dumpsLength; size_t dumpsLength;
/* check */ /* check */
if (srcSize < 5) return (size_t)-ZSTD_ERROR_SrcSize; if (srcSize < 5) return (size_t)-ZSTD_ERROR_SrcSize;
/* SeqHead */ /* SeqHead */
*nbSeq = ZSTD_readLE16(ip); ip+=2; *nbSeq = ZSTD_readLE16(ip); ip+=2;
@ -1273,8 +1273,8 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
*dumpsPtr = ip; *dumpsPtr = ip;
ip += dumpsLength; ip += dumpsLength;
/* check */ /* check */
if (ip > iend-3) return (size_t)-ZSTD_ERROR_SrcSize; /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */ if (ip > iend-3) return (size_t)-ZSTD_ERROR_SrcSize; /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
/* sequences */ /* sequences */
{ {
@ -1295,7 +1295,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
max = MaxLL; max = MaxLL;
headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip); headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC; if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
if (LLlog > LLFSELog) return (size_t)-ZSTD_ERROR_corruption; if (LLlog > LLFSELog) return (size_t)-ZSTD_ERROR_corruption;
ip += headerSize; ip += headerSize;
FSE_buildDTable(DTableLL, norm, max, LLlog); FSE_buildDTable(DTableLL, norm, max, LLlog);
} }
@ -1314,7 +1314,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
max = MaxOff; max = MaxOff;
headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip); headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC; if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
if (Offlog > OffFSELog) return (size_t)-ZSTD_ERROR_corruption; if (Offlog > OffFSELog) return (size_t)-ZSTD_ERROR_corruption;
ip += headerSize; ip += headerSize;
FSE_buildDTable(DTableOffb, norm, max, Offlog); FSE_buildDTable(DTableOffb, norm, max, Offlog);
} }
@ -1333,7 +1333,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
max = MaxML; max = MaxML;
headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip); headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC; if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
if (MLlog > MLFSELog) return (size_t)-ZSTD_ERROR_corruption; if (MLlog > MLFSELog) return (size_t)-ZSTD_ERROR_corruption;
ip += headerSize; ip += headerSize;
FSE_buildDTable(DTableML, norm, max, MLlog); FSE_buildDTable(DTableML, norm, max, MLlog);
} }
@ -1417,9 +1417,9 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
static size_t ZSTD_execSequence(BYTE* op, static size_t ZSTD_execSequence(BYTE* op,
seq_t sequence, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit, const BYTE** litPtr, const BYTE* const litLimit,
BYTE* const base, BYTE* const oend) BYTE* const base, BYTE* const oend)
{ {
static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */ static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */ static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */
@ -1430,7 +1430,7 @@ static size_t ZSTD_execSequence(BYTE* op,
/* check */ /* check */
if (endMatch > oend) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* overwrite beyond dst buffer */ if (endMatch > oend) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* overwrite beyond dst buffer */
if (litEnd > litLimit) return (size_t)-ZSTD_ERROR_corruption; if (litEnd > litLimit) return (size_t)-ZSTD_ERROR_corruption;
if (sequence.matchLength > (size_t)(*litPtr-op)) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* overwrite literal segment */ if (sequence.matchLength > (size_t)(*litPtr-op)) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* overwrite literal segment */
/* copy Literals */ /* copy Literals */
@ -1442,18 +1442,18 @@ static size_t ZSTD_execSequence(BYTE* op,
*litPtr = litEnd; /* update for next sequence */ *litPtr = litEnd; /* update for next sequence */
/* check : last match must be at a minimum distance of 8 from end of dest buffer */ /* check : last match must be at a minimum distance of 8 from end of dest buffer */
if (oend-op < 8) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; if (oend-op < 8) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
/* copy Match */ /* copy Match */
{ {
const U32 overlapRisk = (((size_t)(litEnd - endMatch)) < 12); const U32 overlapRisk = (((size_t)(litEnd - endMatch)) < 12);
const BYTE* match = op - sequence.offset; /* possible underflow at op - offset ? */ const BYTE* match = op - sequence.offset; /* possible underflow at op - offset ? */
size_t qutt = 12; size_t qutt = 12;
U64 saved[2]; U64 saved[2];
/* check */ /* check */
if (match < base) return (size_t)-ZSTD_ERROR_corruption; if (match < base) return (size_t)-ZSTD_ERROR_corruption;
if (sequence.offset > (size_t)base) return (size_t)-ZSTD_ERROR_corruption; if (sequence.offset > (size_t)base) return (size_t)-ZSTD_ERROR_corruption;
/* save beginning of literal sequence, in case of write overlap */ /* save beginning of literal sequence, in case of write overlap */
if (overlapRisk) if (overlapRisk)
@ -1473,7 +1473,7 @@ static size_t ZSTD_execSequence(BYTE* op,
ZSTD_copy4(op+4, match); ZSTD_copy4(op+4, match);
match -= dec64; match -= dec64;
} else { ZSTD_copy8(op, match); } } else { ZSTD_copy8(op, match); }
op += 8; match += 8; op += 8; match += 8;
if (endMatch > oend-12) if (endMatch > oend-12)
{ {
@ -1498,11 +1498,11 @@ static size_t ZSTD_execSequence(BYTE* op,
typedef struct ZSTD_Dctx_s typedef struct ZSTD_Dctx_s
{ {
U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)]; U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)]; U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)]; U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
void* previousDstEnd; void* previousDstEnd;
void* base; void* base;
size_t expected; size_t expected;
blockType_t bType; blockType_t bType;
U32 phase; U32 phase;
} dctx_t; } dctx_t;
@ -1514,7 +1514,7 @@ static size_t ZSTD_decompressSequences(
const void* seqStart, size_t seqSize, const void* seqStart, size_t seqSize,
const BYTE* litStart, size_t litSize) const BYTE* litStart, size_t litSize)
{ {
dctx_t* dctx = (dctx_t*)ctx; dctx_t* dctx = (dctx_t*)ctx;
const BYTE* ip = (const BYTE*)seqStart; const BYTE* ip = (const BYTE*)seqStart;
const BYTE* const iend = ip + seqSize; const BYTE* const iend = ip + seqSize;
BYTE* const ostart = (BYTE* const)dst; BYTE* const ostart = (BYTE* const)dst;
@ -1526,9 +1526,9 @@ static size_t ZSTD_decompressSequences(
int nbSeq; int nbSeq;
const BYTE* dumps; const BYTE* dumps;
U32* DTableLL = dctx->LLTable; U32* DTableLL = dctx->LLTable;
U32* DTableML = dctx->MLTable; U32* DTableML = dctx->MLTable;
U32* DTableOffb = dctx->OffTable; U32* DTableOffb = dctx->OffTable;
BYTE* const base = (BYTE*) (dctx->base); BYTE* const base = (BYTE*) (dctx->base);
/* Build Decoding Tables */ /* Build Decoding Tables */
errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps, errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps,
@ -1658,8 +1658,8 @@ static size_t ZSTD_decompressDCtx(void* ctx, void* dst, size_t maxDstSize, const
size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize) size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{ {
dctx_t ctx; dctx_t ctx;
ctx.base = dst; ctx.base = dst;
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize); return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
} }
@ -1668,14 +1668,20 @@ size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t src
* Streaming Decompression API * Streaming Decompression API
*******************************/ *******************************/
size_t ZSTD_resetDCtx(ZSTD_Dctx* dctx)
{
dctx->expected = ZSTD_frameHeaderSize;
dctx->phase = 0;
dctx->previousDstEnd = NULL;
dctx->base = NULL;
return 0;
}
ZSTD_Dctx* ZSTD_createDCtx(void) ZSTD_Dctx* ZSTD_createDCtx(void)
{ {
ZSTD_Dctx* dctx = (ZSTD_Dctx*)malloc(sizeof(ZSTD_Dctx)); ZSTD_Dctx* dctx = (ZSTD_Dctx*)malloc(sizeof(ZSTD_Dctx));
if (dctx==NULL) return NULL; if (dctx==NULL) return NULL;
dctx->expected = ZSTD_frameHeaderSize; ZSTD_resetDCtx(dctx);
dctx->phase = 0;
dctx->previousDstEnd = NULL;
dctx->base = NULL;
return dctx; return dctx;
} }
@ -1685,7 +1691,6 @@ size_t ZSTD_freeDCtx(ZSTD_Dctx* dctx)
return 0; return 0;
} }
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_Dctx* dctx) size_t ZSTD_nextSrcSizeToDecompress(ZSTD_Dctx* dctx)
{ {
return ((dctx_t*)dctx)->expected; return ((dctx_t*)dctx)->expected;
@ -1697,8 +1702,8 @@ size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, co
/* Sanity check */ /* Sanity check */
if (srcSize != ctx->expected) return (size_t)-ZSTD_ERROR_SrcSize; if (srcSize != ctx->expected) return (size_t)-ZSTD_ERROR_SrcSize;
if (dst != ctx->previousDstEnd) /* not contiguous */ if (dst != ctx->previousDstEnd) /* not contiguous */
ctx->base = dst; ctx->base = dst;
/* Decompress : frame header */ /* Decompress : frame header */
if (ctx->phase == 0) if (ctx->phase == 0)
@ -1754,7 +1759,7 @@ size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, co
} }
ctx->phase = 1; ctx->phase = 1;
ctx->expected = ZSTD_blockHeaderSize; ctx->expected = ZSTD_blockHeaderSize;
ctx->previousDstEnd = (void*)( ((char*)dst) + rSize); ctx->previousDstEnd = (void*)( ((char*)dst) + rSize);
return rSize; return rSize;
} }

1
lib/zstd_static.h
View File

@ -56,6 +56,7 @@ size_t ZSTD_compressEnd(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize);
typedef struct ZSTD_Dctx_s ZSTD_Dctx; typedef struct ZSTD_Dctx_s ZSTD_Dctx;
ZSTD_Dctx* ZSTD_createDCtx(void); ZSTD_Dctx* ZSTD_createDCtx(void);
size_t ZSTD_resetDCtx(ZSTD_Dctx* dctx);
size_t ZSTD_freeDCtx(ZSTD_Dctx* dctx); size_t ZSTD_freeDCtx(ZSTD_Dctx* dctx);
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_Dctx* dctx); size_t ZSTD_nextSrcSizeToDecompress(ZSTD_Dctx* dctx);

16
programs/Makefile
View File

@ -120,10 +120,22 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32
test-all: test test32 memtest test-all: test test32 memtest
test-zstd: zstd datagen test-zstd: zstd datagen
@echo "*** flush write error test ***" @echo "\n**** frame concatenation **** "
@echo "hello " > hello.tmp
@echo "world!" > world.tmp
@cat hello.tmp world.tmp > helloworld.tmp
./zstd hello.tmp > hello.zstd
./zstd world.tmp > world.zstd
@cat hello.zstd world.zstd > helloworld.zstd
./zstd -d helloworld.zstd > result.tmp
cat result.tmp
sdiff helloworld.tmp result.tmp
@rm *.tmp *.zstd
@echo frame concatenation test completed
@echo "**** flush write error test **** "
echo foo | ./zstd > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi echo foo | ./zstd > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
echo foo | ./zstd | ./zstd -d > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi echo foo | ./zstd | ./zstd -d > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
@echo "*** zstd round-trip tests *** " @echo "**** zstd round-trip tests **** "
./datagen | ./zstd -v | ./zstd -d > $(VOID) ./datagen | ./zstd -v | ./zstd -d > $(VOID)
./datagen -g256MB | ./zstd -v | ./zstd -d > $(VOID) ./datagen -g256MB | ./zstd -v | ./zstd -d > $(VOID)
./datagen -g6GB -P99 | ./zstd -vq | ./zstd -d > $(VOID) ./datagen -g6GB -P99 | ./zstd -vq | ./zstd -d > $(VOID)

100
programs/fileio.c
View File

@ -309,45 +309,18 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
#define MAXHEADERSIZE FIO_FRAMEHEADERSIZE+3 #define MAXHEADERSIZE FIO_FRAMEHEADERSIZE+3
unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename) unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
BYTE* inBuff, size_t inBuffSize,
BYTE* outBuff, size_t outBuffSize,
ZSTD_Dctx* dctx)
{ {
FILE* finput, *foutput; BYTE* op = outBuff;
BYTE* inBuff; BYTE* const oend = outBuff + outBuffSize;
size_t inBuffSize;
BYTE* outBuff, *op, *oend;
size_t outBuffSize;
U32 blockSize = 128 KB;
U32 wNbBlocks = 4;
U64 filesize = 0; U64 filesize = 0;
BYTE* header[MAXHEADERSIZE];
ZSTD_Dctx* dctx;
size_t toRead; size_t toRead;
size_t sizeCheck; size_t sizeCheck;
/* Init */
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
dctx = ZSTD_createDCtx();
/* check header */
toRead = ZSTD_nextSrcSizeToDecompress(dctx);
if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
sizeCheck = fread(header, (size_t)1, toRead, finput);
if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead); // Decode frame header
if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
/* Here later : blockSize determination */
/* Allocate Memory */
inBuffSize = blockSize + FIO_blockHeaderSize;
inBuff = (BYTE*)malloc(inBuffSize);
outBuffSize = wNbBlocks * blockSize;
outBuff = (BYTE*)malloc(outBuffSize);
op = outBuff;
oend = outBuff + outBuffSize;
if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
/* Main decompression Loop */ /* Main decompression Loop */
toRead = ZSTD_nextSrcSizeToDecompress(dctx); toRead = ZSTD_nextSrcSizeToDecompress(dctx);
while (toRead) while (toRead)
@ -380,6 +353,67 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
toRead = ZSTD_nextSrcSizeToDecompress(dctx); toRead = ZSTD_nextSrcSizeToDecompress(dctx);
} }
return filesize;
}
unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
{
FILE* finput, *foutput;
BYTE* inBuff=NULL;
size_t inBuffSize = 0;
BYTE* outBuff=NULL;
size_t outBuffSize = 0;
U32 blockSize = 128 KB;
U32 wNbBlocks = 4;
U64 filesize = 0;
BYTE* header[MAXHEADERSIZE];
ZSTD_Dctx* dctx;
size_t toRead;
size_t sizeCheck;
/* Init */
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
dctx = ZSTD_createDCtx();
/* for each frame */
for ( ; ; )
{
/* check header */
ZSTD_resetDCtx(dctx);
toRead = ZSTD_nextSrcSizeToDecompress(dctx);
if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
sizeCheck = fread(header, (size_t)1, toRead, finput);
if (sizeCheck==0) break; /* no more input */
if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead); // Decode frame header
if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
/* Here later : blockSize determination */
/* Allocate Memory (if needed) */
{
size_t newInBuffSize = blockSize + FIO_blockHeaderSize;
size_t newOutBuffSize = wNbBlocks * blockSize;
if (newInBuffSize > inBuffSize)
{
free(inBuff);
inBuffSize = newInBuffSize;
inBuff = (BYTE*)malloc(inBuffSize);
}
if (newOutBuffSize > outBuffSize)
{
free(outBuff);
outBuffSize = newOutBuffSize;
outBuff = (BYTE*)malloc(outBuffSize);
}
}
if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, outBuff, outBuffSize, dctx);
}
DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2, "\r%79s\r", "");
DISPLAYLEVEL(2,"Decoded %llu bytes \n", (long long unsigned)filesize); DISPLAYLEVEL(2,"Decoded %llu bytes \n", (long long unsigned)filesize);