Added : Frame concatenation ability
This commit is contained in:
parent
5abd8203cb
commit
be50aaa0ee
60
lib/fse.c
60
lib/fse.c
@ -914,8 +914,8 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
|
||||
else
|
||||
{
|
||||
bitCount -= (int)(8 * (iend - 4 - ip));
|
||||
ip = iend - 4;
|
||||
}
|
||||
ip = iend - 4;
|
||||
}
|
||||
bitStream = FSE_readLE32(ip) >> (bitCount & 31);
|
||||
}
|
||||
}
|
||||
@ -967,20 +967,20 @@ void FSE_freeCTable (FSE_CTable* ct)
|
||||
/* provides the minimum logSize to safely represent a distribution */
|
||||
static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
|
||||
{
|
||||
U32 minBitsSrc = FSE_highbit32((U32)(srcSize - 1)) + 1;
|
||||
U32 minBitsSymbols = FSE_highbit32(maxSymbolValue) + 2;
|
||||
U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
|
||||
return minBits;
|
||||
U32 minBitsSrc = FSE_highbit32((U32)(srcSize - 1)) + 1;
|
||||
U32 minBitsSymbols = FSE_highbit32(maxSymbolValue) + 2;
|
||||
U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
|
||||
return minBits;
|
||||
}
|
||||
|
||||
unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
|
||||
{
|
||||
U32 maxBitsSrc = FSE_highbit32((U32)(srcSize - 1)) - 2;
|
||||
U32 maxBitsSrc = FSE_highbit32((U32)(srcSize - 1)) - 2;
|
||||
U32 tableLog = maxTableLog;
|
||||
U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
|
||||
U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
|
||||
if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
|
||||
if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */
|
||||
if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */
|
||||
if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */
|
||||
if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */
|
||||
if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG;
|
||||
if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG;
|
||||
return tableLog;
|
||||
@ -1569,8 +1569,8 @@ size_t FSE_readBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >=
|
||||
|
||||
unsigned FSE_reloadDStream(FSE_DStream_t* bitD)
|
||||
{
|
||||
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
|
||||
return FSE_DStream_tooFar;
|
||||
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
|
||||
return FSE_DStream_tooFar;
|
||||
|
||||
if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer))
|
||||
{
|
||||
@ -1834,7 +1834,7 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* tree, U32
|
||||
if (maxSymbolValue > (241-128)) return (size_t)-FSE_ERROR_GENERIC; /* not implemented (not possible with current format) */
|
||||
if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* not enough space within dst buffer */
|
||||
op[0] = (BYTE)(128 /*special case*/ + 0 /* Not Compressible */ + (maxSymbolValue-1));
|
||||
huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */
|
||||
huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */
|
||||
for (n=0; n<maxSymbolValue; n+=2)
|
||||
op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
|
||||
return ((maxSymbolValue+1)/2) + 1;
|
||||
@ -1878,7 +1878,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
|
||||
U32 rankLast[HUF_MAX_TABLELOG];
|
||||
U32 currentNbBits = maxNbBits;
|
||||
int pos;
|
||||
memset(rankLast, 0xF0, sizeof(rankLast));
|
||||
memset(rankLast, 0xF0, sizeof(rankLast));
|
||||
for (pos=n ; pos >= 0; pos--)
|
||||
{
|
||||
if (huffNode[pos].nbBits >= currentNbBits) continue;
|
||||
@ -1917,20 +1917,20 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
|
||||
}
|
||||
}
|
||||
|
||||
while (totalCost < 0) /* Sometimes, cost correction overshoot */
|
||||
{
|
||||
if (rankLast[1] == noOne) /* special case, no weight 1, let's find it back at n */
|
||||
{
|
||||
while (huffNode[n].nbBits == maxNbBits) n--;
|
||||
huffNode[n+1].nbBits--;
|
||||
rankLast[1] = n+1;
|
||||
totalCost++;
|
||||
continue;
|
||||
}
|
||||
huffNode[ rankLast[1] + 1 ].nbBits--;
|
||||
rankLast[1]++;
|
||||
totalCost ++;
|
||||
}
|
||||
while (totalCost < 0) /* Sometimes, cost correction overshoot */
|
||||
{
|
||||
if (rankLast[1] == noOne) /* special case, no weight 1, let's find it back at n */
|
||||
{
|
||||
while (huffNode[n].nbBits == maxNbBits) n--;
|
||||
huffNode[n+1].nbBits--;
|
||||
rankLast[1] = n+1;
|
||||
totalCost++;
|
||||
continue;
|
||||
}
|
||||
huffNode[ rankLast[1] + 1 ].nbBits--;
|
||||
rankLast[1]++;
|
||||
totalCost ++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1981,7 +1981,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
|
||||
/* safety checks */
|
||||
if (maxNbBits == 0) maxNbBits = HUF_DEFAULT_TABLELOG;
|
||||
if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE) return (size_t)-FSE_ERROR_GENERIC;
|
||||
memset(huffNode0, 0, sizeof(huffNode0));
|
||||
memset(huffNode0, 0, sizeof(huffNode0));
|
||||
|
||||
// sort, decreasing order
|
||||
HUF_sort(huffNode, count, maxSymbolValue);
|
||||
@ -2066,7 +2066,7 @@ size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size
|
||||
FSE_CStream_t bitC;
|
||||
|
||||
/* init */
|
||||
if (dstSize < 8) return 0;
|
||||
if (dstSize < 8) return 0;
|
||||
op += 6; /* jump Table -- could be optimized by delta / deviation */
|
||||
errorCode = FSE_initCStream(&bitC, op, oend-op);
|
||||
if (FSE_isError(errorCode)) return 0;
|
||||
|
103
lib/zstd.c
103
lib/zstd.c
@ -542,11 +542,11 @@ static size_t ZSTD_compressLiterals (void* dst, size_t dstSize,
|
||||
const size_t minGain = ZSTD_minGain(srcSize);
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
size_t hsize;
|
||||
static const size_t LHSIZE = 5;
|
||||
static const size_t LHSIZE = 5;
|
||||
|
||||
if (dstSize < LHSIZE+1) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* not enough space for compression */
|
||||
if (dstSize < LHSIZE+1) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* not enough space for compression */
|
||||
|
||||
hsize = HUF_compress(ostart+LHSIZE, dstSize-LHSIZE, src, srcSize);
|
||||
hsize = HUF_compress(ostart+LHSIZE, dstSize-LHSIZE, src, srcSize);
|
||||
if (hsize<2) return hsize; /* special cases */
|
||||
if (hsize >= srcSize - minGain) return 0;
|
||||
|
||||
@ -619,8 +619,8 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
|
||||
}
|
||||
|
||||
/* Sequences Header */
|
||||
if ((oend-op) < 2+3+6) /* nbSeq + dumpsLength + 3*rleCTable*/
|
||||
return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
|
||||
if ((oend-op) < 2+3+6) /* nbSeq + dumpsLength + 3*rleCTable*/
|
||||
return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
|
||||
ZSTD_writeLE16(op, (U16)nbSeq); op+=2;
|
||||
seqHead = op;
|
||||
|
||||
@ -640,7 +640,7 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
|
||||
op[2] = (BYTE)(dumpsLength);
|
||||
op += 3;
|
||||
}
|
||||
if ((size_t)(oend-op) < dumpsLength+6) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
|
||||
if ((size_t)(oend-op) < dumpsLength+6) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
|
||||
memcpy(op, seqStorePtr->dumpsStart, dumpsLength);
|
||||
op += dumpsLength;
|
||||
}
|
||||
@ -661,11 +661,11 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
|
||||
}
|
||||
else
|
||||
{
|
||||
size_t NCountSize;
|
||||
size_t NCountSize;
|
||||
tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max);
|
||||
FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
|
||||
NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
|
||||
if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
|
||||
NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
|
||||
if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
|
||||
op += NCountSize;
|
||||
FSE_buildCTable(CTable_LitLength, norm, max, tableLog);
|
||||
LLtype = bt_compressed;
|
||||
@ -696,11 +696,11 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
|
||||
}
|
||||
else
|
||||
{
|
||||
size_t NCountSize;
|
||||
size_t NCountSize;
|
||||
tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max);
|
||||
FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
|
||||
NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
|
||||
if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
|
||||
NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
|
||||
if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
|
||||
op += NCountSize;
|
||||
FSE_buildCTable(CTable_OffsetBits, norm, max, tableLog);
|
||||
Offtype = bt_compressed;
|
||||
@ -722,11 +722,11 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
|
||||
}
|
||||
else
|
||||
{
|
||||
size_t NCountSize;
|
||||
size_t NCountSize;
|
||||
tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max);
|
||||
FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
|
||||
NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
|
||||
if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
|
||||
NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
|
||||
if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
|
||||
op += NCountSize;
|
||||
FSE_buildCTable(CTable_MatchLength, norm, max, tableLog);
|
||||
MLtype = bt_compressed;
|
||||
@ -1028,7 +1028,7 @@ size_t ZSTD_compressContinue(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize, con
|
||||
size_t blockSize = BLOCKSIZE;
|
||||
if (blockSize > srcSize) blockSize = srcSize;
|
||||
|
||||
if (maxDstSize < 2*ZSTD_blockHeaderSize+1) /* one RLE block + endMark */
|
||||
if (maxDstSize < 2*ZSTD_blockHeaderSize+1) /* one RLE block + endMark */
|
||||
return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
|
||||
|
||||
/* update hash table */
|
||||
@ -1250,8 +1250,8 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
|
||||
U32 LLlog, Offlog, MLlog;
|
||||
size_t dumpsLength;
|
||||
|
||||
/* check */
|
||||
if (srcSize < 5) return (size_t)-ZSTD_ERROR_SrcSize;
|
||||
/* check */
|
||||
if (srcSize < 5) return (size_t)-ZSTD_ERROR_SrcSize;
|
||||
|
||||
/* SeqHead */
|
||||
*nbSeq = ZSTD_readLE16(ip); ip+=2;
|
||||
@ -1273,8 +1273,8 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
|
||||
*dumpsPtr = ip;
|
||||
ip += dumpsLength;
|
||||
|
||||
/* check */
|
||||
if (ip > iend-3) return (size_t)-ZSTD_ERROR_SrcSize; /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
|
||||
/* check */
|
||||
if (ip > iend-3) return (size_t)-ZSTD_ERROR_SrcSize; /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
|
||||
|
||||
/* sequences */
|
||||
{
|
||||
@ -1295,7 +1295,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
|
||||
max = MaxLL;
|
||||
headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
|
||||
if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
|
||||
if (LLlog > LLFSELog) return (size_t)-ZSTD_ERROR_corruption;
|
||||
if (LLlog > LLFSELog) return (size_t)-ZSTD_ERROR_corruption;
|
||||
ip += headerSize;
|
||||
FSE_buildDTable(DTableLL, norm, max, LLlog);
|
||||
}
|
||||
@ -1314,7 +1314,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
|
||||
max = MaxOff;
|
||||
headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
|
||||
if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
|
||||
if (Offlog > OffFSELog) return (size_t)-ZSTD_ERROR_corruption;
|
||||
if (Offlog > OffFSELog) return (size_t)-ZSTD_ERROR_corruption;
|
||||
ip += headerSize;
|
||||
FSE_buildDTable(DTableOffb, norm, max, Offlog);
|
||||
}
|
||||
@ -1333,7 +1333,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
|
||||
max = MaxML;
|
||||
headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
|
||||
if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
|
||||
if (MLlog > MLFSELog) return (size_t)-ZSTD_ERROR_corruption;
|
||||
if (MLlog > MLFSELog) return (size_t)-ZSTD_ERROR_corruption;
|
||||
ip += headerSize;
|
||||
FSE_buildDTable(DTableML, norm, max, MLlog);
|
||||
}
|
||||
@ -1417,9 +1417,9 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
||||
|
||||
|
||||
static size_t ZSTD_execSequence(BYTE* op,
|
||||
seq_t sequence,
|
||||
const BYTE** litPtr, const BYTE* const litLimit,
|
||||
BYTE* const base, BYTE* const oend)
|
||||
seq_t sequence,
|
||||
const BYTE** litPtr, const BYTE* const litLimit,
|
||||
BYTE* const base, BYTE* const oend)
|
||||
{
|
||||
static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
|
||||
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */
|
||||
@ -1430,7 +1430,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
||||
|
||||
/* check */
|
||||
if (endMatch > oend) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* overwrite beyond dst buffer */
|
||||
if (litEnd > litLimit) return (size_t)-ZSTD_ERROR_corruption;
|
||||
if (litEnd > litLimit) return (size_t)-ZSTD_ERROR_corruption;
|
||||
if (sequence.matchLength > (size_t)(*litPtr-op)) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* overwrite literal segment */
|
||||
|
||||
/* copy Literals */
|
||||
@ -1442,18 +1442,18 @@ static size_t ZSTD_execSequence(BYTE* op,
|
||||
*litPtr = litEnd; /* update for next sequence */
|
||||
|
||||
/* check : last match must be at a minimum distance of 8 from end of dest buffer */
|
||||
if (oend-op < 8) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
|
||||
if (oend-op < 8) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
|
||||
|
||||
/* copy Match */
|
||||
/* copy Match */
|
||||
{
|
||||
const U32 overlapRisk = (((size_t)(litEnd - endMatch)) < 12);
|
||||
const BYTE* match = op - sequence.offset; /* possible underflow at op - offset ? */
|
||||
size_t qutt = 12;
|
||||
U64 saved[2];
|
||||
|
||||
/* check */
|
||||
if (match < base) return (size_t)-ZSTD_ERROR_corruption;
|
||||
if (sequence.offset > (size_t)base) return (size_t)-ZSTD_ERROR_corruption;
|
||||
/* check */
|
||||
if (match < base) return (size_t)-ZSTD_ERROR_corruption;
|
||||
if (sequence.offset > (size_t)base) return (size_t)-ZSTD_ERROR_corruption;
|
||||
|
||||
/* save beginning of literal sequence, in case of write overlap */
|
||||
if (overlapRisk)
|
||||
@ -1473,7 +1473,7 @@ static size_t ZSTD_execSequence(BYTE* op,
|
||||
ZSTD_copy4(op+4, match);
|
||||
match -= dec64;
|
||||
} else { ZSTD_copy8(op, match); }
|
||||
op += 8; match += 8;
|
||||
op += 8; match += 8;
|
||||
|
||||
if (endMatch > oend-12)
|
||||
{
|
||||
@ -1498,11 +1498,11 @@ static size_t ZSTD_execSequence(BYTE* op,
|
||||
typedef struct ZSTD_Dctx_s
|
||||
{
|
||||
U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
|
||||
U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
|
||||
U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
|
||||
U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
|
||||
U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
|
||||
void* previousDstEnd;
|
||||
void* base;
|
||||
size_t expected;
|
||||
size_t expected;
|
||||
blockType_t bType;
|
||||
U32 phase;
|
||||
} dctx_t;
|
||||
@ -1514,7 +1514,7 @@ static size_t ZSTD_decompressSequences(
|
||||
const void* seqStart, size_t seqSize,
|
||||
const BYTE* litStart, size_t litSize)
|
||||
{
|
||||
dctx_t* dctx = (dctx_t*)ctx;
|
||||
dctx_t* dctx = (dctx_t*)ctx;
|
||||
const BYTE* ip = (const BYTE*)seqStart;
|
||||
const BYTE* const iend = ip + seqSize;
|
||||
BYTE* const ostart = (BYTE* const)dst;
|
||||
@ -1526,9 +1526,9 @@ static size_t ZSTD_decompressSequences(
|
||||
int nbSeq;
|
||||
const BYTE* dumps;
|
||||
U32* DTableLL = dctx->LLTable;
|
||||
U32* DTableML = dctx->MLTable;
|
||||
U32* DTableML = dctx->MLTable;
|
||||
U32* DTableOffb = dctx->OffTable;
|
||||
BYTE* const base = (BYTE*) (dctx->base);
|
||||
BYTE* const base = (BYTE*) (dctx->base);
|
||||
|
||||
/* Build Decoding Tables */
|
||||
errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps,
|
||||
@ -1658,8 +1658,8 @@ static size_t ZSTD_decompressDCtx(void* ctx, void* dst, size_t maxDstSize, const
|
||||
|
||||
size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
dctx_t ctx;
|
||||
ctx.base = dst;
|
||||
dctx_t ctx;
|
||||
ctx.base = dst;
|
||||
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
||||
}
|
||||
|
||||
@ -1668,14 +1668,20 @@ size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t src
|
||||
* Streaming Decompression API
|
||||
*******************************/
|
||||
|
||||
size_t ZSTD_resetDCtx(ZSTD_Dctx* dctx)
|
||||
{
|
||||
dctx->expected = ZSTD_frameHeaderSize;
|
||||
dctx->phase = 0;
|
||||
dctx->previousDstEnd = NULL;
|
||||
dctx->base = NULL;
|
||||
return 0;
|
||||
}
|
||||
|
||||
ZSTD_Dctx* ZSTD_createDCtx(void)
|
||||
{
|
||||
ZSTD_Dctx* dctx = (ZSTD_Dctx*)malloc(sizeof(ZSTD_Dctx));
|
||||
if (dctx==NULL) return NULL;
|
||||
dctx->expected = ZSTD_frameHeaderSize;
|
||||
dctx->phase = 0;
|
||||
dctx->previousDstEnd = NULL;
|
||||
dctx->base = NULL;
|
||||
ZSTD_resetDCtx(dctx);
|
||||
return dctx;
|
||||
}
|
||||
|
||||
@ -1685,7 +1691,6 @@ size_t ZSTD_freeDCtx(ZSTD_Dctx* dctx)
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_Dctx* dctx)
|
||||
{
|
||||
return ((dctx_t*)dctx)->expected;
|
||||
@ -1697,8 +1702,8 @@ size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, co
|
||||
|
||||
/* Sanity check */
|
||||
if (srcSize != ctx->expected) return (size_t)-ZSTD_ERROR_SrcSize;
|
||||
if (dst != ctx->previousDstEnd) /* not contiguous */
|
||||
ctx->base = dst;
|
||||
if (dst != ctx->previousDstEnd) /* not contiguous */
|
||||
ctx->base = dst;
|
||||
|
||||
/* Decompress : frame header */
|
||||
if (ctx->phase == 0)
|
||||
@ -1754,7 +1759,7 @@ size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, co
|
||||
}
|
||||
ctx->phase = 1;
|
||||
ctx->expected = ZSTD_blockHeaderSize;
|
||||
ctx->previousDstEnd = (void*)( ((char*)dst) + rSize);
|
||||
ctx->previousDstEnd = (void*)( ((char*)dst) + rSize);
|
||||
return rSize;
|
||||
}
|
||||
|
||||
|
@ -56,6 +56,7 @@ size_t ZSTD_compressEnd(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize);
|
||||
|
||||
typedef struct ZSTD_Dctx_s ZSTD_Dctx;
|
||||
ZSTD_Dctx* ZSTD_createDCtx(void);
|
||||
size_t ZSTD_resetDCtx(ZSTD_Dctx* dctx);
|
||||
size_t ZSTD_freeDCtx(ZSTD_Dctx* dctx);
|
||||
|
||||
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_Dctx* dctx);
|
||||
|
@ -120,10 +120,22 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32
|
||||
test-all: test test32 memtest
|
||||
|
||||
test-zstd: zstd datagen
|
||||
@echo "*** flush write error test ***"
|
||||
@echo "\n**** frame concatenation **** "
|
||||
@echo "hello " > hello.tmp
|
||||
@echo "world!" > world.tmp
|
||||
@cat hello.tmp world.tmp > helloworld.tmp
|
||||
./zstd hello.tmp > hello.zstd
|
||||
./zstd world.tmp > world.zstd
|
||||
@cat hello.zstd world.zstd > helloworld.zstd
|
||||
./zstd -d helloworld.zstd > result.tmp
|
||||
cat result.tmp
|
||||
sdiff helloworld.tmp result.tmp
|
||||
@rm *.tmp *.zstd
|
||||
@echo frame concatenation test completed
|
||||
@echo "**** flush write error test **** "
|
||||
echo foo | ./zstd > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
|
||||
echo foo | ./zstd | ./zstd -d > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
|
||||
@echo "*** zstd round-trip tests *** "
|
||||
@echo "**** zstd round-trip tests **** "
|
||||
./datagen | ./zstd -v | ./zstd -d > $(VOID)
|
||||
./datagen -g256MB | ./zstd -v | ./zstd -d > $(VOID)
|
||||
./datagen -g6GB -P99 | ./zstd -vq | ./zstd -d > $(VOID)
|
||||
|
@ -309,45 +309,18 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
|
||||
|
||||
|
||||
#define MAXHEADERSIZE FIO_FRAMEHEADERSIZE+3
|
||||
unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
|
||||
unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
|
||||
BYTE* inBuff, size_t inBuffSize,
|
||||
BYTE* outBuff, size_t outBuffSize,
|
||||
ZSTD_Dctx* dctx)
|
||||
{
|
||||
FILE* finput, *foutput;
|
||||
BYTE* inBuff;
|
||||
size_t inBuffSize;
|
||||
BYTE* outBuff, *op, *oend;
|
||||
size_t outBuffSize;
|
||||
U32 blockSize = 128 KB;
|
||||
U32 wNbBlocks = 4;
|
||||
BYTE* op = outBuff;
|
||||
BYTE* const oend = outBuff + outBuffSize;
|
||||
U64 filesize = 0;
|
||||
BYTE* header[MAXHEADERSIZE];
|
||||
ZSTD_Dctx* dctx;
|
||||
size_t toRead;
|
||||
size_t sizeCheck;
|
||||
|
||||
|
||||
/* Init */
|
||||
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
|
||||
dctx = ZSTD_createDCtx();
|
||||
|
||||
/* check header */
|
||||
toRead = ZSTD_nextSrcSizeToDecompress(dctx);
|
||||
if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
|
||||
sizeCheck = fread(header, (size_t)1, toRead, finput);
|
||||
if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
|
||||
sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead); // Decode frame header
|
||||
if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
|
||||
|
||||
/* Here later : blockSize determination */
|
||||
|
||||
/* Allocate Memory */
|
||||
inBuffSize = blockSize + FIO_blockHeaderSize;
|
||||
inBuff = (BYTE*)malloc(inBuffSize);
|
||||
outBuffSize = wNbBlocks * blockSize;
|
||||
outBuff = (BYTE*)malloc(outBuffSize);
|
||||
op = outBuff;
|
||||
oend = outBuff + outBuffSize;
|
||||
if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
|
||||
|
||||
/* Main decompression Loop */
|
||||
toRead = ZSTD_nextSrcSizeToDecompress(dctx);
|
||||
while (toRead)
|
||||
@ -380,6 +353,67 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
|
||||
toRead = ZSTD_nextSrcSizeToDecompress(dctx);
|
||||
}
|
||||
|
||||
return filesize;
|
||||
}
|
||||
|
||||
|
||||
unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
|
||||
{
|
||||
FILE* finput, *foutput;
|
||||
BYTE* inBuff=NULL;
|
||||
size_t inBuffSize = 0;
|
||||
BYTE* outBuff=NULL;
|
||||
size_t outBuffSize = 0;
|
||||
U32 blockSize = 128 KB;
|
||||
U32 wNbBlocks = 4;
|
||||
U64 filesize = 0;
|
||||
BYTE* header[MAXHEADERSIZE];
|
||||
ZSTD_Dctx* dctx;
|
||||
size_t toRead;
|
||||
size_t sizeCheck;
|
||||
|
||||
|
||||
/* Init */
|
||||
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
|
||||
dctx = ZSTD_createDCtx();
|
||||
|
||||
/* for each frame */
|
||||
for ( ; ; )
|
||||
{
|
||||
/* check header */
|
||||
ZSTD_resetDCtx(dctx);
|
||||
toRead = ZSTD_nextSrcSizeToDecompress(dctx);
|
||||
if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
|
||||
sizeCheck = fread(header, (size_t)1, toRead, finput);
|
||||
if (sizeCheck==0) break; /* no more input */
|
||||
if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
|
||||
sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead); // Decode frame header
|
||||
if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
|
||||
|
||||
/* Here later : blockSize determination */
|
||||
|
||||
/* Allocate Memory (if needed) */
|
||||
{
|
||||
size_t newInBuffSize = blockSize + FIO_blockHeaderSize;
|
||||
size_t newOutBuffSize = wNbBlocks * blockSize;
|
||||
if (newInBuffSize > inBuffSize)
|
||||
{
|
||||
free(inBuff);
|
||||
inBuffSize = newInBuffSize;
|
||||
inBuff = (BYTE*)malloc(inBuffSize);
|
||||
}
|
||||
if (newOutBuffSize > outBuffSize)
|
||||
{
|
||||
free(outBuff);
|
||||
outBuffSize = newOutBuffSize;
|
||||
outBuff = (BYTE*)malloc(outBuffSize);
|
||||
}
|
||||
}
|
||||
if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
|
||||
|
||||
filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, outBuff, outBuffSize, dctx);
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
DISPLAYLEVEL(2,"Decoded %llu bytes \n", (long long unsigned)filesize);
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user