AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #44 from Cyan4973/dev

Browse Source 
Dev
This commit is contained in:
Yann Collet 2015-09-11 01:22:25 +02:00
commit 9c57b424d6
11 changed files with 186 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -32,7 +32,7 @@
# ################################################################
# Version number
export VERSION=0.1.1
export VERSION := 0.1.2
PRGDIR = programs
ZSTDDIR = lib

10
NEWS
View File

@ -1,3 +1,9 @@
r0
initial release
v0.1.2
v0.1.1
fix compression bug
detects write-flush errors
v0.1.0
first release

5
lib/Makefile
View File

@ -32,7 +32,7 @@
# ################################################################
# Version numbers
VERSION?= 0.1.1
VERSION?= 0.1.2
LIBVER_MAJOR=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
LIBVER_MINOR=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
LIBVER_PATCH=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
@ -41,8 +41,7 @@ LIBVER = $(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH)
DESTDIR?=
PREFIX ?= /usr/local
CFLAGS ?= -O3
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes
LDFLAGS = -I.
CFLAGS += -I. -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
LIBDIR ?= $(PREFIX)/lib

60
lib/fse.c
View File

@ -914,8 +914,8 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
else
{
bitCount -= (int)(8 * (iend - 4 - ip));
ip = iend - 4;
}
ip = iend - 4;
}
bitStream = FSE_readLE32(ip) >> (bitCount & 31);
}
}
@ -967,20 +967,20 @@ void FSE_freeCTable (FSE_CTable* ct)
/* provides the minimum logSize to safely represent a distribution */
static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
{
U32 minBitsSrc = FSE_highbit32((U32)(srcSize - 1)) + 1;
U32 minBitsSymbols = FSE_highbit32(maxSymbolValue) + 2;
U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
return minBits;
U32 minBitsSrc = FSE_highbit32((U32)(srcSize - 1)) + 1;
U32 minBitsSymbols = FSE_highbit32(maxSymbolValue) + 2;
U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
return minBits;
}
unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
{
U32 maxBitsSrc = FSE_highbit32((U32)(srcSize - 1)) - 2;
U32 maxBitsSrc = FSE_highbit32((U32)(srcSize - 1)) - 2;
U32 tableLog = maxTableLog;
U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */
if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */
if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */
if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */
if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG;
if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG;
return tableLog;
@ -1569,8 +1569,8 @@ size_t FSE_readBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >=
unsigned FSE_reloadDStream(FSE_DStream_t* bitD)
{
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
return FSE_DStream_tooFar;
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
return FSE_DStream_tooFar;
if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer))
{
@ -1834,7 +1834,7 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* tree, U32
if (maxSymbolValue > (241-128)) return (size_t)-FSE_ERROR_GENERIC; /* not implemented (not possible with current format) */
if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* not enough space within dst buffer */
op[0] = (BYTE)(128 /*special case*/ + 0 /* Not Compressible */ + (maxSymbolValue-1));
huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */
huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */
for (n=0; n<maxSymbolValue; n+=2)
op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
return ((maxSymbolValue+1)/2) + 1;
@ -1878,7 +1878,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
U32 rankLast[HUF_MAX_TABLELOG];
U32 currentNbBits = maxNbBits;
int pos;
memset(rankLast, 0xF0, sizeof(rankLast));
memset(rankLast, 0xF0, sizeof(rankLast));
for (pos=n ; pos >= 0; pos--)
{
if (huffNode[pos].nbBits >= currentNbBits) continue;
@ -1917,20 +1917,20 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
}
}
while (totalCost < 0) /* Sometimes, cost correction overshoot */
{
if (rankLast[1] == noOne) /* special case, no weight 1, let's find it back at n */
{
while (huffNode[n].nbBits == maxNbBits) n--;
huffNode[n+1].nbBits--;
rankLast[1] = n+1;
totalCost++;
continue;
}
huffNode[ rankLast[1] + 1 ].nbBits--;
rankLast[1]++;
totalCost ++;
}
while (totalCost < 0) /* Sometimes, cost correction overshoot */
{
if (rankLast[1] == noOne) /* special case, no weight 1, let's find it back at n */
{
while (huffNode[n].nbBits == maxNbBits) n--;
huffNode[n+1].nbBits--;
rankLast[1] = n+1;
totalCost++;
continue;
}
huffNode[ rankLast[1] + 1 ].nbBits--;
rankLast[1]++;
totalCost ++;
}
}
}
@ -1981,7 +1981,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
/* safety checks */
if (maxNbBits == 0) maxNbBits = HUF_DEFAULT_TABLELOG;
if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE) return (size_t)-FSE_ERROR_GENERIC;
memset(huffNode0, 0, sizeof(huffNode0));
memset(huffNode0, 0, sizeof(huffNode0));
// sort, decreasing order
HUF_sort(huffNode, count, maxSymbolValue);
@ -2066,7 +2066,7 @@ size_t HUF_compress_usingCTable(void* dst, size_t dstSize, const void* src, size
FSE_CStream_t bitC;
/* init */
if (dstSize < 8) return 0;
if (dstSize < 8) return 0;
op += 6; /* jump Table -- could be optimized by delta / deviation */
errorCode = FSE_initCStream(&bitC, op, oend-op);
if (FSE_isError(errorCode)) return 0;

103
lib/zstd.c
View File

@ -542,11 +542,11 @@ static size_t ZSTD_compressLiterals (void* dst, size_t dstSize,
const size_t minGain = ZSTD_minGain(srcSize);
BYTE* const ostart = (BYTE*)dst;
size_t hsize;
static const size_t LHSIZE = 5;
static const size_t LHSIZE = 5;
if (dstSize < LHSIZE+1) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* not enough space for compression */
if (dstSize < LHSIZE+1) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* not enough space for compression */
hsize = HUF_compress(ostart+LHSIZE, dstSize-LHSIZE, src, srcSize);
hsize = HUF_compress(ostart+LHSIZE, dstSize-LHSIZE, src, srcSize);
if (hsize<2) return hsize; /* special cases */
if (hsize >= srcSize - minGain) return 0;
@ -619,8 +619,8 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
}
/* Sequences Header */
if ((oend-op) < 2+3+6) /* nbSeq + dumpsLength + 3*rleCTable*/
return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
if ((oend-op) < 2+3+6) /* nbSeq + dumpsLength + 3*rleCTable*/
return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
ZSTD_writeLE16(op, (U16)nbSeq); op+=2;
seqHead = op;
@ -640,7 +640,7 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
op[2] = (BYTE)(dumpsLength);
op += 3;
}
if ((size_t)(oend-op) < dumpsLength+6) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
if ((size_t)(oend-op) < dumpsLength+6) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
memcpy(op, seqStorePtr->dumpsStart, dumpsLength);
op += dumpsLength;
}
@ -661,11 +661,11 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
}
else
{
size_t NCountSize;
size_t NCountSize;
tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max);
FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
op += NCountSize;
FSE_buildCTable(CTable_LitLength, norm, max, tableLog);
LLtype = bt_compressed;
@ -696,11 +696,11 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
}
else
{
size_t NCountSize;
size_t NCountSize;
tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max);
FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
op += NCountSize;
FSE_buildCTable(CTable_OffsetBits, norm, max, tableLog);
Offtype = bt_compressed;
@ -722,11 +722,11 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
}
else
{
size_t NCountSize;
size_t NCountSize;
tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max);
FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
op += NCountSize;
FSE_buildCTable(CTable_MatchLength, norm, max, tableLog);
MLtype = bt_compressed;
@ -1028,7 +1028,7 @@ size_t ZSTD_compressContinue(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize, con
size_t blockSize = BLOCKSIZE;
if (blockSize > srcSize) blockSize = srcSize;
if (maxDstSize < 2*ZSTD_blockHeaderSize+1) /* one RLE block + endMark */
if (maxDstSize < 2*ZSTD_blockHeaderSize+1) /* one RLE block + endMark */
return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
/* update hash table */
@ -1250,8 +1250,8 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
U32 LLlog, Offlog, MLlog;
size_t dumpsLength;
/* check */
if (srcSize < 5) return (size_t)-ZSTD_ERROR_SrcSize;
/* check */
if (srcSize < 5) return (size_t)-ZSTD_ERROR_SrcSize;
/* SeqHead */
*nbSeq = ZSTD_readLE16(ip); ip+=2;
@ -1273,8 +1273,8 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
*dumpsPtr = ip;
ip += dumpsLength;
/* check */
if (ip > iend-3) return (size_t)-ZSTD_ERROR_SrcSize; /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
/* check */
if (ip > iend-3) return (size_t)-ZSTD_ERROR_SrcSize; /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
/* sequences */
{
@ -1295,7 +1295,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
max = MaxLL;
headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
if (LLlog > LLFSELog) return (size_t)-ZSTD_ERROR_corruption;
if (LLlog > LLFSELog) return (size_t)-ZSTD_ERROR_corruption;
ip += headerSize;
FSE_buildDTable(DTableLL, norm, max, LLlog);
}
@ -1314,7 +1314,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
max = MaxOff;
headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
if (Offlog > OffFSELog) return (size_t)-ZSTD_ERROR_corruption;
if (Offlog > OffFSELog) return (size_t)-ZSTD_ERROR_corruption;
ip += headerSize;
FSE_buildDTable(DTableOffb, norm, max, Offlog);
}
@ -1333,7 +1333,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr,
max = MaxML;
headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
if (MLlog > MLFSELog) return (size_t)-ZSTD_ERROR_corruption;
if (MLlog > MLFSELog) return (size_t)-ZSTD_ERROR_corruption;
ip += headerSize;
FSE_buildDTable(DTableML, norm, max, MLlog);
}
@ -1417,9 +1417,9 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
static size_t ZSTD_execSequence(BYTE* op,
seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit,
BYTE* const base, BYTE* const oend)
seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit,
BYTE* const base, BYTE* const oend)
{
static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */
static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */
@ -1430,7 +1430,7 @@ static size_t ZSTD_execSequence(BYTE* op,
/* check */
if (endMatch > oend) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* overwrite beyond dst buffer */
if (litEnd > litLimit) return (size_t)-ZSTD_ERROR_corruption;
if (litEnd > litLimit) return (size_t)-ZSTD_ERROR_corruption;
if (sequence.matchLength > (size_t)(*litPtr-op)) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* overwrite literal segment */
/* copy Literals */
@ -1442,18 +1442,18 @@ static size_t ZSTD_execSequence(BYTE* op,
*litPtr = litEnd; /* update for next sequence */
/* check : last match must be at a minimum distance of 8 from end of dest buffer */
if (oend-op < 8) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
if (oend-op < 8) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
/* copy Match */
/* copy Match */
{
const U32 overlapRisk = (((size_t)(litEnd - endMatch)) < 12);
const BYTE* match = op - sequence.offset; /* possible underflow at op - offset ? */
size_t qutt = 12;
U64 saved[2];
/* check */
if (match < base) return (size_t)-ZSTD_ERROR_corruption;
if (sequence.offset > (size_t)base) return (size_t)-ZSTD_ERROR_corruption;
/* check */
if (match < base) return (size_t)-ZSTD_ERROR_corruption;
if (sequence.offset > (size_t)base) return (size_t)-ZSTD_ERROR_corruption;
/* save beginning of literal sequence, in case of write overlap */
if (overlapRisk)
@ -1473,7 +1473,7 @@ static size_t ZSTD_execSequence(BYTE* op,
ZSTD_copy4(op+4, match);
match -= dec64;
} else { ZSTD_copy8(op, match); }
op += 8; match += 8;
op += 8; match += 8;
if (endMatch > oend-12)
{
@ -1498,11 +1498,11 @@ static size_t ZSTD_execSequence(BYTE* op,
typedef struct ZSTD_Dctx_s
{
U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
void* previousDstEnd;
void* base;
size_t expected;
size_t expected;
blockType_t bType;
U32 phase;
} dctx_t;
@ -1514,7 +1514,7 @@ static size_t ZSTD_decompressSequences(
const void* seqStart, size_t seqSize,
const BYTE* litStart, size_t litSize)
{
dctx_t* dctx = (dctx_t*)ctx;
dctx_t* dctx = (dctx_t*)ctx;
const BYTE* ip = (const BYTE*)seqStart;
const BYTE* const iend = ip + seqSize;
BYTE* const ostart = (BYTE* const)dst;
@ -1526,9 +1526,9 @@ static size_t ZSTD_decompressSequences(
int nbSeq;
const BYTE* dumps;
U32* DTableLL = dctx->LLTable;
U32* DTableML = dctx->MLTable;
U32* DTableML = dctx->MLTable;
U32* DTableOffb = dctx->OffTable;
BYTE* const base = (BYTE*) (dctx->base);
BYTE* const base = (BYTE*) (dctx->base);
/* Build Decoding Tables */
errorCode = ZSTD_decodeSeqHeaders(&nbSeq, &dumps,
@ -1658,8 +1658,8 @@ static size_t ZSTD_decompressDCtx(void* ctx, void* dst, size_t maxDstSize, const
size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
dctx_t ctx;
ctx.base = dst;
dctx_t ctx;
ctx.base = dst;
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
}
@ -1668,14 +1668,20 @@ size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t src
* Streaming Decompression API
*******************************/
size_t ZSTD_resetDCtx(ZSTD_Dctx* dctx)
{
dctx->expected = ZSTD_frameHeaderSize;
dctx->phase = 0;
dctx->previousDstEnd = NULL;
dctx->base = NULL;
return 0;
}
ZSTD_Dctx* ZSTD_createDCtx(void)
{
ZSTD_Dctx* dctx = (ZSTD_Dctx*)malloc(sizeof(ZSTD_Dctx));
if (dctx==NULL) return NULL;
dctx->expected = ZSTD_frameHeaderSize;
dctx->phase = 0;
dctx->previousDstEnd = NULL;
dctx->base = NULL;
ZSTD_resetDCtx(dctx);
return dctx;
}
@ -1685,7 +1691,6 @@ size_t ZSTD_freeDCtx(ZSTD_Dctx* dctx)
return 0;
}
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_Dctx* dctx)
{
return ((dctx_t*)dctx)->expected;
@ -1697,8 +1702,8 @@ size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, co
/* Sanity check */
if (srcSize != ctx->expected) return (size_t)-ZSTD_ERROR_SrcSize;
if (dst != ctx->previousDstEnd) /* not contiguous */
ctx->base = dst;
if (dst != ctx->previousDstEnd) /* not contiguous */
ctx->base = dst;
/* Decompress : frame header */
if (ctx->phase == 0)
@ -1754,7 +1759,7 @@ size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, co
}
ctx->phase = 1;
ctx->expected = ZSTD_blockHeaderSize;
ctx->previousDstEnd = (void*)( ((char*)dst) + rSize);
ctx->previousDstEnd = (void*)( ((char*)dst) + rSize);
return rSize;
}

2
lib/zstd.h
View File

@ -47,7 +47,7 @@ extern "C" {
**************************************/
#define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */
#define ZSTD_VERSION_MINOR 1 /* for new (non-breaking) interface capabilities */
#define ZSTD_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */
#define ZSTD_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
unsigned ZSTD_versionNumber (void);

1
lib/zstd_static.h
View File

@ -56,6 +56,7 @@ size_t ZSTD_compressEnd(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize);
typedef struct ZSTD_Dctx_s ZSTD_Dctx;
ZSTD_Dctx* ZSTD_createDCtx(void);
size_t ZSTD_resetDCtx(ZSTD_Dctx* dctx);
size_t ZSTD_freeDCtx(ZSTD_Dctx* dctx);
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_Dctx* dctx);

27
programs/Makefile
View File

@ -30,7 +30,7 @@
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
# ##########################################################################
VERSION?= v0.1.1
VERSION?= v0.1.2
DESTDIR?=
PREFIX ?= /usr/local
@ -100,17 +100,17 @@ install: zstd
@ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/unzstd
@echo Installing man pages
@install -m 644 zstd.1 $(DESTDIR)$(MANDIR)/zstd.1
@install -m 644 zstdcat.1 $(DESTDIR)$(MANDIR)/zstdcat.1
@install -m 644 unzstd.1 $(DESTDIR)$(MANDIR)/unzstd.1
@ln -sf zstd.1 $(DESTDIR)$(MANDIR)/zstdcat.1
@ln -sf zstd.1 $(DESTDIR)$(MANDIR)/unzstd.1
@echo zstd installation completed
uninstall:
rm -f $(DESTDIR)$(BINDIR)/zstdcat
rm -f $(DESTDIR)$(BINDIR)/unzstd
[ -x $(DESTDIR)$(BINDIR)/zstd$(EXT) ] && rm -f $(DESTDIR)$(BINDIR)/zstd$(EXT)
rm -f $(DESTDIR)$(MANDIR)/zstdcat.1
rm -f $(DESTDIR)$(MANDIR)/unzstd.1
[ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstd.1
[ -f $(DESTDIR)$(MANDIR)/zstdcat.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstdcat.1
[ -f $(DESTDIR)$(MANDIR)/unzstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/unzstd.1
@echo zstd programs successfully uninstalled
test: test-zstd test-fullbench test-fuzzer
@ -120,9 +120,22 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32
test-all: test test32 memtest
test-zstd: zstd datagen
@echo "*** zstd cli write error test ***"
@echo "\n**** frame concatenation **** "
@echo "hello " > hello.tmp
@echo "world!" > world.tmp
@cat hello.tmp world.tmp > helloworld.tmp
./zstd hello.tmp > hello.zstd
./zstd world.tmp > world.zstd
@cat hello.zstd world.zstd > helloworld.zstd
./zstd -d helloworld.zstd > result.tmp
cat result.tmp
sdiff helloworld.tmp result.tmp
@rm *.tmp *.zstd
@echo frame concatenation test completed
@echo "**** flush write error test **** "
echo foo | ./zstd > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
@echo "*** zstd round-trip tests *** "
echo foo | ./zstd | ./zstd -d > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
@echo "**** zstd round-trip tests **** "
./datagen | ./zstd -v | ./zstd -d > $(VOID)
./datagen -g256MB | ./zstd -v | ./zstd -d > $(VOID)
./datagen -g6GB -P99 | ./zstd -vq | ./zstd -d > $(VOID)

104
programs/fileio.c
View File

@ -309,45 +309,18 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
#define MAXHEADERSIZE FIO_FRAMEHEADERSIZE+3
unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
BYTE* inBuff, size_t inBuffSize,
BYTE* outBuff, size_t outBuffSize,
ZSTD_Dctx* dctx)
{
FILE* finput, *foutput;
BYTE* inBuff;
size_t inBuffSize;
BYTE* outBuff, *op, *oend;
size_t outBuffSize;
U32 blockSize = 128 KB;
U32 wNbBlocks = 4;
BYTE* op = outBuff;
BYTE* const oend = outBuff + outBuffSize;
U64 filesize = 0;
BYTE* header[MAXHEADERSIZE];
ZSTD_Dctx* dctx;
size_t toRead;
size_t sizeCheck;
/* Init */
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
dctx = ZSTD_createDCtx();
/* check header */
toRead = ZSTD_nextSrcSizeToDecompress(dctx);
if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
sizeCheck = fread(header, (size_t)1, toRead, finput);
if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead); // Decode frame header
if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
/* Here later : blockSize determination */
/* Allocate Memory */
inBuffSize = blockSize + FIO_blockHeaderSize;
inBuff = (BYTE*)malloc(inBuffSize);
outBuffSize = wNbBlocks * blockSize;
outBuff = (BYTE*)malloc(outBuffSize);
op = outBuff;
oend = outBuff + outBuffSize;
if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
/* Main decompression Loop */
toRead = ZSTD_nextSrcSizeToDecompress(dctx);
while (toRead)
@ -380,15 +353,76 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
toRead = ZSTD_nextSrcSizeToDecompress(dctx);
}
return filesize;
}
unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
{
FILE* finput, *foutput;
BYTE* inBuff=NULL;
size_t inBuffSize = 0;
BYTE* outBuff=NULL;
size_t outBuffSize = 0;
U32 blockSize = 128 KB;
U32 wNbBlocks = 4;
U64 filesize = 0;
BYTE* header[MAXHEADERSIZE];
ZSTD_Dctx* dctx;
size_t toRead;
size_t sizeCheck;
/* Init */
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
dctx = ZSTD_createDCtx();
/* for each frame */
for ( ; ; )
{
/* check header */
ZSTD_resetDCtx(dctx);
toRead = ZSTD_nextSrcSizeToDecompress(dctx);
if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
sizeCheck = fread(header, (size_t)1, toRead, finput);
if (sizeCheck==0) break; /* no more input */
if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead); // Decode frame header
if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
/* Here later : blockSize determination */
/* Allocate Memory (if needed) */
{
size_t newInBuffSize = blockSize + FIO_blockHeaderSize;
size_t newOutBuffSize = wNbBlocks * blockSize;
if (newInBuffSize > inBuffSize)
{
free(inBuff);
inBuffSize = newInBuffSize;
inBuff = (BYTE*)malloc(inBuffSize);
}
if (newOutBuffSize > outBuffSize)
{
free(outBuff);
outBuffSize = newOutBuffSize;
outBuff = (BYTE*)malloc(outBuffSize);
}
}
if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, outBuff, outBuffSize, dctx);
}
DISPLAYLEVEL(2, "\r%79s\r", "");
DISPLAYLEVEL(2,"Decoded %llu bytes \n", (long long unsigned)filesize);
/* clean */
free(inBuff);
free(outBuff);
fclose(finput);
fclose(foutput);
ZSTD_freeDCtx(dctx);
fclose(finput);
if (fclose(foutput)) EXM_THROW(38, "Write error : cannot properly close %s", output_filename);
return filesize;
}

1
programs/unzstd.1
View File

@ -1 +0,0 @@
zstd.1

1
programs/zstdcat.1
View File

@ -1 +0,0 @@
zstd.1