AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #921 from facebook/ubfix

Browse Source 
Fixes some arithmetic pointer operations
This commit is contained in:
Yann Collet 2017-11-17 17:11:18 -08:00 committed by GitHub
commit 9708c46879
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 35 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lib/common/mem.h
View File

@ -56,8 +56,6 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size
typedef int32_t S32;
typedef uint64_t U64;
typedef int64_t S64;
typedef intptr_t iPtrDiff;
typedef uintptr_t uPtrDiff;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
@ -66,8 +64,6 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size
typedef signed int S32;
typedef unsigned long long U64;
typedef signed long long S64;
typedef ptrdiff_t iPtrDiff;
typedef size_t uPtrDiff;
#endif

2
lib/common/zstd_internal.h
View File

@ -187,7 +187,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
/*! ZSTD_wildcopy() :
* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
* custom version of memcpy(), can overwrite up to WILDCOPY_OVERLENGTH bytes (if length==0) */
#define WILDCOPY_OVERLENGTH 8
MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
{

18
lib/compress/zstd_compress.c
View File

@ -1819,17 +1819,21 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
cctx->stage = ZSTDcs_ongoing;
}
if (!srcSize) return fhSize; /* do not generate an empty block if no input */
/* Check if blocks follow each other */
if (src != cctx->nextSrc) {
/* not contiguous */
ptrdiff_t const delta = cctx->nextSrc - ip;
size_t const distanceFromBase = (size_t)(cctx->nextSrc - cctx->base);
cctx->lowLimit = cctx->dictLimit;
cctx->dictLimit = (U32)(cctx->nextSrc - cctx->base);
assert(distanceFromBase == (size_t)(U32)distanceFromBase); /* should never overflow */
cctx->dictLimit = (U32)distanceFromBase;
cctx->dictBase = cctx->base;
cctx->base -= delta;
cctx->base = ip - distanceFromBase;
cctx->nextToUpdate = cctx->dictLimit;
if (cctx->dictLimit - cctx->lowLimit < HASH_READ_SIZE) cctx->lowLimit = cctx->dictLimit; /* too small extDict */
}
cctx->nextSrc = ip + srcSize;
/* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */
if ((ip+srcSize > cctx->dictBase + cctx->lowLimit) & (ip < cctx->dictBase + cctx->dictLimit)) {
@ -1838,17 +1842,13 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
cctx->lowLimit = lowLimitMax;
}
cctx->nextSrc = ip + srcSize;
if (srcSize) {
size_t const cSize = frame ?
{ size_t const cSize = frame ?
ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) :
ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize);
if (ZSTD_isError(cSize)) return cSize;
cctx->consumedSrcSize += srcSize;
return cSize + fhSize;
} else
return fhSize;
}
}
size_t ZSTD_compressContinue (ZSTD_CCtx* cctx,

3
lib/compress/zstd_opt.c
View File

@ -588,9 +588,8 @@ size_t ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
while (mlen <= lastML) {
U32 const pos = cur + mlen;
U32 const price = basePrice + ZSTD_getPrice(optStatePtr, litlen, baseLiterals, offset, mlen, ultra);
assert(pos < ZSTD_OPT_NUM);
if ((pos > last_pos) | (price < opt[pos].price)) {
if ((pos > last_pos) || (price < opt[pos].price)) {
DEBUGLOG(7, "rPos:%u => new better price (%u<%u)",
pos, price, opt[pos].price);
SET_PRICE(pos, mlen, offset, litlen, price, repHistory); /* note : macro modifies last_pos */

39
lib/decompress/zstd_decompress.c
View File

@ -827,9 +827,9 @@ typedef struct {
FSE_DState_t stateOffb;
FSE_DState_t stateML;
size_t prevOffset[ZSTD_REP_NUM];
const BYTE* base;
const BYTE* prefixStart;
const BYTE* dictEnd;
size_t pos;
uPtrDiff gotoDict;
} seqState_t;
@ -1224,8 +1224,9 @@ seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, ZSTD_longOffset_e const long
BIT_reloadDStream(&seqState->DStream);
{ size_t const pos = seqState->pos + seq.litLength;
seq.match = seqState->base + pos - seq.offset; /* single memory segment */
if (seq.offset > pos) seq.match += seqState->gotoDict; /* separate memory segment */
const BYTE* const matchBase = (seq.offset > pos) ? seqState->dictEnd : seqState->prefixStart;
seq.match = matchBase + pos - seq.offset; /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted.
* No consequence though : no memory access will occur, overly large offset will be detected in ZSTD_execSequenceLong() */
seqState->pos = pos + seq.matchLength;
}
@ -1243,7 +1244,7 @@ HINT_INLINE
size_t ZSTD_execSequenceLong(BYTE* op,
BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
const BYTE* const prefixStart, const BYTE* const dictStart, const BYTE* const dictEnd)
{
BYTE* const oLitEnd = op + sequence.litLength;
size_t const sequenceLength = sequence.litLength + sequence.matchLength;
@ -1253,21 +1254,21 @@ size_t ZSTD_execSequenceLong(BYTE* op,
const BYTE* match = sequence.match;
/* check */
if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */
if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd);
if (oLitEnd > oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, dictStart, dictEnd);
/* copy Literals */
ZSTD_copy8(op, *litPtr);
ZSTD_copy8(op, *litPtr); /* note : op <= oLitEnd <= oend_w == oend - 8 */
if (sequence.litLength > 8)
ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
op = oLitEnd;
*litPtr = iLitEnd; /* update for next sequence */
/* copy Match */
if (sequence.offset > (size_t)(oLitEnd - base)) {
if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
/* offset beyond prefix */
if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected);
if (sequence.offset > (size_t)(oLitEnd - dictStart)) return ERROR(corruption_detected);
if (match + sequence.matchLength <= dictEnd) {
memmove(oLitEnd, match, sequence.matchLength);
return sequenceLength;
@ -1277,7 +1278,7 @@ size_t ZSTD_execSequenceLong(BYTE* op,
memmove(oLitEnd, match, length1);
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
match = prefixStart;
if (op > oend_w || sequence.matchLength < MINMATCH) {
U32 i;
for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i];
@ -1331,8 +1332,8 @@ static size_t ZSTD_decompressSequencesLong(
BYTE* op = ostart;
const BYTE* litPtr = dctx->litPtr;
const BYTE* const litEnd = litPtr + dctx->litSize;
const BYTE* const base = (const BYTE*) (dctx->base);
const BYTE* const vBase = (const BYTE*) (dctx->vBase);
const BYTE* const prefixStart = (const BYTE*) (dctx->base);
const BYTE* const dictStart = (const BYTE*) (dctx->vBase);
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
int nbSeq;
@ -1353,9 +1354,9 @@ static size_t ZSTD_decompressSequencesLong(
int seqNb;
dctx->fseEntropy = 1;
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
seqState.base = base;
seqState.pos = (size_t)(op-base);
seqState.gotoDict = (uPtrDiff)dictEnd - (uPtrDiff)base; /* cast to avoid undefined behaviour */
seqState.prefixStart = prefixStart;
seqState.pos = (size_t)(op-prefixStart);
seqState.dictEnd = dictEnd;
CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected);
FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
@ -1370,9 +1371,9 @@ static size_t ZSTD_decompressSequencesLong(
/* decode and decompress */
for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && seqNb<nbSeq ; seqNb++) {
seq_t const sequence = ZSTD_decodeSequenceLong(&seqState, isLongOffset);
size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[(seqNb-ADVANCED_SEQS) & STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd);
size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[(seqNb-ADVANCED_SEQS) & STOSEQ_MASK], &litPtr, litEnd, prefixStart, dictStart, dictEnd);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
PREFETCH(sequence.match);
PREFETCH(sequence.match); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */
sequences[seqNb&STOSEQ_MASK] = sequence;
op += oneSeqSize;
}
@ -1381,7 +1382,7 @@ static size_t ZSTD_decompressSequencesLong(
/* finish queue */
seqNb -= seqAdvance;
for ( ; seqNb<nbSeq ; seqNb++) {
size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[seqNb&STOSEQ_MASK], &litPtr, litEnd, base, vBase, dictEnd);
size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[seqNb&STOSEQ_MASK], &litPtr, litEnd, prefixStart, dictStart, dictEnd);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
}

11
tests/Makefile
View File

@ -131,14 +131,11 @@ zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zbufftest.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
ZSTREAMFILES := $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c seqgen.c zstreamtest.c
zstreamtest : CPPFLAGS += $(MULTITHREAD_CPP)
zstreamtest : LDFLAGS += $(MULTITHREAD_LD)
zstreamtest : $(ZSTREAMFILES)
$(CC) $(FLAGS) $^ -o $@$(EXT)
zstreamtest32 : CFLAGS += -m32
zstreamtest32 : $(ZSTREAMFILES)
$(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
zstreamtest zstreamtest32 : CPPFLAGS += $(MULTITHREAD_CPP)
zstreamtest zstreamtest32 : LDFLAGS += $(MULTITHREAD_LD)
zstreamtest zstreamtest32 : $(ZSTREAMFILES)
$(CC) $(FLAGS) $^ -o $@$(EXT)
zstreamtest_asan : CFLAGS += -fsanitize=address
zstreamtest_asan : $(ZSTREAMFILES)