AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix cSize calculation for noCompressBlocks

Browse Source
This commit is contained in:
senhuang42 2020-10-30 11:55:20 -04:00
parent 6145ebb400
commit e5fe485dcc
5 changed files with 79 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
lib/compress/zstd_compress.c
View File

@ -1688,7 +1688,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, ldmBucketSize);
ZSTD_memset(zc->ldmState.bucketOffsets, 0, ldmBucketSize);
}
printf("Reserving space for seqs\n");
DEBUGLOG(4, "Reserving space for seqs\n");
/* sequences storage */
ZSTD_referenceExternalSequences(zc, NULL, 0);
zc->seqStore.maxNbSeq = maxNbSeq;
@ -2148,13 +2148,20 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
entropyWkspSize -= (MaxSeq + 1) * sizeof(*count);
DEBUGLOG(4, "ZSTD_compressSequences_internal (nbSeq=%zu)", nbSeq);
DEBUGLOG(4, "First seqs:", nbSeq);
for (int i = 0; i < 5; ++i) {
DEBUGLOG(4, "(of: %u ml: %u ll: %u)", seqStorePtr->sequencesStart[i].offset, seqStorePtr->sequencesStart[i].matchLength, seqStorePtr->sequencesStart[i].litLength);
}
DEBUGLOG(4, "Final seqs:", nbSeq);
for (int i = 1; i < 5; ++i) {
DEBUGLOG(4, "(of: %u ml: %u ll: %u)", seqStorePtr->sequencesStart[nbSeq-i].offset, seqStorePtr->sequencesStart[nbSeq-i].matchLength, seqStorePtr->sequencesStart[nbSeq-i].litLength);
}
ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog)));
assert(entropyWkspSize >= HUF_WORKSPACE_SIZE);
/* Compress literals */
{ const BYTE* const literals = seqStorePtr->litStart;
size_t const litSize = (size_t)(seqStorePtr->lit - literals);
DEBUGLOG(4, "litSize: %zu", litSize);
size_t const cSize = ZSTD_compressLiterals(
&prevEntropy->huf, &nextEntropy->huf,
cctxParams->cParams.strategy,
@ -2167,7 +2174,6 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
assert(cSize <= dstCapacity);
op += cSize;
}
DEBUGLOG(4, "literals done (nbSeq=%zu)", nbSeq);
/* Sequences Header */
RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/,
@ -2193,7 +2199,6 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
/* seqHead : flags for FSE encoding type */
seqHead = op++;
assert(op <= oend);
DEBUGLOG(4, "seqHead done (nbSeq=%zu)", nbSeq);
/* convert length/distances into codes */
ZSTD_seqToCodes(seqStorePtr);
@ -2337,7 +2342,7 @@ ZSTD_compressSequences(seqStore_t* seqStorePtr,
{ size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy);
if (cSize >= maxCSize) return 0; /* block not compressed */
}
printf("compressSequences: %u\n", cSize);
DEBUGLOG(4, "compressSequences: %u\n", cSize);
return cSize;
}
@ -2654,7 +2659,11 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
srcSize,
zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */,
zc->bmi2);
printf("cSize compressed seqs: %u\n", cSize);
DEBUGLOG(4, "cSize compressed seqs: %u\n", cSize);
if (cSize == 1) {
printf("Csize was 1 for some reason\n");
exit(1);
}
if (zc->seqCollector.collectSequences) {
ZSTD_copyBlockSequences(zc);
@ -2867,7 +2876,7 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) :
lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
MEM_writeLE24(op, cBlockHeader);
printBits(ZSTD_blockHeaderSize, &cBlockHeader);
//printBits(ZSTD_blockHeaderSize, &cBlockHeader);
DEBUGLOG(4, "Block header: %u", cBlockHeader);
cSize += ZSTD_blockHeaderSize;
}
@ -2903,13 +2912,13 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) );
size_t pos=0;
printf("ZSTD_writeFrameHeader: pledgedSrcSize: %u\n", pledgedSrcSize);;
DEBUGLOG(4, "ZSTD_writeFrameHeader: pledgedSrcSize: %u\n", pledgedSrcSize);;
assert(!(params->fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN));
RETURN_ERROR_IF(dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX, dstSize_tooSmall,
"dst buf is too small to fit worst-case frame header size.");
DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u",
!params->fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode);
printf("dictIDSizeCodeLength: %u dictIDSizeCode: %u checksumFlag: %u, windowSize: %u singleSegment: %u windowLogByte: %u fcsCode: %u frameHeaderDescriptionByte: %u\n",
DEBUGLOG(4, "dictIDSizeCodeLength: %u dictIDSizeCode: %u checksumFlag: %u, windowSize: %u singleSegment: %u windowLogByte: %u fcsCode: %u frameHeaderDescriptionByte: %u\n",
dictIDSizeCodeLength, dictIDSizeCode, checksumFlag, windowSize, singleSegment, windowLogByte, fcsCode, frameHeaderDescriptionByte);
if (params->format == ZSTD_f_zstd1) {
MEM_writeLE32(dst, ZSTD_MAGICNUMBER);
@ -3436,7 +3445,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
}
if (cctx->stage != ZSTDcs_ending) {
printf("did this\n");
DEBUGLOG(4, "did this\n");
/* write one last empty block, make it the "last" block */
U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1) + 0;
RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for epilogue");
@ -4534,7 +4543,7 @@ static void ZSTD_updateSequenceRange(ZSTD_sequenceRange* sequenceRange, size_t n
sequenceRange->endIdx = idx;
sequenceRange->endPosInSequence = endPosInSequence;
DEBUGLOG(4, "endidx: (of: %u ml: %u ll: %u", inSeqs[sequenceRange->endIdx].offset, inSeqs[sequenceRange->endIdx].matchLength, inSeqs[sequenceRange->endIdx].litLength);
DEBUGLOG(4, "endidx: (of: %u ml: %u ll: %u)", inSeqs[sequenceRange->endIdx].offset, inSeqs[sequenceRange->endIdx].matchLength, inSeqs[sequenceRange->endIdx].litLength);
DEBUGLOG(4, "finished update: startidx %u startpos: %u endidx: %u endpos: %u",
sequenceRange->startIdx, sequenceRange->startPosInSequence, sequenceRange->endIdx, sequenceRange->endPosInSequence);
}
@ -4554,11 +4563,15 @@ static size_t ZSTD_copySequencesToSeqStore(ZSTD_CCtx* zc, const ZSTD_sequenceRan
U32 litLength = inSeqs[idx].litLength;
U32 matchLength = inSeqs[idx].matchLength;
U32 offCode = inSeqs[idx].offset + ZSTD_REP_MOVE;
//DEBUGLOG(4, "Seqstore idx: %zu, seq: (ll: %u, ml: %u, of: %u)", idx, litLength, matchLength, offCode);
//DEBUGLOG(4, "Seqstore idx: %zu, seq: (ll: %u, ml: %u, of: %u), rep: %u", idx, litLength, matchLength, offCode, inSeqs[idx].rep);
/* Adjust litLength and matchLength for the sequence at startIdx */
if (idx == seqRange->startIdx) {
U32 posInSequence = seqRange->startPosInSequence;
if (posInSequence != 0) {
DEBUGLOG(4, "nope\n");
exit(1);
}
DEBUGLOG(4, "Reached startIdx. idx: %u PIS: %u", idx, posInSequence);
assert(posInSequence <= litLength + matchLength);
if (posInSequence >= litLength) {
@ -4578,7 +4591,19 @@ static size_t ZSTD_copySequencesToSeqStore(ZSTD_CCtx* zc, const ZSTD_sequenceRan
/* Adjust litLength and matchLength for the sequence at endIdx */
if (idx == seqRange->endIdx) {
U32 posInSequence = seqRange->endPosInSequence;
if (posInSequence != 0) {
DEBUGLOG(4, "nope\n");
exit(1);
}
DEBUGLOG(4, "Reached endIdx. idx: %u PIS: %u", idx, posInSequence);
if (posInSequence == 0) {
if (inSeqs[seqRange->endIdx - 1].matchLength != 0 || inSeqs[seqRange->endIdx - 1].matchLength != 0) {
printf("Contract violated\n");
exit(1);
}
// this will be our new start idx
return 0;
}
assert(posInSequence <= litLength + matchLength);
if (posInSequence < litLength) {
litLength = posInSequence;
@ -4601,9 +4626,10 @@ static size_t ZSTD_copySequencesToSeqStore(ZSTD_CCtx* zc, const ZSTD_sequenceRan
const BYTE* const lastLiterals = (const BYTE*)src + srcSize - litLength;
ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, litLength);
}
break;
continue;
}
//DEBUGLOG(4, "Storing in actual seqStore idx: %zu, seq: (ll: %u, ml: %u, of: %u), rep: %u", idx, litLength, matchLength - MINMATCH, offCode, inSeqs[idx].rep);
RETURN_ERROR_IF(matchLength < MINMATCH, corruption_detected, "Matchlength too small! of: %u ml: %u ll: %u", offCode, matchLength, litLength);
if (inSeqs[idx].rep) {
ZSTD_storeSeq(&zc->seqStore, litLength, ip, iend, inSeqs[idx].rep - 1, matchLength - MINMATCH);
@ -4628,6 +4654,9 @@ size_t ZSTD_compressSequences_ext_internal(void* dst, size_t dstCapacity,
size_t remaining = srcSize;
ZSTD_sequenceRange seqRange = {0, 0, 0, 0};
seqStore_t baseSeqStore = cctx->seqStore;
baseSeqStore.longLengthID = 0;
baseSeqStore.longLengthPos = 0;
size_t origDstCapacity = dstCapacity;
DEBUGLOG(4, "ZSTD_compressSequences_ext_internal srcSize: %u, inSeqsSize: %u", srcSize, inSeqsSize);
@ -4669,11 +4698,13 @@ size_t ZSTD_compressSequences_ext_internal(void* dst, size_t dstCapacity,
if (ZSTD_isError(compressedSeqsSize)) {
DEBUGLOG(4, "ERROR");
exit(1);
return compressedSeqsSize;
} else if (compressedSeqsSize == 0) {
DEBUGLOG(4, "NO compress BLOCK");
/* ZSTD_noCompressBlock writes block header as well */
cSize += ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
cSize += cBlockSize;
} else {
cSize += compressedSeqsSize;
/* Error checking */
@ -4684,12 +4715,17 @@ size_t ZSTD_compressSequences_ext_internal(void* dst, size_t dstCapacity,
U32 const cBlockHeader = compressedSeqsSize == 1 ?
lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3):
lastBlock + (((U32)bt_compressed)<<1) + (U32)(compressedSeqsSize << 3);
printBits(ZSTD_blockHeaderSize, &cBlockHeader);
//printBits(ZSTD_blockHeaderSize, &cBlockHeader);
MEM_writeLE24(op, cBlockHeader);
cSize += ZSTD_blockHeaderSize;
DEBUGLOG(4, "Block header: %u", cBlockHeader);
/*DEBUGLOG(3, "block header type: %u", ((cBlockHeader >> 1) & 3));
if (((cBlockHeader >> 1) & 3) != 2) {
exit(1);
}*/
DEBUGLOG(4, "typical block, size: %u", compressedSeqsSize + ZSTD_blockHeaderSize);
cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize;
//if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
//cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
@ -4749,7 +4785,7 @@ size_t ZSTD_compressSequences_ext(void* dst, size_t dstCapacity,
ZSTDb_buffered) , "");
assert(cctx->appliedParams.nbWorkers == 0);
}
printf("blocksize: %u\n", cctx->blockSize);
DEBUGLOG(4, "blocksize: %u\n", cctx->blockSize);
if (dstCapacity < ZSTD_compressBound(srcSize))
RETURN_ERROR(dstSize_tooSmall, "Destination buffer too small!");
DEBUGLOG(4, "SeqStore: maxNbSeq: %u, maxNbLits: %u", cctx->seqStore.maxNbSeq, cctx->seqStore.maxNbLit);

1
lib/compress/zstd_compress_internal.h
View File

@ -433,6 +433,7 @@ MEM_STATIC size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const voi
MEM_writeLE24(dst, cBlockHeader24);
ZSTD_memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize);
DEBUGLOG(4, "nocompress: %u", ZSTD_blockHeaderSize + srcSize);
DEBUGLOG(4, "nocompress header: %u", cBlockHeader24);
return ZSTD_blockHeaderSize + srcSize;
}

1
lib/compress/zstd_compress_literals.c
View File

@ -79,7 +79,6 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
BYTE* const ostart = (BYTE*)dst;
U32 singleStream = srcSize < 256;
DEBUGLOG(4, "singleStream: %u", singleStream);
symbolEncodingType_e hType = set_compressed;
size_t cLitSize;

7
lib/decompress/zstd_decompress.c
View File

@ -645,8 +645,13 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
while (1) {
size_t decodedSize;
blockProperties_t blockProperties;
printf("Getting blocksize\n");
size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties);
if (ZSTD_isError(cBlockSize)) return cBlockSize;
printf("BlockSize: %u\n", cBlockSize);
if (ZSTD_isError(cBlockSize)) {
printf("Errored\n");
return cBlockSize;
}
ip += ZSTD_blockHeaderSize;
remainingSrcSize -= ZSTD_blockHeaderSize;

24
lib/decompress/zstd_decompress_block.c
View File

@ -46,6 +46,20 @@
**********************************************************/
static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); }
void printBits1(size_t const size, void const * const ptr)
{
unsigned char *b = (unsigned char*) ptr;
unsigned char byte;
int i, j;
for (i = size-1; i >= 0; i--) {
for (j = 7; j >= 0; j--) {
byte = (b[i] >> j) & 1;
printf("%u", byte);
}
printf("\n");
}
}
/*-*************************************************************
* Block decoding
@ -64,7 +78,12 @@ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
bpPtr->lastBlock = cBlockHeader & 1;
bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3);
bpPtr->origSize = cSize; /* only useful for RLE */
if (bpPtr->blockType == bt_rle) return 1;
printBits1(3, &cBlockHeader);
if (bpPtr->blockType == bt_rle) {
printf("RLE BLOCK FOUND\n");
exit(1);
return 1;
}
RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, "");
return cSize;
}
@ -1212,7 +1231,6 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx,
/* last literal segment */
{ size_t const lastLLSize = litEnd - litPtr;
printf("Last LL: %u\n", lastLLSize);
RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
if (op != NULL) {
ZSTD_memcpy(op, litPtr, lastLLSize);
@ -1220,8 +1238,6 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx,
}
}
printf("op - ostart: %u\n", (U32)(op-ostart));
return op-ostart;
}