diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 75ba812b..31a5f85a 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1688,7 +1688,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, ldmBucketSize); ZSTD_memset(zc->ldmState.bucketOffsets, 0, ldmBucketSize); } - printf("Reserving space for seqs\n"); + DEBUGLOG(4, "Reserving space for seqs\n"); /* sequences storage */ ZSTD_referenceExternalSequences(zc, NULL, 0); zc->seqStore.maxNbSeq = maxNbSeq; @@ -2148,13 +2148,20 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, entropyWkspSize -= (MaxSeq + 1) * sizeof(*count); DEBUGLOG(4, "ZSTD_compressSequences_internal (nbSeq=%zu)", nbSeq); + DEBUGLOG(4, "First seqs:", nbSeq); + for (int i = 0; i < 5; ++i) { + DEBUGLOG(4, "(of: %u ml: %u ll: %u)", seqStorePtr->sequencesStart[i].offset, seqStorePtr->sequencesStart[i].matchLength, seqStorePtr->sequencesStart[i].litLength); + } + DEBUGLOG(4, "Final seqs:", nbSeq); + for (int i = 1; i < 5; ++i) { + DEBUGLOG(4, "(of: %u ml: %u ll: %u)", seqStorePtr->sequencesStart[nbSeq-i].offset, seqStorePtr->sequencesStart[nbSeq-i].matchLength, seqStorePtr->sequencesStart[nbSeq-i].litLength); + } ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<= HUF_WORKSPACE_SIZE); /* Compress literals */ { const BYTE* const literals = seqStorePtr->litStart; size_t const litSize = (size_t)(seqStorePtr->lit - literals); - DEBUGLOG(4, "litSize: %zu", litSize); size_t const cSize = ZSTD_compressLiterals( &prevEntropy->huf, &nextEntropy->huf, cctxParams->cParams.strategy, @@ -2167,7 +2174,6 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, assert(cSize <= dstCapacity); op += cSize; } - DEBUGLOG(4, "literals done (nbSeq=%zu)", nbSeq); /* Sequences Header */ RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, @@ -2193,7 +2199,6 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, /* seqHead : flags for FSE encoding type */ seqHead = op++; assert(op <= oend); - DEBUGLOG(4, "seqHead done (nbSeq=%zu)", nbSeq); /* convert length/distances into codes */ ZSTD_seqToCodes(seqStorePtr); @@ -2337,7 +2342,7 @@ ZSTD_compressSequences(seqStore_t* seqStorePtr, { size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy); if (cSize >= maxCSize) return 0; /* block not compressed */ } - printf("compressSequences: %u\n", cSize); + DEBUGLOG(4, "compressSequences: %u\n", cSize); return cSize; } @@ -2654,7 +2659,11 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, srcSize, zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, zc->bmi2); - printf("cSize compressed seqs: %u\n", cSize); + DEBUGLOG(4, "cSize compressed seqs: %u\n", cSize); + if (cSize == 1) { + printf("Csize was 1 for some reason\n"); + exit(1); + } if (zc->seqCollector.collectSequences) { ZSTD_copyBlockSequences(zc); @@ -2867,7 +2876,7 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) : lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); MEM_writeLE24(op, cBlockHeader); - printBits(ZSTD_blockHeaderSize, &cBlockHeader); + //printBits(ZSTD_blockHeaderSize, &cBlockHeader); DEBUGLOG(4, "Block header: %u", cBlockHeader); cSize += ZSTD_blockHeaderSize; } @@ -2903,13 +2912,13 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) ); size_t pos=0; - printf("ZSTD_writeFrameHeader: pledgedSrcSize: %u\n", pledgedSrcSize);; + DEBUGLOG(4, "ZSTD_writeFrameHeader: pledgedSrcSize: %u\n", pledgedSrcSize);; assert(!(params->fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)); RETURN_ERROR_IF(dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX, dstSize_tooSmall, "dst buf is too small to fit worst-case frame header size."); DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u", !params->fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode); - printf("dictIDSizeCodeLength: %u dictIDSizeCode: %u checksumFlag: %u, windowSize: %u singleSegment: %u windowLogByte: %u fcsCode: %u frameHeaderDescriptionByte: %u\n", + DEBUGLOG(4, "dictIDSizeCodeLength: %u dictIDSizeCode: %u checksumFlag: %u, windowSize: %u singleSegment: %u windowLogByte: %u fcsCode: %u frameHeaderDescriptionByte: %u\n", dictIDSizeCodeLength, dictIDSizeCode, checksumFlag, windowSize, singleSegment, windowLogByte, fcsCode, frameHeaderDescriptionByte); if (params->format == ZSTD_f_zstd1) { MEM_writeLE32(dst, ZSTD_MAGICNUMBER); @@ -3436,7 +3445,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) } if (cctx->stage != ZSTDcs_ending) { - printf("did this\n"); + DEBUGLOG(4, "did this\n"); /* write one last empty block, make it the "last" block */ U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1) + 0; RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for epilogue"); @@ -4534,7 +4543,7 @@ static void ZSTD_updateSequenceRange(ZSTD_sequenceRange* sequenceRange, size_t n sequenceRange->endIdx = idx; sequenceRange->endPosInSequence = endPosInSequence; - DEBUGLOG(4, "endidx: (of: %u ml: %u ll: %u", inSeqs[sequenceRange->endIdx].offset, inSeqs[sequenceRange->endIdx].matchLength, inSeqs[sequenceRange->endIdx].litLength); + DEBUGLOG(4, "endidx: (of: %u ml: %u ll: %u)", inSeqs[sequenceRange->endIdx].offset, inSeqs[sequenceRange->endIdx].matchLength, inSeqs[sequenceRange->endIdx].litLength); DEBUGLOG(4, "finished update: startidx %u startpos: %u endidx: %u endpos: %u", sequenceRange->startIdx, sequenceRange->startPosInSequence, sequenceRange->endIdx, sequenceRange->endPosInSequence); } @@ -4554,11 +4563,15 @@ static size_t ZSTD_copySequencesToSeqStore(ZSTD_CCtx* zc, const ZSTD_sequenceRan U32 litLength = inSeqs[idx].litLength; U32 matchLength = inSeqs[idx].matchLength; U32 offCode = inSeqs[idx].offset + ZSTD_REP_MOVE; - //DEBUGLOG(4, "Seqstore idx: %zu, seq: (ll: %u, ml: %u, of: %u)", idx, litLength, matchLength, offCode); + //DEBUGLOG(4, "Seqstore idx: %zu, seq: (ll: %u, ml: %u, of: %u), rep: %u", idx, litLength, matchLength, offCode, inSeqs[idx].rep); /* Adjust litLength and matchLength for the sequence at startIdx */ if (idx == seqRange->startIdx) { U32 posInSequence = seqRange->startPosInSequence; + if (posInSequence != 0) { + DEBUGLOG(4, "nope\n"); + exit(1); + } DEBUGLOG(4, "Reached startIdx. idx: %u PIS: %u", idx, posInSequence); assert(posInSequence <= litLength + matchLength); if (posInSequence >= litLength) { @@ -4578,7 +4591,19 @@ static size_t ZSTD_copySequencesToSeqStore(ZSTD_CCtx* zc, const ZSTD_sequenceRan /* Adjust litLength and matchLength for the sequence at endIdx */ if (idx == seqRange->endIdx) { U32 posInSequence = seqRange->endPosInSequence; + if (posInSequence != 0) { + DEBUGLOG(4, "nope\n"); + exit(1); + } DEBUGLOG(4, "Reached endIdx. idx: %u PIS: %u", idx, posInSequence); + if (posInSequence == 0) { + if (inSeqs[seqRange->endIdx - 1].matchLength != 0 || inSeqs[seqRange->endIdx - 1].matchLength != 0) { + printf("Contract violated\n"); + exit(1); + } + // this will be our new start idx + return 0; + } assert(posInSequence <= litLength + matchLength); if (posInSequence < litLength) { litLength = posInSequence; @@ -4601,9 +4626,10 @@ static size_t ZSTD_copySequencesToSeqStore(ZSTD_CCtx* zc, const ZSTD_sequenceRan const BYTE* const lastLiterals = (const BYTE*)src + srcSize - litLength; ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, litLength); } - break; + continue; } + //DEBUGLOG(4, "Storing in actual seqStore idx: %zu, seq: (ll: %u, ml: %u, of: %u), rep: %u", idx, litLength, matchLength - MINMATCH, offCode, inSeqs[idx].rep); RETURN_ERROR_IF(matchLength < MINMATCH, corruption_detected, "Matchlength too small! of: %u ml: %u ll: %u", offCode, matchLength, litLength); if (inSeqs[idx].rep) { ZSTD_storeSeq(&zc->seqStore, litLength, ip, iend, inSeqs[idx].rep - 1, matchLength - MINMATCH); @@ -4628,6 +4654,9 @@ size_t ZSTD_compressSequences_ext_internal(void* dst, size_t dstCapacity, size_t remaining = srcSize; ZSTD_sequenceRange seqRange = {0, 0, 0, 0}; seqStore_t baseSeqStore = cctx->seqStore; + baseSeqStore.longLengthID = 0; + baseSeqStore.longLengthPos = 0; + size_t origDstCapacity = dstCapacity; DEBUGLOG(4, "ZSTD_compressSequences_ext_internal srcSize: %u, inSeqsSize: %u", srcSize, inSeqsSize); @@ -4669,11 +4698,13 @@ size_t ZSTD_compressSequences_ext_internal(void* dst, size_t dstCapacity, if (ZSTD_isError(compressedSeqsSize)) { DEBUGLOG(4, "ERROR"); + exit(1); return compressedSeqsSize; } else if (compressedSeqsSize == 0) { DEBUGLOG(4, "NO compress BLOCK"); /* ZSTD_noCompressBlock writes block header as well */ - cSize += ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); + cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); + cSize += cBlockSize; } else { cSize += compressedSeqsSize; /* Error checking */ @@ -4684,12 +4715,17 @@ size_t ZSTD_compressSequences_ext_internal(void* dst, size_t dstCapacity, U32 const cBlockHeader = compressedSeqsSize == 1 ? lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3): lastBlock + (((U32)bt_compressed)<<1) + (U32)(compressedSeqsSize << 3); - printBits(ZSTD_blockHeaderSize, &cBlockHeader); + //printBits(ZSTD_blockHeaderSize, &cBlockHeader); MEM_writeLE24(op, cBlockHeader); cSize += ZSTD_blockHeaderSize; DEBUGLOG(4, "Block header: %u", cBlockHeader); + /*DEBUGLOG(3, "block header type: %u", ((cBlockHeader >> 1) & 3)); + if (((cBlockHeader >> 1) & 3) != 2) { + exit(1); + }*/ DEBUGLOG(4, "typical block, size: %u", compressedSeqsSize + ZSTD_blockHeaderSize); cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize; + //if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) //cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; @@ -4749,7 +4785,7 @@ size_t ZSTD_compressSequences_ext(void* dst, size_t dstCapacity, ZSTDb_buffered) , ""); assert(cctx->appliedParams.nbWorkers == 0); } - printf("blocksize: %u\n", cctx->blockSize); + DEBUGLOG(4, "blocksize: %u\n", cctx->blockSize); if (dstCapacity < ZSTD_compressBound(srcSize)) RETURN_ERROR(dstSize_tooSmall, "Destination buffer too small!"); DEBUGLOG(4, "SeqStore: maxNbSeq: %u, maxNbLits: %u", cctx->seqStore.maxNbSeq, cctx->seqStore.maxNbLit); diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 0a1da925..33e85465 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -433,6 +433,7 @@ MEM_STATIC size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const voi MEM_writeLE24(dst, cBlockHeader24); ZSTD_memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize); DEBUGLOG(4, "nocompress: %u", ZSTD_blockHeaderSize + srcSize); + DEBUGLOG(4, "nocompress header: %u", cBlockHeader24); return ZSTD_blockHeaderSize + srcSize; } diff --git a/lib/compress/zstd_compress_literals.c b/lib/compress/zstd_compress_literals.c index 4a633ea3..6dd1c144 100644 --- a/lib/compress/zstd_compress_literals.c +++ b/lib/compress/zstd_compress_literals.c @@ -79,7 +79,6 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); BYTE* const ostart = (BYTE*)dst; U32 singleStream = srcSize < 256; - DEBUGLOG(4, "singleStream: %u", singleStream); symbolEncodingType_e hType = set_compressed; size_t cLitSize; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 09fd0ecb..fc8772a5 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -645,8 +645,13 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, while (1) { size_t decodedSize; blockProperties_t blockProperties; + printf("Getting blocksize\n"); size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties); - if (ZSTD_isError(cBlockSize)) return cBlockSize; + printf("BlockSize: %u\n", cBlockSize); + if (ZSTD_isError(cBlockSize)) { + printf("Errored\n"); + return cBlockSize; + } ip += ZSTD_blockHeaderSize; remainingSrcSize -= ZSTD_blockHeaderSize; diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c index 261a8f92..d4c7d383 100644 --- a/lib/decompress/zstd_decompress_block.c +++ b/lib/decompress/zstd_decompress_block.c @@ -46,6 +46,20 @@ **********************************************************/ static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); } +void printBits1(size_t const size, void const * const ptr) +{ + unsigned char *b = (unsigned char*) ptr; + unsigned char byte; + int i, j; + + for (i = size-1; i >= 0; i--) { + for (j = 7; j >= 0; j--) { + byte = (b[i] >> j) & 1; + printf("%u", byte); + } + printf("\n"); + } +} /*-************************************************************* * Block decoding @@ -64,7 +78,12 @@ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, bpPtr->lastBlock = cBlockHeader & 1; bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); bpPtr->origSize = cSize; /* only useful for RLE */ - if (bpPtr->blockType == bt_rle) return 1; + printBits1(3, &cBlockHeader); + if (bpPtr->blockType == bt_rle) { + printf("RLE BLOCK FOUND\n"); + exit(1); + return 1; + } RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, ""); return cSize; } @@ -1212,7 +1231,6 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, /* last literal segment */ { size_t const lastLLSize = litEnd - litPtr; - printf("Last LL: %u\n", lastLLSize); RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); if (op != NULL) { ZSTD_memcpy(op, litPtr, lastLLSize); @@ -1220,8 +1238,6 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, } } - printf("op - ostart: %u\n", (U32)(op-ostart)); - return op-ostart; }