Merge pull request #950 from facebook/srcSizeAdaptation

fix adaptation on srcSize

lib/compress/zstd_compress.c

@@ -144,6 +144,8 @@ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStor
 static ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
         ZSTD_CCtx_params CCtxParams, U64 srcSizeHint, size_t dictSize)
 {
+    DEBUGLOG(4, "ZSTD_getCParamsFromCCtxParams: srcSize = %u, dictSize = %u",
+                (U32)srcSizeHint, (U32)dictSize);
     return (CCtxParams.compressionLevel == ZSTD_CLEVEL_CUSTOM) ?
                 CCtxParams.cParams :
                 ZSTD_getCParams(CCtxParams.compressionLevel, srcSizeHint, dictSize);
@@ -151,18 +153,22 @@ static ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
 
 static void ZSTD_cLevelToCCtxParams_srcSize(ZSTD_CCtx_params* CCtxParams, U64 srcSize)
 {
+    DEBUGLOG(4, "ZSTD_cLevelToCCtxParams_srcSize: srcSize = %u",
+                (U32)srcSize);
     CCtxParams->cParams = ZSTD_getCParamsFromCCtxParams(*CCtxParams, srcSize, 0);
     CCtxParams->compressionLevel = ZSTD_CLEVEL_CUSTOM;
 }
 
 static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx)
 {
+    DEBUGLOG(4, "ZSTD_cLevelToCParams");
     ZSTD_cLevelToCCtxParams_srcSize(
             &cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1);
 }
 
 static void ZSTD_cLevelToCCtxParams(ZSTD_CCtx_params* CCtxParams)
 {
+    DEBUGLOG(4, "ZSTD_cLevelToCCtxParams");
     ZSTD_cLevelToCCtxParams_srcSize(CCtxParams, 0);
 }
 
@@ -261,7 +267,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
     case ZSTD_p_targetLength:
     case ZSTD_p_compressionStrategy:
         if (cctx->cdict) return ERROR(stage_wrong);
-        ZSTD_cLevelToCParams(cctx);  /* Can optimize if srcSize is known */
+        if (value>0) ZSTD_cLevelToCParams(cctx);  /* Can optimize if srcSize is known */
         return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
 
     case ZSTD_p_contentSizeFlag:
@@ -422,7 +428,7 @@ size_t ZSTD_CCtxParam_setParameter(
 #endif
 
     case ZSTD_p_enableLongDistanceMatching :
-        if (value != 0) {
+        if (value) {
             ZSTD_cLevelToCCtxParams(CCtxParams);
             CCtxParams->cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG;
         }
@@ -1736,6 +1742,8 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
             op += cSize;
             assert(dstCapacity >= cSize);
             dstCapacity -= cSize;
+            DEBUGLOG(5, "ZSTD_compress_frameChunk: adding a block of size %u",
+                        (U32)cSize);
     }   }
 
     if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending;
@@ -3070,6 +3078,8 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
     }
 #endif
 
+    DEBUGLOG(4, "ZSTD_getCParams: cLevel=%i, srcSize=%u, dictSize=%u => table %u",
+                compressionLevel, (U32)srcSizeHint, (U32)dictSize, tableID);
     if (compressionLevel <= 0) compressionLevel = ZSTD_CLEVEL_DEFAULT;   /* 0 == default; no negative compressionLevel yet */
     if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
     { ZSTD_compressionParameters const cp = ZSTD_defaultCParameters[tableID][compressionLevel];

programs/fileio.c

@@ -454,7 +454,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
         DISPLAYLEVEL(5,"set nb threads = %u \n", g_nbThreads);
         CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_p_nbThreads, g_nbThreads) );
         /* dictionary */
-        CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) );  /* just for dictionary loading, using good compression parameters */
+        CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) );  /* just for dictionary loading, for compression parameters adaptation */
         CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, dictBuffer, dictBuffSize) );
         CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, ZSTD_CONTENTSIZE_UNKNOWN) );  /* reset */
 
@@ -789,6 +789,8 @@ static int FIO_compressFilename_internal(cRess_t ress,
                         &outBuff, &inBuff, ZSTD_e_continue) );
 
             /* Write compressed stream */
+            DISPLAYLEVEL(6, "ZSTD_compress_generic,ZSTD_e_continue: generated %u bytes \n",
+                            (U32)outBuff.pos);
             if (outBuff.pos) {
                 size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
                 if (sizeCheck!=outBuff.pos)
@@ -824,6 +826,8 @@ static int FIO_compressFilename_internal(cRess_t ress,
                 EXM_THROW(26, "Compression error during frame end : %s",
                             ZSTD_getErrorName(result));
             }
+            DISPLAYLEVEL(6, "ZSTD_compress_generic,ZSTD_e_end: generated %u bytes \n",
+                            (U32)outBuff.pos);
             {   size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
                 if (sizeCheck != outBuff.pos)
                     EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName);