From d54566f334c03ec751c69c014ffaeea8a89ec15a Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Fri, 28 Aug 2020 11:01:04 -0400 Subject: [PATCH 01/12] First pass at refactoring and creating new FIO_ctx_t */ --- programs/fileio.c | 66 +++++++++++++++++++++++++++++++++++++--------- programs/fileio.h | 16 +++++++++-- programs/zstdcli.c | 7 ++--- 3 files changed, 72 insertions(+), 17 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 72f716f3..1ace731e 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -319,8 +319,20 @@ struct FIO_prefs_s { int excludeCompressedFiles; int patchFromMode; int contentSize; +}; + +/*-************************************* +* Parameters: FIO_prefs_t +***************************************/ + +/* typedef'd to FIO_ctx_t within fileio.h */ +struct FIO_ctx_s { + + /* multiple file processing info */ int currFileIdx; int nbFiles; + size_t totalBytesInput; + size_t totalBytesOutput; }; @@ -362,8 +374,18 @@ FIO_prefs_t* FIO_createPreferences(void) ret->testMode = 0; ret->literalCompressionMode = ZSTD_lcm_auto; ret->excludeCompressedFiles = 0; - ret->nbFiles = 1; + return ret; +} + +FIO_ctx_t* FIO_createContext(void) +{ + FIO_ctx_t* const ret = (FIO_ctx_t*)malloc(sizeof(FIO_ctx_t)); + if (!ret) EXM_THROW(21, "Allocation error : not enough memory"); + ret->currFileIdx = 0; + ret->nbFiles = 1; + ret->totalBytesInput = 0; + ret->totalBytesOutput = 0; return ret; } @@ -372,6 +394,11 @@ void FIO_freePreferences(FIO_prefs_t* const prefs) free(prefs); } +void FIO_freeContext(FIO_ctx_t* const fCtx) +{ + free(fCtx); +} + /*-************************************* * Parameters: Display Options @@ -386,6 +413,8 @@ void FIO_setNoProgress(unsigned noProgress) { g_display_prefs.noProgress = noPro * Parameters: Setters ***************************************/ +/* FIO_prefs_t functions */ + void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t compressionType) { prefs->compressionType = compressionType; } void FIO_overwriteMode(FIO_prefs_t* const prefs) { prefs->overwrite = 1; } @@ -499,14 +528,26 @@ void FIO_setContentSize(FIO_prefs_t* const prefs, int value) prefs->contentSize = value != 0; } -void FIO_setNbFiles(FIO_prefs_t* const prefs, int value) +/* FIO_ctx_t functions */ + +void FIO_setNbFiles(FIO_ctx_t* const fCtx, int value) { - prefs->nbFiles = value; + fCtx->nbFiles = value; } -void FIO_setCurrFileIdx(FIO_prefs_t* const prefs, int value) +void FIO_setCurrFileIdx(FIO_ctx_t* const fCtx, int value) { - prefs->currFileIdx = value; + fCtx->currFileIdx = value; +} + +void FIO_setTotalBytesInput(FIO_ctx_t* const fCtx, size_t value) +{ + fCtx->totalBytesInput = value; +} + +void FIO_setTotalBytesOutput(FIO_ctx_t* const fCtx, size_t value) +{ + fCtx->totalBytesOutput = value; } /*-************************************* @@ -1685,6 +1726,7 @@ static unsigned long long FIO_getLargestFileSize(const char** inFileNames, unsig * or into a destination folder (specified with -O) */ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, + FIO_ctx_t* const fCtx, const char** inFileNamesTable, const char* outMirroredRootDirName, const char* outDirName, @@ -1694,7 +1736,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, { int error = 0; cRess_t ress = FIO_createCResources(prefs, dictFileName, - FIO_getLargestFileSize(inFileNamesTable, prefs->nbFiles), + FIO_getLargestFileSize(inFileNamesTable, fCtx->nbFiles), compressionLevel, comprParams); /* init */ @@ -1704,8 +1746,8 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, if (ress.dstFile == NULL) { /* could not open outFileName */ error = 1; } else { - for (; prefs->currFileIdx < prefs->nbFiles; ++prefs->currFileIdx) { - error |= FIO_compressFilename_srcFile(prefs, ress, outFileName, inFileNamesTable[prefs->currFileIdx], compressionLevel); + for (; fCtx->currFileIdx < fCtx->nbFiles; ++fCtx->currFileIdx) { + error |= FIO_compressFilename_srcFile(prefs, ress, outFileName, inFileNamesTable[fCtx->currFileIdx], compressionLevel); } if (fclose(ress.dstFile)) EXM_THROW(29, "Write error (%s) : cannot properly close %s", @@ -1714,10 +1756,10 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, } } else { if (outMirroredRootDirName) - UTIL_mirrorSourceFilesDirectories(inFileNamesTable, prefs->nbFiles, outMirroredRootDirName); + UTIL_mirrorSourceFilesDirectories(inFileNamesTable, fCtx->nbFiles, outMirroredRootDirName); - for (; prefs->currFileIdx < prefs->nbFiles; ++prefs->currFileIdx) { - const char* const srcFileName = inFileNamesTable[prefs->currFileIdx]; + for (; fCtx->currFileIdx < fCtx->nbFiles; ++fCtx->currFileIdx) { + const char* const srcFileName = inFileNamesTable[fCtx->currFileIdx]; const char* dstFileName = NULL; if (outMirroredRootDirName) { char* validMirroredDirName = UTIL_createMirroredDestDirName(srcFileName, outMirroredRootDirName); @@ -1736,7 +1778,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, } if (outDirName) - FIO_checkFilenameCollisions(inFileNamesTable , prefs->nbFiles); + FIO_checkFilenameCollisions(inFileNamesTable , fCtx->nbFiles); } FIO_freeCResources(ress); diff --git a/programs/fileio.h b/programs/fileio.h index f86ae417..8339b3e2 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -54,16 +54,24 @@ extern "C" { ***************************************/ typedef enum { FIO_zstdCompression, FIO_gzipCompression, FIO_xzCompression, FIO_lzmaCompression, FIO_lz4Compression } FIO_compressionType_t; +/* Immutable struct containing preferences for file I/O */ typedef struct FIO_prefs_s FIO_prefs_t; FIO_prefs_t* FIO_createPreferences(void); void FIO_freePreferences(FIO_prefs_t* const prefs); +/* Mutable struct containing relevant context and state of (de)compression with respect to file I/O */ +typedef struct FIO_ctx_s FIO_ctx_t; + +FIO_ctx_t* FIO_createContext(void); +void FIO_freeContext(FIO_ctx_t* const fCtx); + typedef struct FIO_display_prefs_s FIO_display_prefs_t; /*-************************************* * Parameters ***************************************/ +/* FIO_prefs_t functions */ void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t compressionType); void FIO_overwriteMode(FIO_prefs_t* const prefs); void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt); @@ -96,8 +104,12 @@ void FIO_setNotificationLevel(int level); void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles); void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value); void FIO_setContentSize(FIO_prefs_t* const prefs, int value); -void FIO_setNbFiles(FIO_prefs_t* const prefs, int value); -void FIO_setCurrFileIdx(FIO_prefs_t* const prefs, int value); + +/* FIO_ctx_t functions */ +void FIO_setNbFiles(FIO_ctx_t* const fCtx, int value); +void FIO_setCurrFileIdx(FIO_ctx_t* const fCtx, int value); +void FIO_setTotalBytesInput(FIO_ctx_t* const fCtx, size_t value); +void FIO_setTotalBytesOutput(FIO_ctx_t* const fCtx, size_t value); /*-************************************* * Single File functions diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 9dab010f..6bd869e4 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -693,6 +693,7 @@ int main(int const argCount, const char* argv[]) size_t blockSize = 0; FIO_prefs_t* const prefs = FIO_createPreferences(); + FIO_ctx_t* const fCtx = FIO_createContext(); zstd_operation_mode operation = zom_compress; ZSTD_compressionParameters compressionParams; int cLevel = init_cLevel(); @@ -1260,7 +1261,7 @@ int main(int const argCount, const char* argv[]) if (!strcmp(filenames->fileNames[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (g_displayLevel==2)) g_displayLevel=1; /* IO Stream/File */ - FIO_setNbFiles(prefs, (int)filenames->tableSize); + FIO_setNbFiles(fCtx, (int)filenames->tableSize); FIO_setNotificationLevel(g_displayLevel); FIO_setPatchFromMode(prefs, patchFromDictFileName != NULL); if (memLimit == 0) { @@ -1319,9 +1320,9 @@ int main(int const argCount, const char* argv[]) } if ((filenames->tableSize==1) && outFileName) - operationResult = FIO_compressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams); + operationResult = FIO_compressFilename(prefs, fCtx, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams); else - operationResult = FIO_compressMultipleFilenames(prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams); + operationResult = FIO_compressMultipleFilenames(prefs, fCtx, filenames->fileNames, outMirroredDirName, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams); #else (void)contentSize; (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; (void)ZSTD_strategyMap; /* not used when ZSTD_NOCOMPRESS set */ DISPLAY("Compression not supported \n"); From a6414f12475520b18cadf3cfd240986d77581a91 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Tue, 1 Sep 2020 12:32:18 -0400 Subject: [PATCH 02/12] Integrate refactor into status print for multifiles, adjust logic for printing as needed --- programs/fileio.c | 137 +++++++++++++++++++++++++++++---------------- programs/fileio.h | 6 +- programs/zstdcli.c | 4 +- 3 files changed, 95 insertions(+), 52 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 1ace731e..f4859bb1 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -330,7 +330,8 @@ struct FIO_ctx_s { /* multiple file processing info */ int currFileIdx; - int nbFiles; + int nbFilesTotal; + int nbFilesProcessed; size_t totalBytesInput; size_t totalBytesOutput; }; @@ -383,7 +384,8 @@ FIO_ctx_t* FIO_createContext(void) if (!ret) EXM_THROW(21, "Allocation error : not enough memory"); ret->currFileIdx = 0; - ret->nbFiles = 1; + ret->nbFilesTotal = 1; + ret->nbFilesProcessed = 0; ret->totalBytesInput = 0; ret->totalBytesOutput = 0; return ret; @@ -532,7 +534,7 @@ void FIO_setContentSize(FIO_prefs_t* const prefs, int value) void FIO_setNbFiles(FIO_ctx_t* const fCtx, int value) { - fCtx->nbFiles = value; + fCtx->nbFilesTotal = value; } void FIO_setCurrFileIdx(FIO_ctx_t* const fCtx, int value) @@ -1235,6 +1237,7 @@ FIO_compressLz4Frame(cRess_t* ress, static unsigned long long FIO_compressZstdFrame(FIO_prefs_t* const prefs, + FIO_ctx_t* const fCtx, const cRess_t* ressPtr, const char* srcFileName, U64 fileSize, int compressionLevel, U64* readsize) @@ -1316,9 +1319,9 @@ FIO_compressZstdFrame(FIO_prefs_t* const prefs, (unsigned)(zfp.consumed >> 20), (unsigned)(zfp.produced >> 20), cShare ); - } else { /* summarized notifications if == 2; */ - if (prefs->nbFiles > 1) { - DISPLAYLEVEL(2, "\rCompressing %u/%u files. Current source: %s ", prefs->currFileIdx+1, prefs->nbFiles, srcFileName); + } else { /* summarized notifications if == 2 */ + if (fCtx->nbFilesTotal > 1) { + DISPLAYLEVEL(2, "\rCompressing %u/%u files. Current source: %s ", fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName); } else { DISPLAYLEVEL(2, "\r"); } @@ -1437,6 +1440,7 @@ FIO_compressZstdFrame(FIO_prefs_t* const prefs, */ static int FIO_compressFilename_internal(FIO_prefs_t* const prefs, + FIO_ctx_t* const fCtx, cRess_t ress, const char* dstFileName, const char* srcFileName, int compressionLevel) @@ -1452,7 +1456,7 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs, switch (prefs->compressionType) { default: case FIO_zstdCompression: - compressedfilesize = FIO_compressZstdFrame(prefs, &ress, srcFileName, fileSize, compressionLevel, &readsize); + compressedfilesize = FIO_compressZstdFrame(prefs, fCtx, &ress, srcFileName, fileSize, compressionLevel, &readsize); break; case FIO_gzipCompression: @@ -1490,18 +1494,23 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs, /* Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); - if (g_display_prefs.displayLevel > 2 || (g_display_prefs.displayLevel == 2 && prefs->nbFiles <= 1)) { - if (readsize == 0) { - DISPLAYLEVEL(2,"%-20s : (%6llu => %6llu bytes, %s) \n", - srcFileName, - (unsigned long long)readsize, (unsigned long long) compressedfilesize, - dstFileName); - } else { - DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", - srcFileName, - (double)compressedfilesize / readsize * 100, - (unsigned long long)readsize, (unsigned long long) compressedfilesize, - dstFileName); + if (g_display_prefs.displayLevel >= 2) { + if (g_display_prefs.displayLevel >= 3 || fCtx->nbFilesTotal <= 1) { + if (readsize == 0) { + DISPLAYLEVEL(2,"%-20s : (%6llu => %6llu bytes, %s) \n", + srcFileName, + (unsigned long long)readsize, (unsigned long long) compressedfilesize, + dstFileName); + } else { + DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", + srcFileName, + (double)compressedfilesize / readsize * 100, + (unsigned long long)readsize, (unsigned long long) compressedfilesize, + dstFileName); + } + } else if (fCtx->nbFilesTotal > 1) { + fCtx->totalBytesInput += (size_t)readsize; + fCtx->totalBytesOutput += (size_t)compressedfilesize; } } @@ -1528,6 +1537,7 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs, * 1 : pb */ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs, + FIO_ctx_t* const fCtx, cRess_t ress, const char* dstFileName, const char* srcFileName, @@ -1555,7 +1565,7 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs, transfer_permissions = 1; } - result = FIO_compressFilename_internal(prefs, ress, dstFileName, srcFileName, compressionLevel); + result = FIO_compressFilename_internal(prefs, fCtx, ress, dstFileName, srcFileName, compressionLevel); if (closeDstFile) { FILE* const dstFile = ress.dstFile; @@ -1605,6 +1615,7 @@ static const char *compressedFileExtensions[] = { */ static int FIO_compressFilename_srcFile(FIO_prefs_t* const prefs, + FIO_ctx_t* const fCtx, cRess_t ress, const char* dstFileName, const char* srcFileName, @@ -1637,7 +1648,7 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs, ress.srcFile = FIO_openSrcFile(srcFileName); if (ress.srcFile == NULL) return 1; /* srcFile could not be opened */ - result = FIO_compressFilename_dstFile(prefs, ress, dstFileName, srcFileName, compressionLevel); + result = FIO_compressFilename_dstFile(prefs, fCtx, ress, dstFileName, srcFileName, compressionLevel); fclose(ress.srcFile); ress.srcFile = NULL; @@ -1655,12 +1666,12 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs, return result; } -int FIO_compressFilename(FIO_prefs_t* const prefs, const char* dstFileName, +int FIO_compressFilename(FIO_prefs_t* const prefs, FIO_ctx_t* const fCtx, const char* dstFileName, const char* srcFileName, const char* dictFileName, int compressionLevel, ZSTD_compressionParameters comprParams) { cRess_t const ress = FIO_createCResources(prefs, dictFileName, UTIL_getFileSize(srcFileName), compressionLevel, comprParams); - int const result = FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); + int const result = FIO_compressFilename_srcFile(prefs, fCtx, ress, dstFileName, srcFileName, compressionLevel); #define DISPLAY_LEVEL_DEFAULT 2 @@ -1734,9 +1745,9 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char* dictFileName, int compressionLevel, ZSTD_compressionParameters comprParams) { - int error = 0; + int error = 0, status = 0; cRess_t ress = FIO_createCResources(prefs, dictFileName, - FIO_getLargestFileSize(inFileNamesTable, fCtx->nbFiles), + FIO_getLargestFileSize(inFileNamesTable, fCtx->nbFilesTotal), compressionLevel, comprParams); /* init */ @@ -1746,8 +1757,10 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, if (ress.dstFile == NULL) { /* could not open outFileName */ error = 1; } else { - for (; fCtx->currFileIdx < fCtx->nbFiles; ++fCtx->currFileIdx) { - error |= FIO_compressFilename_srcFile(prefs, ress, outFileName, inFileNamesTable[fCtx->currFileIdx], compressionLevel); + for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) { + status = FIO_compressFilename_srcFile(prefs, fCtx, ress, outFileName, inFileNamesTable[fCtx->currFileIdx], compressionLevel); + fCtx->nbFilesProcessed = status ? fCtx->nbFilesProcessed : fCtx->nbFilesProcessed + 1; + error |= status; } if (fclose(ress.dstFile)) EXM_THROW(29, "Write error (%s) : cannot properly close %s", @@ -1756,9 +1769,9 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, } } else { if (outMirroredRootDirName) - UTIL_mirrorSourceFilesDirectories(inFileNamesTable, fCtx->nbFiles, outMirroredRootDirName); + UTIL_mirrorSourceFilesDirectories(inFileNamesTable, fCtx->nbFilesTotal, outMirroredRootDirName); - for (; fCtx->currFileIdx < fCtx->nbFiles; ++fCtx->currFileIdx) { + for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) { const char* const srcFileName = inFileNamesTable[fCtx->currFileIdx]; const char* dstFileName = NULL; if (outMirroredRootDirName) { @@ -1774,11 +1787,24 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, } else { dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix); /* cannot fail */ } - error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); + status = FIO_compressFilename_srcFile(prefs, fCtx, ress, dstFileName, srcFileName, compressionLevel); + fCtx->nbFilesProcessed = status ? fCtx->nbFilesProcessed : fCtx->nbFilesProcessed + 1; + error |= status; } if (outDirName) - FIO_checkFilenameCollisions(inFileNamesTable , fCtx->nbFiles); + FIO_checkFilenameCollisions(inFileNamesTable , fCtx->nbFilesTotal); + } + + if (fCtx->nbFilesProcessed > 1) { + if (fCtx->totalBytesInput != 0) { + DISPLAYLEVEL(2, "%d files compressed : %.2f%% (%6zu => %6zu bytes)\n", fCtx->nbFilesProcessed, + (double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100, + fCtx->totalBytesInput, fCtx->totalBytesOutput); + } else { + DISPLAYLEVEL(2, "%d files compressed : (%6zu => %6zu bytes)\n", fCtx->nbFilesProcessed, + fCtx->totalBytesInput, fCtx->totalBytesOutput); + } } FIO_freeCResources(ress); @@ -2014,6 +2040,7 @@ FIO_zstdErrorHelp(const FIO_prefs_t* const prefs, static unsigned long long FIO_decompressZstdFrame(dRess_t* ress, FILE* finput, const FIO_prefs_t* const prefs, + FIO_ctx_t* const fCtx, const char* srcFileName, U64 alreadyDecoded) /* for multi-frames streams */ { @@ -2050,9 +2077,9 @@ FIO_decompressZstdFrame(dRess_t* ress, FILE* finput, /* Write block */ storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, prefs, storedSkips); frameSize += outBuff.pos; - if (prefs->nbFiles > 1) { + if (fCtx->nbFilesTotal > 1) { DISPLAYUPDATE(2, "\rDecompressing %u/%u files. Current source: %-20.20s : %u MB... ", - prefs->currFileIdx+1, prefs->nbFiles, srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); + fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); } else { DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ", srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); @@ -2316,6 +2343,7 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, */ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, const FIO_prefs_t* const prefs, + FIO_ctx_t* const fCtx, const char* dstFileName, const char* srcFileName) { unsigned readSomething = 0; @@ -2343,7 +2371,7 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, return 1; } if (ZSTD_isFrame(buf, ress.srcBufferLoaded)) { - unsigned long long const frameSize = FIO_decompressZstdFrame(&ress, srcFile, prefs, srcFileName, filesize); + unsigned long long const frameSize = FIO_decompressZstdFrame(&ress, srcFile, prefs, fCtx, srcFileName, filesize); if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; filesize += frameSize; } else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */ @@ -2387,8 +2415,12 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, /* Final Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); /* No status message in pipe mode (stdin - stdout) or multi-files mode */ - if (g_display_prefs.displayLevel > 2 || (g_display_prefs.displayLevel == 2 && prefs->nbFiles <= 1)) { + if (g_display_prefs.displayLevel >= 2) { + if (fCtx->nbFilesTotal <= 1 || g_display_prefs.displayLevel >= 3) { DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize); + } else if (fCtx->nbFilesTotal > 1) { + fCtx->totalBytesOutput += filesize; + } } return 0; @@ -2402,6 +2434,7 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, 1 : operation aborted */ static int FIO_decompressDstFile(FIO_prefs_t* const prefs, + FIO_ctx_t* const fCtx, dRess_t ress, FILE* srcFile, const char* dstFileName, const char* srcFileName) { @@ -2428,7 +2461,7 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs, transfer_permissions = 1; } - result = FIO_decompressFrames(ress, srcFile, prefs, dstFileName, srcFileName); + result = FIO_decompressFrames(ress, srcFile, prefs, fCtx, dstFileName, srcFileName); if (releaseDstFile) { FILE* const dstFile = ress.dstFile; @@ -2457,7 +2490,7 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs, @return : 0 : OK 1 : error */ -static int FIO_decompressSrcFile(FIO_prefs_t* const prefs, dRess_t ress, const char* dstFileName, const char* srcFileName) +static int FIO_decompressSrcFile(FIO_prefs_t* const prefs, FIO_ctx_t* const fCtx, dRess_t ress, const char* dstFileName, const char* srcFileName) { FILE* srcFile; int result; @@ -2471,7 +2504,7 @@ static int FIO_decompressSrcFile(FIO_prefs_t* const prefs, dRess_t ress, const c if (srcFile==NULL) return 1; ress.srcBufferLoaded = 0; - result = FIO_decompressDstFile(prefs, ress, srcFile, dstFileName, srcFileName); + result = FIO_decompressDstFile(prefs, fCtx, ress, srcFile, dstFileName, srcFileName); /* Close file */ if (fclose(srcFile)) { @@ -2495,13 +2528,13 @@ static int FIO_decompressSrcFile(FIO_prefs_t* const prefs, dRess_t ress, const c -int FIO_decompressFilename(FIO_prefs_t* const prefs, +int FIO_decompressFilename(FIO_prefs_t* const prefs, FIO_ctx_t* const fCtx, const char* dstFileName, const char* srcFileName, const char* dictFileName) { dRess_t const ress = FIO_createDResources(prefs, dictFileName); - int const decodingError = FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName); + int const decodingError = FIO_decompressSrcFile(prefs, fCtx, ress, dstFileName, srcFileName); FIO_freeDResources(ress); return decodingError; @@ -2627,12 +2660,13 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName) int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, + FIO_ctx_t* const fCtx, const char** srcNamesTable, const char* outMirroredRootDirName, const char* outDirName, const char* outFileName, const char* dictFileName) { - int error = 0; + int error = 0, status = 0; dRess_t ress = FIO_createDResources(prefs, dictFileName); if (outFileName) { @@ -2640,18 +2674,20 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName); if (ress.dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName); } - for (; prefs->currFileIdx < prefs->nbFiles; prefs->currFileIdx++) { - error |= FIO_decompressSrcFile(prefs, ress, outFileName, srcNamesTable[prefs->currFileIdx]); + for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { + status = FIO_decompressSrcFile(prefs, fCtx, ress, outFileName, srcNamesTable[fCtx->currFileIdx]); + fCtx->nbFilesProcessed = status ? fCtx->nbFilesProcessed : fCtx->nbFilesProcessed + 1; + error |= status; } if ((!prefs->testMode) && (fclose(ress.dstFile))) EXM_THROW(72, "Write error : %s : cannot properly close output file", strerror(errno)); } else { if (outMirroredRootDirName) - UTIL_mirrorSourceFilesDirectories(srcNamesTable, prefs->nbFiles, outMirroredRootDirName); + UTIL_mirrorSourceFilesDirectories(srcNamesTable, fCtx->nbFilesTotal, outMirroredRootDirName); - for (; prefs->currFileIdx < prefs->nbFiles; prefs->currFileIdx++) { /* create dstFileName */ - const char* const srcFileName = srcNamesTable[prefs->currFileIdx]; + for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { /* create dstFileName */ + const char* const srcFileName = srcNamesTable[fCtx->currFileIdx]; const char* dstFileName = NULL; if (outMirroredRootDirName) { char* validMirroredDirName = UTIL_createMirroredDestDirName(srcFileName, outMirroredRootDirName); @@ -2665,11 +2701,16 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, dstFileName = FIO_determineDstName(srcFileName, outDirName); } if (dstFileName == NULL) { error=1; continue; } - error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName); + status = FIO_decompressSrcFile(prefs, fCtx, ress, dstFileName, srcFileName); + fCtx->nbFilesProcessed = status ? fCtx->nbFilesProcessed : fCtx->nbFilesProcessed + 1; + error |= status; } if (outDirName) - FIO_checkFilenameCollisions(srcNamesTable , prefs->nbFiles); + FIO_checkFilenameCollisions(srcNamesTable , fCtx->nbFilesTotal); } + + if (fCtx->nbFilesProcessed > 1) + DISPLAYLEVEL(2, "%d files decompressed : %6zu bytes total \n", fCtx->nbFilesProcessed, fCtx->totalBytesOutput); FIO_freeDResources(ress); return error; diff --git a/programs/fileio.h b/programs/fileio.h index 8339b3e2..ae0e2c75 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -116,14 +116,14 @@ void FIO_setTotalBytesOutput(FIO_ctx_t* const fCtx, size_t value); ***************************************/ /** FIO_compressFilename() : * @return : 0 == ok; 1 == pb with src file. */ -int FIO_compressFilename (FIO_prefs_t* const prefs, +int FIO_compressFilename (FIO_prefs_t* const prefs, FIO_ctx_t* const fCtx, const char* outfilename, const char* infilename, const char* dictFileName, int compressionLevel, ZSTD_compressionParameters comprParams); /** FIO_decompressFilename() : * @return : 0 == ok; 1 == pb with src file. */ -int FIO_decompressFilename (FIO_prefs_t* const prefs, +int FIO_decompressFilename (FIO_prefs_t* const prefs, FIO_ctx_t* const fCtx, const char* outfilename, const char* infilename, const char* dictFileName); int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel); @@ -135,6 +135,7 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis /** FIO_compressMultipleFilenames() : * @return : nb of missing files */ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, + FIO_ctx_t* const fCtx, const char** inFileNamesTable, const char* outMirroredDirName, const char* outDirName, @@ -145,6 +146,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, /** FIO_decompressMultipleFilenames() : * @return : nb of missing or skipped files */ int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, + FIO_ctx_t* const fCtx, const char** srcNamesTable, const char* outMirroredDirName, const char* outDirName, diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 6bd869e4..7ca07b21 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1330,9 +1330,9 @@ int main(int const argCount, const char* argv[]) } else { /* decompression or test */ #ifndef ZSTD_NODECOMPRESS if (filenames->tableSize == 1 && outFileName) { - operationResult = FIO_decompressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName); + operationResult = FIO_decompressFilename(prefs, fCtx, outFileName, filenames->fileNames[0], dictFileName); } else { - operationResult = FIO_decompressMultipleFilenames(prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, dictFileName); + operationResult = FIO_decompressMultipleFilenames(prefs, fCtx, filenames->fileNames, outMirroredDirName, outDirName, outFileName, dictFileName); } #else DISPLAY("Decompression not supported \n"); From a03917c751fad3acd9e8af99e9a419df7cddc244 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Tue, 1 Sep 2020 12:46:06 -0400 Subject: [PATCH 03/12] Fix the logic in printing final status updates --- programs/fileio.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index f4859bb1..43fe3ead 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1508,7 +1508,8 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs, (unsigned long long)readsize, (unsigned long long) compressedfilesize, dstFileName); } - } else if (fCtx->nbFilesTotal > 1) { + } + if (fCtx->nbFilesTotal > 1) { fCtx->totalBytesInput += (size_t)readsize; fCtx->totalBytesOutput += (size_t)compressedfilesize; } @@ -2418,7 +2419,8 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, if (g_display_prefs.displayLevel >= 2) { if (fCtx->nbFilesTotal <= 1 || g_display_prefs.displayLevel >= 3) { DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize); - } else if (fCtx->nbFilesTotal > 1) { + } + if (fCtx->nbFilesTotal > 1) { fCtx->totalBytesOutput += filesize; } } From 3a7d625d6ba679a058bf223188628fa83b1982ca Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Tue, 1 Sep 2020 12:52:18 -0400 Subject: [PATCH 04/12] Cleanup comments, add function to set FIO_ctx_t.nbFilesProcessed --- programs/fileio.c | 11 ++++++++--- programs/fileio.h | 4 ++-- programs/zstdcli.c | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 43fe3ead..53d06671 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -322,7 +322,7 @@ struct FIO_prefs_s { }; /*-************************************* -* Parameters: FIO_prefs_t +* Parameters: FIO_ctx_t ***************************************/ /* typedef'd to FIO_ctx_t within fileio.h */ @@ -532,11 +532,16 @@ void FIO_setContentSize(FIO_prefs_t* const prefs, int value) /* FIO_ctx_t functions */ -void FIO_setNbFiles(FIO_ctx_t* const fCtx, int value) +void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value) { fCtx->nbFilesTotal = value; } +void FIO_setNbFilesProcessed(FIO_ctx_t* const fCtx, int value) +{ + fCtx->nbFilesProcessed = value; +} + void FIO_setCurrFileIdx(FIO_ctx_t* const fCtx, int value) { fCtx->currFileIdx = value; @@ -2421,7 +2426,7 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize); } if (fCtx->nbFilesTotal > 1) { - fCtx->totalBytesOutput += filesize; + fCtx->totalBytesOutput += (size_t)filesize; } } diff --git a/programs/fileio.h b/programs/fileio.h index ae0e2c75..f4008a44 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -54,7 +54,6 @@ extern "C" { ***************************************/ typedef enum { FIO_zstdCompression, FIO_gzipCompression, FIO_xzCompression, FIO_lzmaCompression, FIO_lz4Compression } FIO_compressionType_t; -/* Immutable struct containing preferences for file I/O */ typedef struct FIO_prefs_s FIO_prefs_t; FIO_prefs_t* FIO_createPreferences(void); @@ -106,7 +105,8 @@ void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value); void FIO_setContentSize(FIO_prefs_t* const prefs, int value); /* FIO_ctx_t functions */ -void FIO_setNbFiles(FIO_ctx_t* const fCtx, int value); +void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value); +void FIO_setNbFilesProcessed(FIO_ctx_t* const fCtx, int value); void FIO_setCurrFileIdx(FIO_ctx_t* const fCtx, int value); void FIO_setTotalBytesInput(FIO_ctx_t* const fCtx, size_t value); void FIO_setTotalBytesOutput(FIO_ctx_t* const fCtx, size_t value); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 7ca07b21..da6c24de 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1261,7 +1261,7 @@ int main(int const argCount, const char* argv[]) if (!strcmp(filenames->fileNames[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (g_displayLevel==2)) g_displayLevel=1; /* IO Stream/File */ - FIO_setNbFiles(fCtx, (int)filenames->tableSize); + FIO_setNbFilesTotal(fCtx, (int)filenames->tableSize); FIO_setNotificationLevel(g_displayLevel); FIO_setPatchFromMode(prefs, patchFromDictFileName != NULL); if (memLimit == 0) { From 7842f431971536284c416d67a742939d0740e578 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Thu, 3 Sep 2020 09:22:07 -0400 Subject: [PATCH 05/12] Fix 1 file multifile logic, remove unnecessary variable assignments --- programs/fileio.c | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 53d06671..94a2cc83 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1751,7 +1751,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char* dictFileName, int compressionLevel, ZSTD_compressionParameters comprParams) { - int error = 0, status = 0; + int error = 0, status; cRess_t ress = FIO_createCResources(prefs, dictFileName, FIO_getLargestFileSize(inFileNamesTable, fCtx->nbFilesTotal), compressionLevel, comprParams); @@ -1765,7 +1765,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, } else { for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) { status = FIO_compressFilename_srcFile(prefs, fCtx, ress, outFileName, inFileNamesTable[fCtx->currFileIdx], compressionLevel); - fCtx->nbFilesProcessed = status ? fCtx->nbFilesProcessed : fCtx->nbFilesProcessed + 1; + if (!status) fCtx->nbFilesProcessed++; error |= status; } if (fclose(ress.dstFile)) @@ -1794,7 +1794,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix); /* cannot fail */ } status = FIO_compressFilename_srcFile(prefs, fCtx, ress, dstFileName, srcFileName, compressionLevel); - fCtx->nbFilesProcessed = status ? fCtx->nbFilesProcessed : fCtx->nbFilesProcessed + 1; + if (!status) fCtx->nbFilesProcessed++; error |= status; } @@ -1802,16 +1802,10 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, FIO_checkFilenameCollisions(inFileNamesTable , fCtx->nbFilesTotal); } - if (fCtx->nbFilesProcessed > 1) { - if (fCtx->totalBytesInput != 0) { - DISPLAYLEVEL(2, "%d files compressed : %.2f%% (%6zu => %6zu bytes)\n", fCtx->nbFilesProcessed, + if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesInput != 0) + DISPLAYLEVEL(2, "%d files compressed : %.2f%% (%6zu => %6zu bytes)\n", fCtx->nbFilesProcessed, (double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100, fCtx->totalBytesInput, fCtx->totalBytesOutput); - } else { - DISPLAYLEVEL(2, "%d files compressed : (%6zu => %6zu bytes)\n", fCtx->nbFilesProcessed, - fCtx->totalBytesInput, fCtx->totalBytesOutput); - } - } FIO_freeCResources(ress); return error; @@ -2673,7 +2667,7 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, const char* outDirName, const char* outFileName, const char* dictFileName) { - int error = 0, status = 0; + int error = 0, status; dRess_t ress = FIO_createDResources(prefs, dictFileName); if (outFileName) { @@ -2683,7 +2677,7 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, } for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { status = FIO_decompressSrcFile(prefs, fCtx, ress, outFileName, srcNamesTable[fCtx->currFileIdx]); - fCtx->nbFilesProcessed = status ? fCtx->nbFilesProcessed : fCtx->nbFilesProcessed + 1; + if (!status) fCtx->nbFilesProcessed++; error |= status; } if ((!prefs->testMode) && (fclose(ress.dstFile))) @@ -2709,14 +2703,14 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, } if (dstFileName == NULL) { error=1; continue; } status = FIO_decompressSrcFile(prefs, fCtx, ress, dstFileName, srcFileName); - fCtx->nbFilesProcessed = status ? fCtx->nbFilesProcessed : fCtx->nbFilesProcessed + 1; + if (!status) fCtx->nbFilesProcessed++; error |= status; } if (outDirName) FIO_checkFilenameCollisions(srcNamesTable , fCtx->nbFilesTotal); } - if (fCtx->nbFilesProcessed > 1) + if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesInput != 0) DISPLAYLEVEL(2, "%d files decompressed : %6zu bytes total \n", fCtx->nbFilesProcessed, fCtx->totalBytesOutput); FIO_freeDResources(ress); From a480b0204450218c44d7a1650fdc7285115aa79e Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Thu, 3 Sep 2020 09:26:30 -0400 Subject: [PATCH 06/12] Adjust comments and struct to better reflect FIO_ctx_t --- programs/fileio.c | 6 ++++-- programs/fileio.h | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 94a2cc83..de7900cd 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -328,9 +328,11 @@ struct FIO_prefs_s { /* typedef'd to FIO_ctx_t within fileio.h */ struct FIO_ctx_s { - /* multiple file processing info */ - int currFileIdx; + /* file i/o info */ int nbFilesTotal; + + /* file i/o state */ + int currFileIdx; int nbFilesProcessed; size_t totalBytesInput; size_t totalBytesOutput; diff --git a/programs/fileio.h b/programs/fileio.h index f4008a44..a70e4905 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -59,7 +59,7 @@ typedef struct FIO_prefs_s FIO_prefs_t; FIO_prefs_t* FIO_createPreferences(void); void FIO_freePreferences(FIO_prefs_t* const prefs); -/* Mutable struct containing relevant context and state of (de)compression with respect to file I/O */ +/* Mutable struct containing relevant context and state regarding (de)compression with respect to file I/O */ typedef struct FIO_ctx_s FIO_ctx_t; FIO_ctx_t* FIO_createContext(void); From 202b2954a229eee5b0efa0353bd35a8dc19a0399 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Thu, 3 Sep 2020 09:28:40 -0400 Subject: [PATCH 07/12] Remove comma-separated decls --- programs/fileio.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index de7900cd..e0392af8 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1753,7 +1753,8 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char* dictFileName, int compressionLevel, ZSTD_compressionParameters comprParams) { - int error = 0, status; + int status; + int error = 0; cRess_t ress = FIO_createCResources(prefs, dictFileName, FIO_getLargestFileSize(inFileNamesTable, fCtx->nbFilesTotal), compressionLevel, comprParams); @@ -2669,7 +2670,8 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, const char* outDirName, const char* outFileName, const char* dictFileName) { - int error = 0, status; + int status; + int error = 0; dRess_t ress = FIO_createDResources(prefs, dictFileName); if (outFileName) { From 23feaecbe542b752f4aec1c2bf014a3c4a20f5b7 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Thu, 3 Sep 2020 09:49:13 -0400 Subject: [PATCH 08/12] Fix FIO_removeMultiFilesWarning conflict --- programs/fileio.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 28ad6dce..a5f7cb64 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -850,10 +850,10 @@ static void FIO_adjustMemLimitForPatchFromMode(FIO_prefs_t* const prefs, * If neither flag is specified, zstd will prompt the user for confirmation to proceed. * If --rm is not specified, then zstd will print a warning to the user (which can be silenced with -q). */ -static int FIO_removeMultiFilesWarning(const FIO_prefs_t* const prefs, int displayLevelCutoff, const char* outFileName) +static int FIO_removeMultiFilesWarning(const FIO_prefs_t* const prefs, FIO_ctx_t* const fCtx, int displayLevelCutoff, const char* outFileName) { int error = 0; - if (prefs->nbFiles > 1 && !prefs->overwrite) { + if (fCtx->nbFilesTotal > 1 && !prefs->overwrite) { if (g_display_prefs.displayLevel <= displayLevelCutoff) { if (prefs->removeSrcFile) { DISPLAYLEVEL(1, "zstd: Aborting... not deleting files and processing into dst: %s", outFileName); @@ -1791,7 +1791,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, /* init */ assert(outFileName != NULL || suffix != NULL); if (outFileName != NULL) { /* output into a single destination (stdout typically) */ - if (FIO_removeMultiFilesWarning(prefs, 1 /* displayLevelCutoff */, outFileName)) { + if (FIO_removeMultiFilesWarning(prefs, fCtx, 1 /* displayLevelCutoff */, outFileName)) { FIO_freeCResources(ress); return 1; } @@ -2708,7 +2708,7 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, dRess_t ress = FIO_createDResources(prefs, dictFileName); if (outFileName) { - if (FIO_removeMultiFilesWarning(prefs, 1 /* displayLevelCutoff */, outFileName)) { + if (FIO_removeMultiFilesWarning(prefs, fCtx, 1 /* displayLevelCutoff */, outFileName)) { FIO_freeDResources(ress); return 1; } From 48bca107929b5ea1ab21759f70eb7807216628c0 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Thu, 3 Sep 2020 10:14:04 -0400 Subject: [PATCH 09/12] Address memory leak in CLI from fCtx --- programs/zstdcli.c | 1 + 1 file changed, 1 insertion(+) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index da6c24de..c799cb78 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1341,6 +1341,7 @@ int main(int const argCount, const char* argv[]) _end: FIO_freePreferences(prefs); + FIO_freeContext(fCtx); if (main_pause) waitEnter(); UTIL_freeFileNamesTable(filenames); UTIL_freeFileNamesTable(file_of_names); From 60ee0519ca7085f005ce6e91525ca7e5a8c5cdc0 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Thu, 3 Sep 2020 20:22:34 -0400 Subject: [PATCH 10/12] Remove extraneous FIO_ctx_t functions --- programs/fileio.c | 20 -------------------- programs/fileio.h | 4 ---- 2 files changed, 24 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index a5f7cb64..51f8af45 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -539,26 +539,6 @@ void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value) fCtx->nbFilesTotal = value; } -void FIO_setNbFilesProcessed(FIO_ctx_t* const fCtx, int value) -{ - fCtx->nbFilesProcessed = value; -} - -void FIO_setCurrFileIdx(FIO_ctx_t* const fCtx, int value) -{ - fCtx->currFileIdx = value; -} - -void FIO_setTotalBytesInput(FIO_ctx_t* const fCtx, size_t value) -{ - fCtx->totalBytesInput = value; -} - -void FIO_setTotalBytesOutput(FIO_ctx_t* const fCtx, size_t value) -{ - fCtx->totalBytesOutput = value; -} - /*-************************************* * Functions ***************************************/ diff --git a/programs/fileio.h b/programs/fileio.h index a70e4905..eeede526 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -106,10 +106,6 @@ void FIO_setContentSize(FIO_prefs_t* const prefs, int value); /* FIO_ctx_t functions */ void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value); -void FIO_setNbFilesProcessed(FIO_ctx_t* const fCtx, int value); -void FIO_setCurrFileIdx(FIO_ctx_t* const fCtx, int value); -void FIO_setTotalBytesInput(FIO_ctx_t* const fCtx, size_t value); -void FIO_setTotalBytesOutput(FIO_ctx_t* const fCtx, size_t value); /*-************************************* * Single File functions From 28a9dc78c3b4e6d35041c3ffa3dfda241ec102a1 Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Thu, 3 Sep 2020 20:23:30 -0400 Subject: [PATCH 11/12] Always update bytes status in FIO_ctx_t --- programs/fileio.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 51f8af45..5ba39a16 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1508,7 +1508,8 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs, } /* Status */ - + fCtx->totalBytesInput += (size_t)readsize; + fCtx->totalBytesOutput += (size_t)compressedfilesize; DISPLAYLEVEL(2, "\r%79s\r", ""); if (g_display_prefs.displayLevel >= 2) { if (g_display_prefs.displayLevel >= 3 || fCtx->nbFilesTotal <= 1) { @@ -1525,10 +1526,6 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs, dstFileName); } } - if (fCtx->nbFilesTotal > 1) { - fCtx->totalBytesInput += (size_t)readsize; - fCtx->totalBytesOutput += (size_t)compressedfilesize; - } } /* Elapsed Time and CPU Load */ @@ -2429,15 +2426,13 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, } } /* for each frame */ /* Final Status */ + fCtx->totalBytesOutput += (size_t)filesize; DISPLAYLEVEL(2, "\r%79s\r", ""); /* No status message in pipe mode (stdin - stdout) or multi-files mode */ if (g_display_prefs.displayLevel >= 2) { if (fCtx->nbFilesTotal <= 1 || g_display_prefs.displayLevel >= 3) { DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize); } - if (fCtx->nbFilesTotal > 1) { - fCtx->totalBytesOutput += (size_t)filesize; - } } return 0; From 512349610428101a2c3895fec87c3fff86b9202b Mon Sep 17 00:00:00 2001 From: senhuang42 Date: Mon, 7 Sep 2020 13:13:05 -0400 Subject: [PATCH 12/12] Adjust function signatures --- programs/fileio.c | 71 +++++++++++++++++++++++----------------------- programs/fileio.h | 12 ++++---- programs/zstdcli.c | 8 +++--- 3 files changed, 45 insertions(+), 46 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 5ba39a16..d1a68ee6 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -830,7 +830,7 @@ static void FIO_adjustMemLimitForPatchFromMode(FIO_prefs_t* const prefs, * If neither flag is specified, zstd will prompt the user for confirmation to proceed. * If --rm is not specified, then zstd will print a warning to the user (which can be silenced with -q). */ -static int FIO_removeMultiFilesWarning(const FIO_prefs_t* const prefs, FIO_ctx_t* const fCtx, int displayLevelCutoff, const char* outFileName) +static int FIO_removeMultiFilesWarning(FIO_ctx_t* const fCtx, const FIO_prefs_t* const prefs, const char* outFileName, int displayLevelCutoff) { int error = 0; if (fCtx->nbFilesTotal > 1 && !prefs->overwrite) { @@ -1252,8 +1252,8 @@ FIO_compressLz4Frame(cRess_t* ress, static unsigned long long -FIO_compressZstdFrame(FIO_prefs_t* const prefs, - FIO_ctx_t* const fCtx, +FIO_compressZstdFrame(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, const cRess_t* ressPtr, const char* srcFileName, U64 fileSize, int compressionLevel, U64* readsize) @@ -1455,8 +1455,8 @@ FIO_compressZstdFrame(FIO_prefs_t* const prefs, * 1 : missing or pb opening srcFileName */ static int -FIO_compressFilename_internal(FIO_prefs_t* const prefs, - FIO_ctx_t* const fCtx, +FIO_compressFilename_internal(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, cRess_t ress, const char* dstFileName, const char* srcFileName, int compressionLevel) @@ -1472,7 +1472,7 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs, switch (prefs->compressionType) { default: case FIO_zstdCompression: - compressedfilesize = FIO_compressZstdFrame(prefs, fCtx, &ress, srcFileName, fileSize, compressionLevel, &readsize); + compressedfilesize = FIO_compressZstdFrame(fCtx, prefs, &ress, srcFileName, fileSize, compressionLevel, &readsize); break; case FIO_gzipCompression: @@ -1550,8 +1550,8 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs, * @return : 0 : compression completed correctly, * 1 : pb */ -static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs, - FIO_ctx_t* const fCtx, +static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, cRess_t ress, const char* dstFileName, const char* srcFileName, @@ -1579,7 +1579,7 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs, transfer_permissions = 1; } - result = FIO_compressFilename_internal(prefs, fCtx, ress, dstFileName, srcFileName, compressionLevel); + result = FIO_compressFilename_internal(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); if (closeDstFile) { FILE* const dstFile = ress.dstFile; @@ -1628,8 +1628,8 @@ static const char *compressedFileExtensions[] = { * 1 : missing or pb opening srcFileName */ static int -FIO_compressFilename_srcFile(FIO_prefs_t* const prefs, - FIO_ctx_t* const fCtx, +FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, cRess_t ress, const char* dstFileName, const char* srcFileName, @@ -1662,7 +1662,7 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs, ress.srcFile = FIO_openSrcFile(srcFileName); if (ress.srcFile == NULL) return 1; /* srcFile could not be opened */ - result = FIO_compressFilename_dstFile(prefs, fCtx, ress, dstFileName, srcFileName, compressionLevel); + result = FIO_compressFilename_dstFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); fclose(ress.srcFile); ress.srcFile = NULL; @@ -1680,12 +1680,12 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs, return result; } -int FIO_compressFilename(FIO_prefs_t* const prefs, FIO_ctx_t* const fCtx, const char* dstFileName, +int FIO_compressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, const char* dstFileName, const char* srcFileName, const char* dictFileName, int compressionLevel, ZSTD_compressionParameters comprParams) { cRess_t const ress = FIO_createCResources(prefs, dictFileName, UTIL_getFileSize(srcFileName), compressionLevel, comprParams); - int const result = FIO_compressFilename_srcFile(prefs, fCtx, ress, dstFileName, srcFileName, compressionLevel); + int const result = FIO_compressFilename_srcFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); #define DISPLAY_LEVEL_DEFAULT 2 @@ -1750,8 +1750,8 @@ static unsigned long long FIO_getLargestFileSize(const char** inFileNames, unsig * or into one file each (outFileName == NULL, but suffix != NULL), * or into a destination folder (specified with -O) */ -int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, - FIO_ctx_t* const fCtx, +int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, const char** inFileNamesTable, const char* outMirroredRootDirName, const char* outDirName, @@ -1768,7 +1768,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, /* init */ assert(outFileName != NULL || suffix != NULL); if (outFileName != NULL) { /* output into a single destination (stdout typically) */ - if (FIO_removeMultiFilesWarning(prefs, fCtx, 1 /* displayLevelCutoff */, outFileName)) { + if (FIO_removeMultiFilesWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) { FIO_freeCResources(ress); return 1; } @@ -1777,7 +1777,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, error = 1; } else { for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) { - status = FIO_compressFilename_srcFile(prefs, fCtx, ress, outFileName, inFileNamesTable[fCtx->currFileIdx], compressionLevel); + status = FIO_compressFilename_srcFile(fCtx, prefs, ress, outFileName, inFileNamesTable[fCtx->currFileIdx], compressionLevel); if (!status) fCtx->nbFilesProcessed++; error |= status; } @@ -1806,7 +1806,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, } else { dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix); /* cannot fail */ } - status = FIO_compressFilename_srcFile(prefs, fCtx, ress, dstFileName, srcFileName, compressionLevel); + status = FIO_compressFilename_srcFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); if (!status) fCtx->nbFilesProcessed++; error |= status; } @@ -2051,9 +2051,8 @@ FIO_zstdErrorHelp(const FIO_prefs_t* const prefs, */ #define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2)) static unsigned long long -FIO_decompressZstdFrame(dRess_t* ress, FILE* finput, +FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, const FIO_prefs_t* const prefs, - FIO_ctx_t* const fCtx, const char* srcFileName, U64 alreadyDecoded) /* for multi-frames streams */ { @@ -2354,9 +2353,9 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, * @return : 0 : OK * 1 : error */ -static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, +static int FIO_decompressFrames(FIO_ctx_t* const fCtx, + dRess_t ress, FILE* srcFile, const FIO_prefs_t* const prefs, - FIO_ctx_t* const fCtx, const char* dstFileName, const char* srcFileName) { unsigned readSomething = 0; @@ -2384,7 +2383,7 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, return 1; } if (ZSTD_isFrame(buf, ress.srcBufferLoaded)) { - unsigned long long const frameSize = FIO_decompressZstdFrame(&ress, srcFile, prefs, fCtx, srcFileName, filesize); + unsigned long long const frameSize = FIO_decompressZstdFrame(fCtx, &ress, srcFile, prefs, srcFileName, filesize); if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; filesize += frameSize; } else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */ @@ -2445,8 +2444,8 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, @return : 0 : OK 1 : operation aborted */ -static int FIO_decompressDstFile(FIO_prefs_t* const prefs, - FIO_ctx_t* const fCtx, +static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, dRess_t ress, FILE* srcFile, const char* dstFileName, const char* srcFileName) { @@ -2473,7 +2472,7 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs, transfer_permissions = 1; } - result = FIO_decompressFrames(ress, srcFile, prefs, fCtx, dstFileName, srcFileName); + result = FIO_decompressFrames(fCtx, ress, srcFile, prefs, dstFileName, srcFileName); if (releaseDstFile) { FILE* const dstFile = ress.dstFile; @@ -2502,7 +2501,7 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs, @return : 0 : OK 1 : error */ -static int FIO_decompressSrcFile(FIO_prefs_t* const prefs, FIO_ctx_t* const fCtx, dRess_t ress, const char* dstFileName, const char* srcFileName) +static int FIO_decompressSrcFile(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, dRess_t ress, const char* dstFileName, const char* srcFileName) { FILE* srcFile; int result; @@ -2516,7 +2515,7 @@ static int FIO_decompressSrcFile(FIO_prefs_t* const prefs, FIO_ctx_t* const fCtx if (srcFile==NULL) return 1; ress.srcBufferLoaded = 0; - result = FIO_decompressDstFile(prefs, fCtx, ress, srcFile, dstFileName, srcFileName); + result = FIO_decompressDstFile(fCtx, prefs, ress, srcFile, dstFileName, srcFileName); /* Close file */ if (fclose(srcFile)) { @@ -2540,13 +2539,13 @@ static int FIO_decompressSrcFile(FIO_prefs_t* const prefs, FIO_ctx_t* const fCtx -int FIO_decompressFilename(FIO_prefs_t* const prefs, FIO_ctx_t* const fCtx, +int FIO_decompressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, const char* dstFileName, const char* srcFileName, const char* dictFileName) { dRess_t const ress = FIO_createDResources(prefs, dictFileName); - int const decodingError = FIO_decompressSrcFile(prefs, fCtx, ress, dstFileName, srcFileName); + int const decodingError = FIO_decompressSrcFile(fCtx, prefs, ress, dstFileName, srcFileName); FIO_freeDResources(ress); return decodingError; @@ -2671,8 +2670,8 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName) } int -FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, - FIO_ctx_t* const fCtx, +FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, const char** srcNamesTable, const char* outMirroredRootDirName, const char* outDirName, const char* outFileName, @@ -2683,7 +2682,7 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, dRess_t ress = FIO_createDResources(prefs, dictFileName); if (outFileName) { - if (FIO_removeMultiFilesWarning(prefs, fCtx, 1 /* displayLevelCutoff */, outFileName)) { + if (FIO_removeMultiFilesWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) { FIO_freeDResources(ress); return 1; } @@ -2692,7 +2691,7 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, if (ress.dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName); } for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { - status = FIO_decompressSrcFile(prefs, fCtx, ress, outFileName, srcNamesTable[fCtx->currFileIdx]); + status = FIO_decompressSrcFile(fCtx, prefs, ress, outFileName, srcNamesTable[fCtx->currFileIdx]); if (!status) fCtx->nbFilesProcessed++; error |= status; } @@ -2718,7 +2717,7 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, dstFileName = FIO_determineDstName(srcFileName, outDirName); } if (dstFileName == NULL) { error=1; continue; } - status = FIO_decompressSrcFile(prefs, fCtx, ress, dstFileName, srcFileName); + status = FIO_decompressSrcFile(fCtx, prefs, ress, dstFileName, srcFileName); if (!status) fCtx->nbFilesProcessed++; error |= status; } diff --git a/programs/fileio.h b/programs/fileio.h index eeede526..bec651a1 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -112,14 +112,14 @@ void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value); ***************************************/ /** FIO_compressFilename() : * @return : 0 == ok; 1 == pb with src file. */ -int FIO_compressFilename (FIO_prefs_t* const prefs, FIO_ctx_t* const fCtx, +int FIO_compressFilename (FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, const char* outfilename, const char* infilename, const char* dictFileName, int compressionLevel, ZSTD_compressionParameters comprParams); /** FIO_decompressFilename() : * @return : 0 == ok; 1 == pb with src file. */ -int FIO_decompressFilename (FIO_prefs_t* const prefs, FIO_ctx_t* const fCtx, +int FIO_decompressFilename (FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, const char* outfilename, const char* infilename, const char* dictFileName); int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel); @@ -130,8 +130,8 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis ***************************************/ /** FIO_compressMultipleFilenames() : * @return : nb of missing files */ -int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, - FIO_ctx_t* const fCtx, +int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, const char** inFileNamesTable, const char* outMirroredDirName, const char* outDirName, @@ -141,8 +141,8 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, /** FIO_decompressMultipleFilenames() : * @return : nb of missing or skipped files */ -int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, - FIO_ctx_t* const fCtx, +int FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, const char** srcNamesTable, const char* outMirroredDirName, const char* outDirName, diff --git a/programs/zstdcli.c b/programs/zstdcli.c index c799cb78..da1d8a36 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -1320,9 +1320,9 @@ int main(int const argCount, const char* argv[]) } if ((filenames->tableSize==1) && outFileName) - operationResult = FIO_compressFilename(prefs, fCtx, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams); + operationResult = FIO_compressFilename(fCtx, prefs, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams); else - operationResult = FIO_compressMultipleFilenames(prefs, fCtx, filenames->fileNames, outMirroredDirName, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams); + operationResult = FIO_compressMultipleFilenames(fCtx, prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams); #else (void)contentSize; (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; (void)ZSTD_strategyMap; /* not used when ZSTD_NOCOMPRESS set */ DISPLAY("Compression not supported \n"); @@ -1330,9 +1330,9 @@ int main(int const argCount, const char* argv[]) } else { /* decompression or test */ #ifndef ZSTD_NODECOMPRESS if (filenames->tableSize == 1 && outFileName) { - operationResult = FIO_decompressFilename(prefs, fCtx, outFileName, filenames->fileNames[0], dictFileName); + operationResult = FIO_decompressFilename(fCtx, prefs, outFileName, filenames->fileNames[0], dictFileName); } else { - operationResult = FIO_decompressMultipleFilenames(prefs, fCtx, filenames->fileNames, outMirroredDirName, outDirName, outFileName, dictFileName); + operationResult = FIO_decompressMultipleFilenames(fCtx, prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, dictFileName); } #else DISPLAY("Decompression not supported \n");