AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #2283 from senhuang42/progress_bars_for_multiple_files

Browse Source 
Refreshing progress bar for processing multiple files
This commit is contained in:
Yann Collet 2020-08-26 11:54:50 -07:00 committed by GitHub
commit a8c66881e5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 76 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
programs/fileio.c
View File

@ -319,6 +319,8 @@ struct FIO_prefs_s {
int excludeCompressedFiles; int excludeCompressedFiles;
int patchFromMode; int patchFromMode;
int contentSize; int contentSize;
int currFileIdx;
int nbFiles;
}; };
@ -360,6 +362,8 @@ FIO_prefs_t* FIO_createPreferences(void)
ret->testMode = 0; ret->testMode = 0;
ret->literalCompressionMode = ZSTD_lcm_auto; ret->literalCompressionMode = ZSTD_lcm_auto;
ret->excludeCompressedFiles = 0; ret->excludeCompressedFiles = 0;
ret->nbFiles = 1;
ret->currFileIdx = 0;
return ret; return ret;
} }
@ -495,6 +499,16 @@ void FIO_setContentSize(FIO_prefs_t* const prefs, int value)
prefs->contentSize = value != 0; prefs->contentSize = value != 0;
} }
void FIO_setNbFiles(FIO_prefs_t* const prefs, int value)
{
prefs->nbFiles = value;
}
void FIO_setCurrFileIdx(FIO_prefs_t* const prefs, int value)
{
prefs->currFileIdx = value;
}
/*-************************************* /*-*************************************
* Functions * Functions
***************************************/ ***************************************/
@ -610,9 +624,10 @@ FIO_openDstFile(FIO_prefs_t* const prefs,
{ int ch = getchar(); { int ch = getchar();
if ((ch!='Y') && (ch!='y')) { if ((ch!='Y') && (ch!='y')) {
DISPLAY(" not overwritten \n"); DISPLAY(" not overwritten \n");
/* flush rest of input line */
while ((ch!=EOF) && (ch!='\n')) ch = getchar();
return NULL; return NULL;
} }
/* flush rest of input line */
while ((ch!=EOF) && (ch!='\n')) ch = getchar(); while ((ch!=EOF) && (ch!='\n')) ch = getchar();
} } } }
/* need to unlink */ /* need to unlink */
@ -1261,10 +1276,15 @@ FIO_compressZstdFrame(FIO_prefs_t* const prefs,
(unsigned)(zfp.produced >> 20), (unsigned)(zfp.produced >> 20),
cShare ); cShare );
} else { /* summarized notifications if == 2; */ } else { /* summarized notifications if == 2; */
DISPLAYLEVEL(2, "\rRead : %u ", (unsigned)(zfp.consumed >> 20)); if (prefs->nbFiles > 1) {
DISPLAYLEVEL(2, "\rCompressing %u/%u files. Current source: %s ", prefs->currFileIdx+1, prefs->nbFiles, srcFileName);
} else {
DISPLAYLEVEL(2, "\r");
}
DISPLAYLEVEL(2, "Read : %u ", (unsigned)(zfp.consumed >> 20));
if (fileSize != UTIL_FILESIZE_UNKNOWN) if (fileSize != UTIL_FILESIZE_UNKNOWN)
DISPLAYLEVEL(2, "/ %u ", (unsigned)(fileSize >> 20)); DISPLAYLEVEL(2, "/ %u ", (unsigned)(fileSize >> 20));
DISPLAYLEVEL(2, "MB ==> %2.f%% ", cShare); DISPLAYLEVEL(2, "MB ==> %2.f%%", cShare);
DELAY_NEXT_UPDATE(); DELAY_NEXT_UPDATE();
} }
@ -1427,18 +1447,21 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs,
} }
/* Status */ /* Status */
DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2, "\r%79s\r", "");
if (readsize == 0) { if (g_display_prefs.displayLevel > 2 || (g_display_prefs.displayLevel == 2 && prefs->nbFiles <= 1)) {
DISPLAYLEVEL(2,"%-20s : (%6llu => %6llu bytes, %s) \n", if (readsize == 0) {
srcFileName, DISPLAYLEVEL(2,"%-20s : (%6llu => %6llu bytes, %s) \n",
(unsigned long long)readsize, (unsigned long long) compressedfilesize, srcFileName,
dstFileName); (unsigned long long)readsize, (unsigned long long) compressedfilesize,
} else { dstFileName);
DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", } else {
srcFileName, DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n",
(double)compressedfilesize / readsize * 100, srcFileName,
(unsigned long long)readsize, (unsigned long long) compressedfilesize, (double)compressedfilesize / readsize * 100,
dstFileName); (unsigned long long)readsize, (unsigned long long) compressedfilesize,
dstFileName);
}
} }
/* Elapsed Time and CPU Load */ /* Elapsed Time and CPU Load */
@ -1662,7 +1685,7 @@ static unsigned long long FIO_getLargestFileSize(const char** inFileNames, unsig
* or into a destination folder (specified with -O) * or into a destination folder (specified with -O)
*/ */
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
const char** inFileNamesTable, unsigned nbFiles, const char** inFileNamesTable,
const char* outMirroredRootDirName, const char* outMirroredRootDirName,
const char* outDirName, const char* outDirName,
const char* outFileName, const char* suffix, const char* outFileName, const char* suffix,
@ -1671,7 +1694,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
{ {
int error = 0; int error = 0;
cRess_t ress = FIO_createCResources(prefs, dictFileName, cRess_t ress = FIO_createCResources(prefs, dictFileName,
FIO_getLargestFileSize(inFileNamesTable, nbFiles), FIO_getLargestFileSize(inFileNamesTable, prefs->nbFiles),
compressionLevel, comprParams); compressionLevel, comprParams);
/* init */ /* init */
@ -1681,21 +1704,20 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
if (ress.dstFile == NULL) { /* could not open outFileName */ if (ress.dstFile == NULL) { /* could not open outFileName */
error = 1; error = 1;
} else { } else {
unsigned u; for (; prefs->currFileIdx < prefs->nbFiles; ++prefs->currFileIdx) {
for (u=0; u<nbFiles; u++) error |= FIO_compressFilename_srcFile(prefs, ress, outFileName, inFileNamesTable[prefs->currFileIdx], compressionLevel);
error |= FIO_compressFilename_srcFile(prefs, ress, outFileName, inFileNamesTable[u], compressionLevel); }
if (fclose(ress.dstFile)) if (fclose(ress.dstFile))
EXM_THROW(29, "Write error (%s) : cannot properly close %s", EXM_THROW(29, "Write error (%s) : cannot properly close %s",
strerror(errno), outFileName); strerror(errno), outFileName);
ress.dstFile = NULL; ress.dstFile = NULL;
} }
} else { } else {
unsigned int u=0;
if (outMirroredRootDirName) if (outMirroredRootDirName)
UTIL_mirrorSourceFilesDirectories(inFileNamesTable, nbFiles, outMirroredRootDirName); UTIL_mirrorSourceFilesDirectories(inFileNamesTable, prefs->nbFiles, outMirroredRootDirName);
for (u=0; u<nbFiles; u++) { for (; prefs->currFileIdx < prefs->nbFiles; ++prefs->currFileIdx) {
const char* const srcFileName = inFileNamesTable[u]; const char* const srcFileName = inFileNamesTable[prefs->currFileIdx];
const char* dstFileName = NULL; const char* dstFileName = NULL;
if (outMirroredRootDirName) { if (outMirroredRootDirName) {
char* validMirroredDirName = UTIL_createMirroredDestDirName(srcFileName, outMirroredRootDirName); char* validMirroredDirName = UTIL_createMirroredDestDirName(srcFileName, outMirroredRootDirName);
@ -1710,12 +1732,11 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
} else { } else {
dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix); /* cannot fail */ dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix); /* cannot fail */
} }
error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
} }
if (outDirName) if (outDirName)
FIO_checkFilenameCollisions(inFileNamesTable ,nbFiles); FIO_checkFilenameCollisions(inFileNamesTable , prefs->nbFiles);
} }
FIO_freeCResources(ress); FIO_freeCResources(ress);
@ -1987,8 +2008,13 @@ FIO_decompressZstdFrame(dRess_t* ress, FILE* finput,
/* Write block */ /* Write block */
storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, prefs, storedSkips); storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, prefs, storedSkips);
frameSize += outBuff.pos; frameSize += outBuff.pos;
DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ", if (prefs->nbFiles > 1) {
srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); DISPLAYUPDATE(2, "\rDecompressing %u/%u files. Current source: %-20.20s : %u MB... ",
prefs->currFileIdx+1, prefs->nbFiles, srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) );
} else {
DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ",
srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) );
}
if (inBuff.pos > 0) { if (inBuff.pos > 0) {
memmove(ress->srcBuffer, (char*)ress->srcBuffer + inBuff.pos, inBuff.size - inBuff.pos); memmove(ress->srcBuffer, (char*)ress->srcBuffer + inBuff.pos, inBuff.size - inBuff.pos);
@ -2318,7 +2344,10 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile,
/* Final Status */ /* Final Status */
DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2, "\r%79s\r", "");
DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize); /* No status message in pipe mode (stdin - stdout) or multi-files mode */
if (g_display_prefs.displayLevel > 2 || (g_display_prefs.displayLevel == 2 && prefs->nbFiles <= 1)) {
DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize);
}
return 0; return 0;
} }
@ -2556,7 +2585,7 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName)
int int
FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
const char** srcNamesTable, unsigned nbFiles, const char** srcNamesTable,
const char* outMirroredRootDirName, const char* outMirroredRootDirName,
const char* outDirName, const char* outFileName, const char* outDirName, const char* outFileName,
const char* dictFileName) const char* dictFileName)
@ -2565,23 +2594,22 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
dRess_t ress = FIO_createDResources(prefs, dictFileName); dRess_t ress = FIO_createDResources(prefs, dictFileName);
if (outFileName) { if (outFileName) {
unsigned u;
if (!prefs->testMode) { if (!prefs->testMode) {
ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName); ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName);
if (ress.dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName); if (ress.dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName);
} }
for (u=0; u<nbFiles; u++) for (; prefs->currFileIdx < prefs->nbFiles; prefs->currFileIdx++) {
error |= FIO_decompressSrcFile(prefs, ress, outFileName, srcNamesTable[u]); error |= FIO_decompressSrcFile(prefs, ress, outFileName, srcNamesTable[prefs->currFileIdx]);
}
if ((!prefs->testMode) && (fclose(ress.dstFile))) if ((!prefs->testMode) && (fclose(ress.dstFile)))
EXM_THROW(72, "Write error : %s : cannot properly close output file", EXM_THROW(72, "Write error : %s : cannot properly close output file",
strerror(errno)); strerror(errno));
} else { } else {
unsigned int u = 0;
if (outMirroredRootDirName) if (outMirroredRootDirName)
UTIL_mirrorSourceFilesDirectories(srcNamesTable, nbFiles, outMirroredRootDirName); UTIL_mirrorSourceFilesDirectories(srcNamesTable, prefs->nbFiles, outMirroredRootDirName);
for (u=0; u<nbFiles; u++) { /* create dstFileName */ for (; prefs->currFileIdx < prefs->nbFiles; prefs->currFileIdx++) { /* create dstFileName */
const char* const srcFileName = srcNamesTable[u]; const char* const srcFileName = srcNamesTable[prefs->currFileIdx];
const char* dstFileName = NULL; const char* dstFileName = NULL;
if (outMirroredRootDirName) { if (outMirroredRootDirName) {
char* validMirroredDirName = UTIL_createMirroredDestDirName(srcFileName, outMirroredRootDirName); char* validMirroredDirName = UTIL_createMirroredDestDirName(srcFileName, outMirroredRootDirName);
@ -2598,7 +2626,7 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName); error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName);
} }
if (outDirName) if (outDirName)
FIO_checkFilenameCollisions(srcNamesTable ,nbFiles); FIO_checkFilenameCollisions(srcNamesTable , prefs->nbFiles);
} }
FIO_freeDResources(ress); FIO_freeDResources(ress);

6
programs/fileio.h
View File

@ -96,6 +96,8 @@ void FIO_setNotificationLevel(int level);
void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles); void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles);
void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value); void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value);
void FIO_setContentSize(FIO_prefs_t* const prefs, int value); void FIO_setContentSize(FIO_prefs_t* const prefs, int value);
void FIO_setNbFiles(FIO_prefs_t* const prefs, int value);
void FIO_setCurrFileIdx(FIO_prefs_t* const prefs, int value);
/*-************************************* /*-*************************************
* Single File functions * Single File functions
@ -121,7 +123,7 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
/** FIO_compressMultipleFilenames() : /** FIO_compressMultipleFilenames() :
* @return : nb of missing files */ * @return : nb of missing files */
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
const char** inFileNamesTable, unsigned nbFiles, const char** inFileNamesTable,
const char* outMirroredDirName, const char* outMirroredDirName,
const char* outDirName, const char* outDirName,
const char* outFileName, const char* suffix, const char* outFileName, const char* suffix,
@ -131,7 +133,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
/** FIO_decompressMultipleFilenames() : /** FIO_decompressMultipleFilenames() :
* @return : nb of missing or skipped files */ * @return : nb of missing or skipped files */
int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs, int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
const char** srcNamesTable, unsigned nbFiles, const char** srcNamesTable,
const char* outMirroredDirName, const char* outMirroredDirName,
const char* outDirName, const char* outDirName,
const char* outFileName, const char* outFileName,

12
programs/zstdcli.c
View File

@ -1255,12 +1255,12 @@ int main(int const argCount, const char* argv[])
DISPLAY("error : can't use --patch-from=# on multiple files \n"); DISPLAY("error : can't use --patch-from=# on multiple files \n");
CLEAN_RETURN(1); CLEAN_RETURN(1);
} }
/* No status message in pipe mode (stdin - stdout) or multi-files mode */ /* No status message in pipe mode (stdin - stdout) */
if (!strcmp(filenames->fileNames[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (g_displayLevel==2)) g_displayLevel=1; if (!strcmp(filenames->fileNames[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (g_displayLevel==2)) g_displayLevel=1;
if ((filenames->tableSize > 1) & (g_displayLevel==2)) g_displayLevel=1;
/* IO Stream/File */ /* IO Stream/File */
FIO_setNbFiles(prefs, (int)filenames->tableSize);
FIO_setNotificationLevel(g_displayLevel); FIO_setNotificationLevel(g_displayLevel);
FIO_setPatchFromMode(prefs, patchFromDictFileName != NULL); FIO_setPatchFromMode(prefs, patchFromDictFileName != NULL);
if (memLimit == 0) { if (memLimit == 0) {
@ -1319,9 +1319,9 @@ int main(int const argCount, const char* argv[])
} }
if ((filenames->tableSize==1) && outFileName) if ((filenames->tableSize==1) && outFileName)
operationResult = FIO_compressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams); operationResult = FIO_compressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams);
else else
operationResult = FIO_compressMultipleFilenames(prefs, filenames->fileNames, (unsigned)filenames->tableSize, outMirroredDirName, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams); operationResult = FIO_compressMultipleFilenames(prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams);
#else #else
(void)contentSize; (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; (void)ZSTD_strategyMap; /* not used when ZSTD_NOCOMPRESS set */ (void)contentSize; (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; (void)ZSTD_strategyMap; /* not used when ZSTD_NOCOMPRESS set */
DISPLAY("Compression not supported \n"); DISPLAY("Compression not supported \n");
@ -1331,7 +1331,7 @@ int main(int const argCount, const char* argv[])
if (filenames->tableSize == 1 && outFileName) { if (filenames->tableSize == 1 && outFileName) {
operationResult = FIO_decompressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName); operationResult = FIO_decompressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName);
} else { } else {
operationResult = FIO_decompressMultipleFilenames(prefs, filenames->fileNames, (unsigned)filenames->tableSize, outMirroredDirName, outDirName, outFileName, dictFileName); operationResult = FIO_decompressMultipleFilenames(prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, dictFileName);
} }
#else #else
DISPLAY("Decompression not supported \n"); DISPLAY("Decompression not supported \n");

2
tests/playTests.sh
View File

@ -368,7 +368,7 @@ zstd tmp1.zst tmp2.zst -o "$INTOVOID" -f
zstd -d tmp1.zst tmp2.zst -o tmp zstd -d tmp1.zst tmp2.zst -o tmp
touch tmpexists touch tmpexists
zstd tmp1 tmp2 -f -o tmpexists zstd tmp1 tmp2 -f -o tmpexists
zstd tmp1 tmp2 -o tmpexists && die "should have refused to overwrite" zstd tmp1 tmp2 -q -o tmpexists && die "should have refused to overwrite"
# Bug: PR #972 # Bug: PR #972
if [ "$?" -eq 139 ]; then if [ "$?" -eq 139 ]; then
die "should not have segfaulted" die "should not have segfaulted"