Merge pull request #2292 from senhuang42/multifile_status_update_fio_refactor
Make multifile (de)compression print out a summary
This commit is contained in:
Yann Collet
GitHub
commit
05622992d5
@ -319,8 +319,23 @@ struct FIO_prefs_s {
|
||||
int excludeCompressedFiles;
|
||||
int patchFromMode;
|
||||
int contentSize;
|
||||
};
|
||||
|
||||
/*-*************************************
|
||||
* Parameters: FIO_ctx_t
|
||||
***************************************/
|
||||
|
||||
/* typedef'd to FIO_ctx_t within fileio.h */
|
||||
struct FIO_ctx_s {
|
||||
|
||||
/* file i/o info */
|
||||
int nbFilesTotal;
|
||||
|
||||
/* file i/o state */
|
||||
int currFileIdx;
|
||||
int nbFiles;
|
||||
int nbFilesProcessed;
|
||||
size_t totalBytesInput;
|
||||
size_t totalBytesOutput;
|
||||
};
|
||||
|
||||
|
||||
@ -362,8 +377,19 @@ FIO_prefs_t* FIO_createPreferences(void)
|
||||
ret->testMode = 0;
|
||||
ret->literalCompressionMode = ZSTD_lcm_auto;
|
||||
ret->excludeCompressedFiles = 0;
|
||||
ret->nbFiles = 1;
|
||||
return ret;
|
||||
}
|
||||
|
||||
FIO_ctx_t* FIO_createContext(void)
|
||||
{
|
||||
FIO_ctx_t* const ret = (FIO_ctx_t*)malloc(sizeof(FIO_ctx_t));
|
||||
if (!ret) EXM_THROW(21, "Allocation error : not enough memory");
|
||||
|
||||
ret->currFileIdx = 0;
|
||||
ret->nbFilesTotal = 1;
|
||||
ret->nbFilesProcessed = 0;
|
||||
ret->totalBytesInput = 0;
|
||||
ret->totalBytesOutput = 0;
|
||||
return ret;
|
||||
}
|
||||
|
||||
@ -372,6 +398,11 @@ void FIO_freePreferences(FIO_prefs_t* const prefs)
|
||||
free(prefs);
|
||||
}
|
||||
|
||||
void FIO_freeContext(FIO_ctx_t* const fCtx)
|
||||
{
|
||||
free(fCtx);
|
||||
}
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Parameters: Display Options
|
||||
@ -386,6 +417,8 @@ void FIO_setNoProgress(unsigned noProgress) { g_display_prefs.noProgress = noPro
|
||||
* Parameters: Setters
|
||||
***************************************/
|
||||
|
||||
/* FIO_prefs_t functions */
|
||||
|
||||
void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t compressionType) { prefs->compressionType = compressionType; }
|
||||
|
||||
void FIO_overwriteMode(FIO_prefs_t* const prefs) { prefs->overwrite = 1; }
|
||||
@ -499,14 +532,11 @@ void FIO_setContentSize(FIO_prefs_t* const prefs, int value)
|
||||
prefs->contentSize = value != 0;
|
||||
}
|
||||
|
||||
void FIO_setNbFiles(FIO_prefs_t* const prefs, int value)
|
||||
{
|
||||
prefs->nbFiles = value;
|
||||
}
|
||||
/* FIO_ctx_t functions */
|
||||
|
||||
void FIO_setCurrFileIdx(FIO_prefs_t* const prefs, int value)
|
||||
void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value)
|
||||
{
|
||||
prefs->currFileIdx = value;
|
||||
fCtx->nbFilesTotal = value;
|
||||
}
|
||||
|
||||
/*-*************************************
|
||||
@ -800,10 +830,10 @@ static void FIO_adjustMemLimitForPatchFromMode(FIO_prefs_t* const prefs,
|
||||
* If neither flag is specified, zstd will prompt the user for confirmation to proceed.
|
||||
* If --rm is not specified, then zstd will print a warning to the user (which can be silenced with -q).
|
||||
*/
|
||||
static int FIO_removeMultiFilesWarning(const FIO_prefs_t* const prefs, int displayLevelCutoff, const char* outFileName)
|
||||
static int FIO_removeMultiFilesWarning(FIO_ctx_t* const fCtx, const FIO_prefs_t* const prefs, const char* outFileName, int displayLevelCutoff)
|
||||
{
|
||||
int error = 0;
|
||||
if (prefs->nbFiles > 1 && !prefs->overwrite) {
|
||||
if (fCtx->nbFilesTotal > 1 && !prefs->overwrite) {
|
||||
if (g_display_prefs.displayLevel <= displayLevelCutoff) {
|
||||
if (prefs->removeSrcFile) {
|
||||
DISPLAYLEVEL(1, "zstd: Aborting... not deleting files and processing into dst: %s", outFileName);
|
||||
@ -1223,7 +1253,8 @@ FIO_compressLz4Frame(cRess_t* ress,
|
||||
|
||||
|
||||
static unsigned long long
|
||||
FIO_compressZstdFrame(FIO_prefs_t* const prefs,
|
||||
FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
|
||||
FIO_prefs_t* const prefs,
|
||||
const cRess_t* ressPtr,
|
||||
const char* srcFileName, U64 fileSize,
|
||||
int compressionLevel, U64* readsize)
|
||||
@ -1305,9 +1336,9 @@ FIO_compressZstdFrame(FIO_prefs_t* const prefs,
|
||||
(unsigned)(zfp.consumed >> 20),
|
||||
(unsigned)(zfp.produced >> 20),
|
||||
cShare );
|
||||
} else { /* summarized notifications if == 2; */
|
||||
if (prefs->nbFiles > 1) {
|
||||
DISPLAYLEVEL(2, "\rCompressing %u/%u files. Current source: %s ", prefs->currFileIdx+1, prefs->nbFiles, srcFileName);
|
||||
} else { /* summarized notifications if == 2 */
|
||||
if (fCtx->nbFilesTotal > 1) {
|
||||
DISPLAYLEVEL(2, "\rCompressing %u/%u files. Current source: %s ", fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName);
|
||||
} else {
|
||||
DISPLAYLEVEL(2, "\r");
|
||||
}
|
||||
@ -1425,7 +1456,8 @@ FIO_compressZstdFrame(FIO_prefs_t* const prefs,
|
||||
* 1 : missing or pb opening srcFileName
|
||||
*/
|
||||
static int
|
||||
FIO_compressFilename_internal(FIO_prefs_t* const prefs,
|
||||
FIO_compressFilename_internal(FIO_ctx_t* const fCtx,
|
||||
FIO_prefs_t* const prefs,
|
||||
cRess_t ress,
|
||||
const char* dstFileName, const char* srcFileName,
|
||||
int compressionLevel)
|
||||
@ -1441,7 +1473,7 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs,
|
||||
switch (prefs->compressionType) {
|
||||
default:
|
||||
case FIO_zstdCompression:
|
||||
compressedfilesize = FIO_compressZstdFrame(prefs, &ress, srcFileName, fileSize, compressionLevel, &readsize);
|
||||
compressedfilesize = FIO_compressZstdFrame(fCtx, prefs, &ress, srcFileName, fileSize, compressionLevel, &readsize);
|
||||
break;
|
||||
|
||||
case FIO_gzipCompression:
|
||||
@ -1477,20 +1509,23 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs,
|
||||
}
|
||||
|
||||
/* Status */
|
||||
|
||||
fCtx->totalBytesInput += (size_t)readsize;
|
||||
fCtx->totalBytesOutput += (size_t)compressedfilesize;
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
if (g_display_prefs.displayLevel > 2 || (g_display_prefs.displayLevel == 2 && prefs->nbFiles <= 1)) {
|
||||
if (readsize == 0) {
|
||||
DISPLAYLEVEL(2,"%-20s : (%6llu => %6llu bytes, %s) \n",
|
||||
srcFileName,
|
||||
(unsigned long long)readsize, (unsigned long long) compressedfilesize,
|
||||
dstFileName);
|
||||
} else {
|
||||
DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n",
|
||||
srcFileName,
|
||||
(double)compressedfilesize / readsize * 100,
|
||||
(unsigned long long)readsize, (unsigned long long) compressedfilesize,
|
||||
dstFileName);
|
||||
if (g_display_prefs.displayLevel >= 2) {
|
||||
if (g_display_prefs.displayLevel >= 3 || fCtx->nbFilesTotal <= 1) {
|
||||
if (readsize == 0) {
|
||||
DISPLAYLEVEL(2,"%-20s : (%6llu => %6llu bytes, %s) \n",
|
||||
srcFileName,
|
||||
(unsigned long long)readsize, (unsigned long long) compressedfilesize,
|
||||
dstFileName);
|
||||
} else {
|
||||
DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n",
|
||||
srcFileName,
|
||||
(double)compressedfilesize / readsize * 100,
|
||||
(unsigned long long)readsize, (unsigned long long) compressedfilesize,
|
||||
dstFileName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1516,7 +1551,8 @@ FIO_compressFilename_internal(FIO_prefs_t* const prefs,
|
||||
* @return : 0 : compression completed correctly,
|
||||
* 1 : pb
|
||||
*/
|
||||
static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs,
|
||||
static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx,
|
||||
FIO_prefs_t* const prefs,
|
||||
cRess_t ress,
|
||||
const char* dstFileName,
|
||||
const char* srcFileName,
|
||||
@ -1544,7 +1580,7 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs,
|
||||
transfer_permissions = 1;
|
||||
}
|
||||
|
||||
result = FIO_compressFilename_internal(prefs, ress, dstFileName, srcFileName, compressionLevel);
|
||||
result = FIO_compressFilename_internal(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel);
|
||||
|
||||
if (closeDstFile) {
|
||||
FILE* const dstFile = ress.dstFile;
|
||||
@ -1593,7 +1629,8 @@ static const char *compressedFileExtensions[] = {
|
||||
* 1 : missing or pb opening srcFileName
|
||||
*/
|
||||
static int
|
||||
FIO_compressFilename_srcFile(FIO_prefs_t* const prefs,
|
||||
FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx,
|
||||
FIO_prefs_t* const prefs,
|
||||
cRess_t ress,
|
||||
const char* dstFileName,
|
||||
const char* srcFileName,
|
||||
@ -1626,7 +1663,7 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs,
|
||||
ress.srcFile = FIO_openSrcFile(srcFileName);
|
||||
if (ress.srcFile == NULL) return 1; /* srcFile could not be opened */
|
||||
|
||||
result = FIO_compressFilename_dstFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
|
||||
result = FIO_compressFilename_dstFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel);
|
||||
|
||||
fclose(ress.srcFile);
|
||||
ress.srcFile = NULL;
|
||||
@ -1644,12 +1681,12 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs,
|
||||
return result;
|
||||
}
|
||||
|
||||
int FIO_compressFilename(FIO_prefs_t* const prefs, const char* dstFileName,
|
||||
int FIO_compressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, const char* dstFileName,
|
||||
const char* srcFileName, const char* dictFileName,
|
||||
int compressionLevel, ZSTD_compressionParameters comprParams)
|
||||
{
|
||||
cRess_t const ress = FIO_createCResources(prefs, dictFileName, UTIL_getFileSize(srcFileName), compressionLevel, comprParams);
|
||||
int const result = FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
|
||||
int const result = FIO_compressFilename_srcFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel);
|
||||
|
||||
#define DISPLAY_LEVEL_DEFAULT 2
|
||||
|
||||
@ -1714,7 +1751,8 @@ static unsigned long long FIO_getLargestFileSize(const char** inFileNames, unsig
|
||||
* or into one file each (outFileName == NULL, but suffix != NULL),
|
||||
* or into a destination folder (specified with -O)
|
||||
*/
|
||||
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
|
||||
int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
|
||||
FIO_prefs_t* const prefs,
|
||||
const char** inFileNamesTable,
|
||||
const char* outMirroredRootDirName,
|
||||
const char* outDirName,
|
||||
@ -1722,15 +1760,16 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
|
||||
const char* dictFileName, int compressionLevel,
|
||||
ZSTD_compressionParameters comprParams)
|
||||
{
|
||||
int status;
|
||||
int error = 0;
|
||||
cRess_t ress = FIO_createCResources(prefs, dictFileName,
|
||||
FIO_getLargestFileSize(inFileNamesTable, prefs->nbFiles),
|
||||
FIO_getLargestFileSize(inFileNamesTable, fCtx->nbFilesTotal),
|
||||
compressionLevel, comprParams);
|
||||
|
||||
/* init */
|
||||
assert(outFileName != NULL || suffix != NULL);
|
||||
if (outFileName != NULL) { /* output into a single destination (stdout typically) */
|
||||
if (FIO_removeMultiFilesWarning(prefs, 1 /* displayLevelCutoff */, outFileName)) {
|
||||
if (FIO_removeMultiFilesWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) {
|
||||
FIO_freeCResources(ress);
|
||||
return 1;
|
||||
}
|
||||
@ -1738,8 +1777,10 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
|
||||
if (ress.dstFile == NULL) { /* could not open outFileName */
|
||||
error = 1;
|
||||
} else {
|
||||
for (; prefs->currFileIdx < prefs->nbFiles; ++prefs->currFileIdx) {
|
||||
error |= FIO_compressFilename_srcFile(prefs, ress, outFileName, inFileNamesTable[prefs->currFileIdx], compressionLevel);
|
||||
for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) {
|
||||
status = FIO_compressFilename_srcFile(fCtx, prefs, ress, outFileName, inFileNamesTable[fCtx->currFileIdx], compressionLevel);
|
||||
if (!status) fCtx->nbFilesProcessed++;
|
||||
error |= status;
|
||||
}
|
||||
if (fclose(ress.dstFile))
|
||||
EXM_THROW(29, "Write error (%s) : cannot properly close %s",
|
||||
@ -1748,10 +1789,10 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
|
||||
}
|
||||
} else {
|
||||
if (outMirroredRootDirName)
|
||||
UTIL_mirrorSourceFilesDirectories(inFileNamesTable, prefs->nbFiles, outMirroredRootDirName);
|
||||
UTIL_mirrorSourceFilesDirectories(inFileNamesTable, fCtx->nbFilesTotal, outMirroredRootDirName);
|
||||
|
||||
for (; prefs->currFileIdx < prefs->nbFiles; ++prefs->currFileIdx) {
|
||||
const char* const srcFileName = inFileNamesTable[prefs->currFileIdx];
|
||||
for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) {
|
||||
const char* const srcFileName = inFileNamesTable[fCtx->currFileIdx];
|
||||
const char* dstFileName = NULL;
|
||||
if (outMirroredRootDirName) {
|
||||
char* validMirroredDirName = UTIL_createMirroredDestDirName(srcFileName, outMirroredRootDirName);
|
||||
@ -1766,13 +1807,20 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
|
||||
} else {
|
||||
dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix); /* cannot fail */
|
||||
}
|
||||
error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
|
||||
status = FIO_compressFilename_srcFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel);
|
||||
if (!status) fCtx->nbFilesProcessed++;
|
||||
error |= status;
|
||||
}
|
||||
|
||||
if (outDirName)
|
||||
FIO_checkFilenameCollisions(inFileNamesTable , prefs->nbFiles);
|
||||
FIO_checkFilenameCollisions(inFileNamesTable , fCtx->nbFilesTotal);
|
||||
}
|
||||
|
||||
if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesInput != 0)
|
||||
DISPLAYLEVEL(2, "%d files compressed : %.2f%% (%6zu => %6zu bytes)\n", fCtx->nbFilesProcessed,
|
||||
(double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100,
|
||||
fCtx->totalBytesInput, fCtx->totalBytesOutput);
|
||||
|
||||
FIO_freeCResources(ress);
|
||||
return error;
|
||||
}
|
||||
@ -2004,7 +2052,7 @@ FIO_zstdErrorHelp(const FIO_prefs_t* const prefs,
|
||||
*/
|
||||
#define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2))
|
||||
static unsigned long long
|
||||
FIO_decompressZstdFrame(dRess_t* ress, FILE* finput,
|
||||
FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput,
|
||||
const FIO_prefs_t* const prefs,
|
||||
const char* srcFileName,
|
||||
U64 alreadyDecoded) /* for multi-frames streams */
|
||||
@ -2042,9 +2090,9 @@ FIO_decompressZstdFrame(dRess_t* ress, FILE* finput,
|
||||
/* Write block */
|
||||
storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, prefs, storedSkips);
|
||||
frameSize += outBuff.pos;
|
||||
if (prefs->nbFiles > 1) {
|
||||
if (fCtx->nbFilesTotal > 1) {
|
||||
DISPLAYUPDATE(2, "\rDecompressing %u/%u files. Current source: %-20.20s : %u MB... ",
|
||||
prefs->currFileIdx+1, prefs->nbFiles, srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) );
|
||||
fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) );
|
||||
} else {
|
||||
DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ",
|
||||
srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) );
|
||||
@ -2306,7 +2354,8 @@ FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile,
|
||||
* @return : 0 : OK
|
||||
* 1 : error
|
||||
*/
|
||||
static int FIO_decompressFrames(dRess_t ress, FILE* srcFile,
|
||||
static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
|
||||
dRess_t ress, FILE* srcFile,
|
||||
const FIO_prefs_t* const prefs,
|
||||
const char* dstFileName, const char* srcFileName)
|
||||
{
|
||||
@ -2335,7 +2384,7 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile,
|
||||
return 1;
|
||||
}
|
||||
if (ZSTD_isFrame(buf, ress.srcBufferLoaded)) {
|
||||
unsigned long long const frameSize = FIO_decompressZstdFrame(&ress, srcFile, prefs, srcFileName, filesize);
|
||||
unsigned long long const frameSize = FIO_decompressZstdFrame(fCtx, &ress, srcFile, prefs, srcFileName, filesize);
|
||||
if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
|
||||
filesize += frameSize;
|
||||
} else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */
|
||||
@ -2377,10 +2426,13 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile,
|
||||
} } /* for each frame */
|
||||
|
||||
/* Final Status */
|
||||
fCtx->totalBytesOutput += (size_t)filesize;
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
/* No status message in pipe mode (stdin - stdout) or multi-files mode */
|
||||
if (g_display_prefs.displayLevel > 2 || (g_display_prefs.displayLevel == 2 && prefs->nbFiles <= 1)) {
|
||||
if (g_display_prefs.displayLevel >= 2) {
|
||||
if (fCtx->nbFilesTotal <= 1 || g_display_prefs.displayLevel >= 3) {
|
||||
DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
@ -2393,7 +2445,8 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile,
|
||||
@return : 0 : OK
|
||||
1 : operation aborted
|
||||
*/
|
||||
static int FIO_decompressDstFile(FIO_prefs_t* const prefs,
|
||||
static int FIO_decompressDstFile(FIO_ctx_t* const fCtx,
|
||||
FIO_prefs_t* const prefs,
|
||||
dRess_t ress, FILE* srcFile,
|
||||
const char* dstFileName, const char* srcFileName)
|
||||
{
|
||||
@ -2420,7 +2473,7 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs,
|
||||
transfer_permissions = 1;
|
||||
}
|
||||
|
||||
result = FIO_decompressFrames(ress, srcFile, prefs, dstFileName, srcFileName);
|
||||
result = FIO_decompressFrames(fCtx, ress, srcFile, prefs, dstFileName, srcFileName);
|
||||
|
||||
if (releaseDstFile) {
|
||||
FILE* const dstFile = ress.dstFile;
|
||||
@ -2449,7 +2502,7 @@ static int FIO_decompressDstFile(FIO_prefs_t* const prefs,
|
||||
@return : 0 : OK
|
||||
1 : error
|
||||
*/
|
||||
static int FIO_decompressSrcFile(FIO_prefs_t* const prefs, dRess_t ress, const char* dstFileName, const char* srcFileName)
|
||||
static int FIO_decompressSrcFile(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, dRess_t ress, const char* dstFileName, const char* srcFileName)
|
||||
{
|
||||
FILE* srcFile;
|
||||
int result;
|
||||
@ -2463,7 +2516,7 @@ static int FIO_decompressSrcFile(FIO_prefs_t* const prefs, dRess_t ress, const c
|
||||
if (srcFile==NULL) return 1;
|
||||
ress.srcBufferLoaded = 0;
|
||||
|
||||
result = FIO_decompressDstFile(prefs, ress, srcFile, dstFileName, srcFileName);
|
||||
result = FIO_decompressDstFile(fCtx, prefs, ress, srcFile, dstFileName, srcFileName);
|
||||
|
||||
/* Close file */
|
||||
if (fclose(srcFile)) {
|
||||
@ -2487,13 +2540,13 @@ static int FIO_decompressSrcFile(FIO_prefs_t* const prefs, dRess_t ress, const c
|
||||
|
||||
|
||||
|
||||
int FIO_decompressFilename(FIO_prefs_t* const prefs,
|
||||
int FIO_decompressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs,
|
||||
const char* dstFileName, const char* srcFileName,
|
||||
const char* dictFileName)
|
||||
{
|
||||
dRess_t const ress = FIO_createDResources(prefs, dictFileName);
|
||||
|
||||
int const decodingError = FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName);
|
||||
int const decodingError = FIO_decompressSrcFile(fCtx, prefs, ress, dstFileName, srcFileName);
|
||||
|
||||
FIO_freeDResources(ress);
|
||||
return decodingError;
|
||||
@ -2618,17 +2671,19 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName)
|
||||
}
|
||||
|
||||
int
|
||||
FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
|
||||
FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx,
|
||||
FIO_prefs_t* const prefs,
|
||||
const char** srcNamesTable,
|
||||
const char* outMirroredRootDirName,
|
||||
const char* outDirName, const char* outFileName,
|
||||
const char* dictFileName)
|
||||
{
|
||||
int status;
|
||||
int error = 0;
|
||||
dRess_t ress = FIO_createDResources(prefs, dictFileName);
|
||||
|
||||
if (outFileName) {
|
||||
if (FIO_removeMultiFilesWarning(prefs, 1 /* displayLevelCutoff */, outFileName)) {
|
||||
if (FIO_removeMultiFilesWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) {
|
||||
FIO_freeDResources(ress);
|
||||
return 1;
|
||||
}
|
||||
@ -2636,18 +2691,20 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
|
||||
ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName);
|
||||
if (ress.dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName);
|
||||
}
|
||||
for (; prefs->currFileIdx < prefs->nbFiles; prefs->currFileIdx++) {
|
||||
error |= FIO_decompressSrcFile(prefs, ress, outFileName, srcNamesTable[prefs->currFileIdx]);
|
||||
for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) {
|
||||
status = FIO_decompressSrcFile(fCtx, prefs, ress, outFileName, srcNamesTable[fCtx->currFileIdx]);
|
||||
if (!status) fCtx->nbFilesProcessed++;
|
||||
error |= status;
|
||||
}
|
||||
if ((!prefs->testMode) && (fclose(ress.dstFile)))
|
||||
EXM_THROW(72, "Write error : %s : cannot properly close output file",
|
||||
strerror(errno));
|
||||
} else {
|
||||
if (outMirroredRootDirName)
|
||||
UTIL_mirrorSourceFilesDirectories(srcNamesTable, prefs->nbFiles, outMirroredRootDirName);
|
||||
UTIL_mirrorSourceFilesDirectories(srcNamesTable, fCtx->nbFilesTotal, outMirroredRootDirName);
|
||||
|
||||
for (; prefs->currFileIdx < prefs->nbFiles; prefs->currFileIdx++) { /* create dstFileName */
|
||||
const char* const srcFileName = srcNamesTable[prefs->currFileIdx];
|
||||
for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { /* create dstFileName */
|
||||
const char* const srcFileName = srcNamesTable[fCtx->currFileIdx];
|
||||
const char* dstFileName = NULL;
|
||||
if (outMirroredRootDirName) {
|
||||
char* validMirroredDirName = UTIL_createMirroredDestDirName(srcFileName, outMirroredRootDirName);
|
||||
@ -2661,11 +2718,16 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
|
||||
dstFileName = FIO_determineDstName(srcFileName, outDirName);
|
||||
}
|
||||
if (dstFileName == NULL) { error=1; continue; }
|
||||
error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName);
|
||||
status = FIO_decompressSrcFile(fCtx, prefs, ress, dstFileName, srcFileName);
|
||||
if (!status) fCtx->nbFilesProcessed++;
|
||||
error |= status;
|
||||
}
|
||||
if (outDirName)
|
||||
FIO_checkFilenameCollisions(srcNamesTable , prefs->nbFiles);
|
||||
FIO_checkFilenameCollisions(srcNamesTable , fCtx->nbFilesTotal);
|
||||
}
|
||||
|
||||
if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesInput != 0)
|
||||
DISPLAYLEVEL(2, "%d files decompressed : %6zu bytes total \n", fCtx->nbFilesProcessed, fCtx->totalBytesOutput);
|
||||
|
||||
FIO_freeDResources(ress);
|
||||
return error;
|
||||
|
||||
@ -59,11 +59,18 @@ typedef struct FIO_prefs_s FIO_prefs_t;
|
||||
FIO_prefs_t* FIO_createPreferences(void);
|
||||
void FIO_freePreferences(FIO_prefs_t* const prefs);
|
||||
|
||||
/* Mutable struct containing relevant context and state regarding (de)compression with respect to file I/O */
|
||||
typedef struct FIO_ctx_s FIO_ctx_t;
|
||||
|
||||
FIO_ctx_t* FIO_createContext(void);
|
||||
void FIO_freeContext(FIO_ctx_t* const fCtx);
|
||||
|
||||
typedef struct FIO_display_prefs_s FIO_display_prefs_t;
|
||||
|
||||
/*-*************************************
|
||||
* Parameters
|
||||
***************************************/
|
||||
/* FIO_prefs_t functions */
|
||||
void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t compressionType);
|
||||
void FIO_overwriteMode(FIO_prefs_t* const prefs);
|
||||
void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt);
|
||||
@ -96,22 +103,23 @@ void FIO_setNotificationLevel(int level);
|
||||
void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles);
|
||||
void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value);
|
||||
void FIO_setContentSize(FIO_prefs_t* const prefs, int value);
|
||||
void FIO_setNbFiles(FIO_prefs_t* const prefs, int value);
|
||||
void FIO_setCurrFileIdx(FIO_prefs_t* const prefs, int value);
|
||||
|
||||
/* FIO_ctx_t functions */
|
||||
void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value);
|
||||
|
||||
/*-*************************************
|
||||
* Single File functions
|
||||
***************************************/
|
||||
/** FIO_compressFilename() :
|
||||
* @return : 0 == ok; 1 == pb with src file. */
|
||||
int FIO_compressFilename (FIO_prefs_t* const prefs,
|
||||
int FIO_compressFilename (FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs,
|
||||
const char* outfilename, const char* infilename,
|
||||
const char* dictFileName, int compressionLevel,
|
||||
ZSTD_compressionParameters comprParams);
|
||||
|
||||
/** FIO_decompressFilename() :
|
||||
* @return : 0 == ok; 1 == pb with src file. */
|
||||
int FIO_decompressFilename (FIO_prefs_t* const prefs,
|
||||
int FIO_decompressFilename (FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs,
|
||||
const char* outfilename, const char* infilename, const char* dictFileName);
|
||||
|
||||
int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel);
|
||||
@ -122,7 +130,8 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
|
||||
***************************************/
|
||||
/** FIO_compressMultipleFilenames() :
|
||||
* @return : nb of missing files */
|
||||
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
|
||||
int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
|
||||
FIO_prefs_t* const prefs,
|
||||
const char** inFileNamesTable,
|
||||
const char* outMirroredDirName,
|
||||
const char* outDirName,
|
||||
@ -132,7 +141,8 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
|
||||
|
||||
/** FIO_decompressMultipleFilenames() :
|
||||
* @return : nb of missing or skipped files */
|
||||
int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
|
||||
int FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx,
|
||||
FIO_prefs_t* const prefs,
|
||||
const char** srcNamesTable,
|
||||
const char* outMirroredDirName,
|
||||
const char* outDirName,
|
||||
|
||||
@ -717,6 +717,7 @@ int main(int const argCount, const char* argv[])
|
||||
size_t blockSize = 0;
|
||||
|
||||
FIO_prefs_t* const prefs = FIO_createPreferences();
|
||||
FIO_ctx_t* const fCtx = FIO_createContext();
|
||||
zstd_operation_mode operation = zom_compress;
|
||||
ZSTD_compressionParameters compressionParams;
|
||||
int cLevel = init_cLevel();
|
||||
@ -1284,7 +1285,7 @@ int main(int const argCount, const char* argv[])
|
||||
if (!strcmp(filenames->fileNames[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (g_displayLevel==2)) g_displayLevel=1;
|
||||
|
||||
/* IO Stream/File */
|
||||
FIO_setNbFiles(prefs, (int)filenames->tableSize);
|
||||
FIO_setNbFilesTotal(fCtx, (int)filenames->tableSize);
|
||||
FIO_setNotificationLevel(g_displayLevel);
|
||||
FIO_setPatchFromMode(prefs, patchFromDictFileName != NULL);
|
||||
if (memLimit == 0) {
|
||||
@ -1343,9 +1344,9 @@ int main(int const argCount, const char* argv[])
|
||||
}
|
||||
|
||||
if ((filenames->tableSize==1) && outFileName)
|
||||
operationResult = FIO_compressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams);
|
||||
operationResult = FIO_compressFilename(fCtx, prefs, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams);
|
||||
else
|
||||
operationResult = FIO_compressMultipleFilenames(prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams);
|
||||
operationResult = FIO_compressMultipleFilenames(fCtx, prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams);
|
||||
#else
|
||||
(void)contentSize; (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; (void)ZSTD_strategyMap; /* not used when ZSTD_NOCOMPRESS set */
|
||||
DISPLAY("Compression not supported \n");
|
||||
@ -1353,9 +1354,9 @@ int main(int const argCount, const char* argv[])
|
||||
} else { /* decompression or test */
|
||||
#ifndef ZSTD_NODECOMPRESS
|
||||
if (filenames->tableSize == 1 && outFileName) {
|
||||
operationResult = FIO_decompressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName);
|
||||
operationResult = FIO_decompressFilename(fCtx, prefs, outFileName, filenames->fileNames[0], dictFileName);
|
||||
} else {
|
||||
operationResult = FIO_decompressMultipleFilenames(prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, dictFileName);
|
||||
operationResult = FIO_decompressMultipleFilenames(fCtx, prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, dictFileName);
|
||||
}
|
||||
#else
|
||||
DISPLAY("Decompression not supported \n");
|
||||
@ -1364,6 +1365,7 @@ int main(int const argCount, const char* argv[])
|
||||
|
||||
_end:
|
||||
FIO_freePreferences(prefs);
|
||||
FIO_freeContext(fCtx);
|
||||
if (main_pause) waitEnter();
|
||||
UTIL_freeFileNamesTable(filenames);
|
||||
UTIL_freeFileNamesTable(file_of_names);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user