From 7f98b46876c475d8a0ef4f895b83730d55a60f9b Mon Sep 17 00:00:00 2001 From: Sen Huang Date: Thu, 5 Sep 2019 16:03:35 -0700 Subject: [PATCH] adding support for -O flag: multiple files into one directory for compressions (decompression to come) --- programs/fileio.c | 37 +++++++++++++++++++++---------------- programs/fileio.h | 15 ++++++++------- programs/util.c | 43 +++++++++++++++++++++++++++++++++++++++++++ programs/util.h | 4 ++++ programs/zstdcli.c | 26 +++++++++++++++++++++++++- 5 files changed, 101 insertions(+), 24 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 20e2ee2a..f2577186 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1276,9 +1276,7 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs, int result; stat_t statbuf; int transfer_permissions = 0; - assert(ress.srcFile != NULL); - if (ress.dstFile == NULL) { closeDstFile = 1; DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s", dstFileName); @@ -1369,11 +1367,9 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs, return result; } - -int FIO_compressFilename(FIO_prefs_t* const prefs, - const char* dstFileName, const char* srcFileName, - const char* dictFileName, int compressionLevel, - ZSTD_compressionParameters comprParams) +int FIO_compressFilename(FIO_prefs_t* const prefs, const char* dstFileName, + const char* srcFileName, const char* dictFileName, + int compressionLevel, ZSTD_compressionParameters comprParams) { cRess_t const ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams); int const result = FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); @@ -1416,22 +1412,30 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix) /* FIO_compressMultipleFilenames() : * compress nbFiles files - * into one destination (outFileName) - * or into one file each (outFileName == NULL, but suffix != NULL). + * into either one destination (outFileName), + * or into one file each (outFileName == NULL, but suffix != NULL), + * or into a destination folder (specified with -O) */ -int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, - const char** inFileNamesTable, unsigned nbFiles, - const char* outFileName, const char* suffix, - const char* dictFileName, int compressionLevel, - ZSTD_compressionParameters comprParams) +int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable, + const char* outDirName, char** dstFileNamesTable, + unsigned nbFiles, const char* outFileName, + const char* suffix, const char* dictFileName, + int compressionLevel, ZSTD_compressionParameters comprParams) { + printf("compressing multiple...\n"); int error = 0; cRess_t ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams); /* init */ assert(outFileName != NULL || suffix != NULL); - - if (outFileName != NULL) { /* output into a single destination (stdout typically) */ + if (outDirName != NULL) { /* output into a particular folder */ + unsigned u; + for (u = 0; u < nbFiles; ++u) { + const char* const srcFileName = inFileNamesTable[u]; + const char* const dstFileName = FIO_determineCompressedName(dstFileNamesTable[u], suffix); + error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel); + } + } else if (outFileName != NULL) { /* output into a single destination (stdout typically) */ ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName); if (ress.dstFile == NULL) { /* could not open outFileName */ error = 1; @@ -1453,6 +1457,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, } } FIO_freeCResources(ress); + UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles); return error; } diff --git a/programs/fileio.h b/programs/fileio.h index 096d90b5..1ed6d153 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -87,8 +87,9 @@ void FIO_setNotificationLevel(int level); /** FIO_compressFilename() : @return : 0 == ok; 1 == pb with src file. */ int FIO_compressFilename (FIO_prefs_t* const prefs, - const char* outfilename, const char* infilename, const char* dictFileName, - int compressionLevel, ZSTD_compressionParameters comprParams); + const char* outfilename, const char* infilename, + const char* dictFileName, int compressionLevel, + ZSTD_compressionParameters comprParams); /** FIO_decompressFilename() : @return : 0 == ok; 1 == pb with src file. */ @@ -103,11 +104,11 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis ***************************************/ /** FIO_compressMultipleFilenames() : @return : nb of missing files */ -int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, - const char** srcNamesTable, unsigned nbFiles, - const char* outFileName, const char* suffix, - const char* dictFileName, int compressionLevel, - ZSTD_compressionParameters comprParams); +int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable, + const char* outDirName, char** dstFileNamesTable, + unsigned nbFiles, const char* outFileName, + const char* suffix, const char* dictFileName, + int compressionLevel, ZSTD_compressionParameters comprParams); /** FIO_decompressMultipleFilenames() : @return : nb of missing or skipped files */ diff --git a/programs/util.c b/programs/util.c index 347e7698..e8b599aa 100644 --- a/programs/util.c +++ b/programs/util.c @@ -87,6 +87,49 @@ U32 UTIL_isDirectory(const char* infilename) return 0; } +int UTIL_createDir(const char* outDirName) { + if (UTIL_isDirectory(outDirName)) { + return 0; /* no need to create if directory already exists */ + } + int r; +#if defined(_MSC_VER) + r = _mkdir(outDirName); + if (r || !UTIL_isDirectory(outDirName)) return 1; +#else + r = mkdir(outDirName, S_IRWXU | S_IRWXG | S_IRWXO); /* dir has all permissions */ + if (r || !UTIL_isDirectory(outDirName)) return 1; +#endif + return 0; +} + +void UTIL_createDestinationDirTable(const char** filenameTable, unsigned nbFiles, + const char* outDirName, char** dstFilenameTable) +{ + unsigned u; + char c; + c = '/'; + + /* duplicate source file table */ + for (u = 0; u < nbFiles; ++u) { + char* filename; + char* finalPath; + size_t finalPathLen; + finalPathLen = strlen(outDirName); + filename = strrchr(filenameTable[u], c); /* filename is the last bit of string after '/' */ + finalPathLen += strlen(filename); + dstFilenameTable[u] = (char*) malloc(finalPathLen * sizeof(char) + 1); + strcpy(dstFilenameTable[u], outDirName); + strcat(dstFilenameTable[u], filename); + } +} + +void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles) { + unsigned u; + for (u = 0; u < nbFiles; ++u) + free(dstDirTable[u]); + free((void*)dstDirTable); +} + int UTIL_isSameFile(const char* file1, const char* file2) { #if defined(_MSC_VER) diff --git a/programs/util.h b/programs/util.h index d6e5bb55..1c127326 100644 --- a/programs/util.h +++ b/programs/util.h @@ -127,8 +127,12 @@ int UTIL_fileExist(const char* filename); int UTIL_isRegularFile(const char* infilename); int UTIL_setFileStat(const char* filename, stat_t* statbuf); U32 UTIL_isDirectory(const char* infilename); +int UTIL_createDir(const char* outDirName); int UTIL_getFileStat(const char* infilename, stat_t* statbuf); int UTIL_isSameFile(const char* file1, const char* file2); +void UTIL_createDestinationDirTable(const char** filenameTable, unsigned filenameIdx, + const char* outDirName, char** dstFilenameTable); +void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles); U32 UTIL_isLink(const char* infilename); #define UTIL_FILESIZE_UNKNOWN ((U64)(-1)) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 98df728a..2d3254ae 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -118,6 +118,7 @@ static int usage(const char* programName) #endif DISPLAY( " -D file: use `file` as Dictionary \n"); DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n"); + DISPLAY( " -O directory: result(s) stored into `directory`, creates one if non-existent \n"); DISPLAY( " -f : overwrite output without prompting and (de)compress links \n"); DISPLAY( "--rm : remove source file(s) after successful de/compression \n"); DISPLAY( " -k : preserve source file(s) (default) \n"); @@ -562,6 +563,7 @@ int main(int argCount, const char* argv[]) adaptMax = MAXCLEVEL, rsyncable = 0, nextArgumentIsOutFileName = 0, + nextArgumentIsOutDirName = 0, nextArgumentIsMaxDict = 0, nextArgumentIsDictID = 0, nextArgumentsAreFiles = 0, @@ -583,9 +585,11 @@ int main(int argCount, const char* argv[]) unsigned recursive = 0; unsigned memLimit = 0; const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ + char** dstFilenameTable = (char**)malloc(argCount * sizeof(char*)); unsigned filenameIdx = 0; const char* programName = argv[0]; const char* outFileName = NULL; + const char* outDirName = NULL; const char* dictFileName = NULL; const char* suffix = ZSTD_EXTENSION; unsigned maxDictSize = g_defaultMaxDictSize; @@ -853,6 +857,9 @@ int main(int argCount, const char* argv[]) /* destination file name */ case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break; + /* destination directory name */ + case 'O': nextArgumentIsOutDirName=1; lastCommand=1; argument++; break; + /* limit decompression memory */ case 'M': argument++; @@ -965,6 +972,13 @@ int main(int argCount, const char* argv[]) continue; } + if (nextArgumentIsOutDirName) { + nextArgumentIsOutDirName = 0; + lastCommand = 0; + outDirName = argument; + continue; + } + /* add filename to list */ filenameTable[filenameIdx++] = argument; } @@ -1163,10 +1177,20 @@ int main(int argCount, const char* argv[]) if (adaptMin > cLevel) cLevel = adaptMin; if (adaptMax < cLevel) cLevel = adaptMax; + if (outDirName) { + int dirResult; + dirResult = UTIL_createDir(outDirName); + if (dirResult) DISPLAY("Directory creation unsuccessful \n"); + + UTIL_createDestinationDirTable(filenameTable, filenameIdx, outDirName, dstFilenameTable); + if (outFileName) { + outFileName = dstFilenameTable[0]; /* in case -O is called with single file */ + } + } if ((filenameIdx==1) && outFileName) operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams); else - operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); + operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, dstFilenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); #else (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */ DISPLAY("Compression not supported \n");