AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

adding support for -O flag: multiple files into one directory for compressions (decompression to come)

Browse Source
This commit is contained in:
Sen Huang 2019-09-05 16:03:35 -07:00
parent b8327ac4c6
commit 7f98b46876
5 changed files with 101 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
programs/fileio.c
View File

@ -1276,9 +1276,7 @@ static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs,
int result;
stat_t statbuf;
int transfer_permissions = 0;
assert(ress.srcFile != NULL);
if (ress.dstFile == NULL) {
closeDstFile = 1;
DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s", dstFileName);
@ -1369,11 +1367,9 @@ FIO_compressFilename_srcFile(FIO_prefs_t* const prefs,
return result;
}
int FIO_compressFilename(FIO_prefs_t* const prefs,
const char* dstFileName, const char* srcFileName,
const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters comprParams)
int FIO_compressFilename(FIO_prefs_t* const prefs, const char* dstFileName,
const char* srcFileName, const char* dictFileName,
int compressionLevel, ZSTD_compressionParameters comprParams)
{
cRess_t const ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams);
int const result = FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
@ -1416,22 +1412,30 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix)
/* FIO_compressMultipleFilenames() :
* compress nbFiles files
* into one destination (outFileName)
* or into one file each (outFileName == NULL, but suffix != NULL).
* into either one destination (outFileName),
* or into one file each (outFileName == NULL, but suffix != NULL),
* or into a destination folder (specified with -O)
*/
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
const char** inFileNamesTable, unsigned nbFiles,
const char* outFileName, const char* suffix,
const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters comprParams)
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable,
const char* outDirName, char** dstFileNamesTable,
unsigned nbFiles, const char* outFileName,
const char* suffix, const char* dictFileName,
int compressionLevel, ZSTD_compressionParameters comprParams)
{
printf("compressing multiple...\n");
int error = 0;
cRess_t ress = FIO_createCResources(prefs, dictFileName, compressionLevel, comprParams);
/* init */
assert(outFileName != NULL || suffix != NULL);
if (outFileName != NULL) { /* output into a single destination (stdout typically) */
if (outDirName != NULL) { /* output into a particular folder */
unsigned u;
for (u = 0; u < nbFiles; ++u) {
const char* const srcFileName = inFileNamesTable[u];
const char* const dstFileName = FIO_determineCompressedName(dstFileNamesTable[u], suffix);
error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
}
} else if (outFileName != NULL) { /* output into a single destination (stdout typically) */
ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName);
if (ress.dstFile == NULL) { /* could not open outFileName */
error = 1;
@ -1453,6 +1457,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
} }
FIO_freeCResources(ress);
UTIL_freeDestinationFilenameTable(dstFileNamesTable, nbFiles);
return error;
}

15
programs/fileio.h
View File

@ -87,8 +87,9 @@ void FIO_setNotificationLevel(int level);
/** FIO_compressFilename() :
@return : 0 == ok; 1 == pb with src file. */
int FIO_compressFilename (FIO_prefs_t* const prefs,
const char* outfilename, const char* infilename, const char* dictFileName,
int compressionLevel, ZSTD_compressionParameters comprParams);
const char* outfilename, const char* infilename,
const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters comprParams);
/** FIO_decompressFilename() :
@return : 0 == ok; 1 == pb with src file. */
@ -103,11 +104,11 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
***************************************/
/** FIO_compressMultipleFilenames() :
@return : nb of missing files */
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
const char** srcNamesTable, unsigned nbFiles,
const char* outFileName, const char* suffix,
const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters comprParams);
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs, const char** inFileNamesTable,
const char* outDirName, char** dstFileNamesTable,
unsigned nbFiles, const char* outFileName,
const char* suffix, const char* dictFileName,
int compressionLevel, ZSTD_compressionParameters comprParams);
/** FIO_decompressMultipleFilenames() :
@return : nb of missing or skipped files */

43
programs/util.c
View File

@ -87,6 +87,49 @@ U32 UTIL_isDirectory(const char* infilename)
return 0;
}
int UTIL_createDir(const char* outDirName) {
if (UTIL_isDirectory(outDirName)) {
return 0; /* no need to create if directory already exists */
}
int r;
#if defined(_MSC_VER)
r = _mkdir(outDirName);
if (r || !UTIL_isDirectory(outDirName)) return 1;
#else
r = mkdir(outDirName, S_IRWXU | S_IRWXG | S_IRWXO); /* dir has all permissions */
if (r || !UTIL_isDirectory(outDirName)) return 1;
#endif
return 0;
}
void UTIL_createDestinationDirTable(const char** filenameTable, unsigned nbFiles,
const char* outDirName, char** dstFilenameTable)
{
unsigned u;
char c;
c = '/';
/* duplicate source file table */
for (u = 0; u < nbFiles; ++u) {
char* filename;
char* finalPath;
size_t finalPathLen;
finalPathLen = strlen(outDirName);
filename = strrchr(filenameTable[u], c); /* filename is the last bit of string after '/' */
finalPathLen += strlen(filename);
dstFilenameTable[u] = (char*) malloc(finalPathLen * sizeof(char) + 1);
strcpy(dstFilenameTable[u], outDirName);
strcat(dstFilenameTable[u], filename);
}
}
void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles) {
unsigned u;
for (u = 0; u < nbFiles; ++u)
free(dstDirTable[u]);
free((void*)dstDirTable);
}
int UTIL_isSameFile(const char* file1, const char* file2)
{
#if defined(_MSC_VER)

4
programs/util.h
View File

@ -127,8 +127,12 @@ int UTIL_fileExist(const char* filename);
int UTIL_isRegularFile(const char* infilename);
int UTIL_setFileStat(const char* filename, stat_t* statbuf);
U32 UTIL_isDirectory(const char* infilename);
int UTIL_createDir(const char* outDirName);
int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
int UTIL_isSameFile(const char* file1, const char* file2);
void UTIL_createDestinationDirTable(const char** filenameTable, unsigned filenameIdx,
const char* outDirName, char** dstFilenameTable);
void UTIL_freeDestinationFilenameTable(char** dstDirTable, unsigned nbFiles);
U32 UTIL_isLink(const char* infilename);
#define UTIL_FILESIZE_UNKNOWN ((U64)(-1))

26
programs/zstdcli.c
View File

@ -118,6 +118,7 @@ static int usage(const char* programName)
#endif
DISPLAY( " -D file: use `file` as Dictionary \n");
DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n");
DISPLAY( " -O directory: result(s) stored into `directory`, creates one if non-existent \n");
DISPLAY( " -f : overwrite output without prompting and (de)compress links \n");
DISPLAY( "--rm : remove source file(s) after successful de/compression \n");
DISPLAY( " -k : preserve source file(s) (default) \n");
@ -562,6 +563,7 @@ int main(int argCount, const char* argv[])
adaptMax = MAXCLEVEL,
rsyncable = 0,
nextArgumentIsOutFileName = 0,
nextArgumentIsOutDirName = 0,
nextArgumentIsMaxDict = 0,
nextArgumentIsDictID = 0,
nextArgumentsAreFiles = 0,
@ -583,9 +585,11 @@ int main(int argCount, const char* argv[])
unsigned recursive = 0;
unsigned memLimit = 0;
const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */
char** dstFilenameTable = (char**)malloc(argCount * sizeof(char*));
unsigned filenameIdx = 0;
const char* programName = argv[0];
const char* outFileName = NULL;
const char* outDirName = NULL;
const char* dictFileName = NULL;
const char* suffix = ZSTD_EXTENSION;
unsigned maxDictSize = g_defaultMaxDictSize;
@ -853,6 +857,9 @@ int main(int argCount, const char* argv[])
/* destination file name */
case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break;
/* destination directory name */
case 'O': nextArgumentIsOutDirName=1; lastCommand=1; argument++; break;
/* limit decompression memory */
case 'M':
argument++;
@ -965,6 +972,13 @@ int main(int argCount, const char* argv[])
continue;
}
if (nextArgumentIsOutDirName) {
nextArgumentIsOutDirName = 0;
lastCommand = 0;
outDirName = argument;
continue;
}
/* add filename to list */
filenameTable[filenameIdx++] = argument;
}
@ -1163,10 +1177,20 @@ int main(int argCount, const char* argv[])
if (adaptMin > cLevel) cLevel = adaptMin;
if (adaptMax < cLevel) cLevel = adaptMax;
if (outDirName) {
int dirResult;
dirResult = UTIL_createDir(outDirName);
if (dirResult) DISPLAY("Directory creation unsuccessful \n");
UTIL_createDestinationDirTable(filenameTable, filenameIdx, outDirName, dstFilenameTable);
if (outFileName) {
outFileName = dstFilenameTable[0]; /* in case -O is called with single file */
}
}
if ((filenameIdx==1) && outFileName)
operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams);
else
operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, outDirName, dstFilenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
#else
(void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; /* not used when ZSTD_NOCOMPRESS set */
DISPLAY("Compression not supported \n");