AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added command --rm : remove source file after successful de/compression

Browse Source
This commit is contained in:
Yann Collet 2016-06-09 22:59:51 +02:00
parent 39f6779874
commit b09b12ce10
5 changed files with 49 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
NEWS
View File

@ -1,5 +1,6 @@
v0.7.0 v0.7.0
New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski
New : Command `--rm`, to remove source file after successful de/compression
New : Visual build scripts, by Christophe Chevalier New : Visual build scripts, by Christophe Chevalier
New : Support for Sparse File-systems (do not use space for zero-filled sectors) New : Support for Sparse File-systems (do not use space for zero-filled sectors)
New : Frame checksum support New : Frame checksum support

52
programs/fileio.c
View File

@ -139,6 +139,8 @@ static U32 g_dictIDFlag = 1;
void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; } void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; }
static U32 g_checksumFlag = 0; static U32 g_checksumFlag = 0;
void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; } void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; }
static U32 g_removeSrcFile = 0;
void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); }
/*-************************************* /*-*************************************
@ -365,8 +367,9 @@ static int FIO_compressFilename_internal(cRess_t ress,
/* Status */ /* Status */
DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2, "\r%79s\r", "");
DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n", DISPLAYLEVEL(2,"%-20.20s :%6.2f%% (%6llu =>%6llu bytes, %s) \n", srcFileName,
(unsigned long long)readsize, (unsigned long long) compressedfilesize, (double)compressedfilesize/readsize*100); (double)compressedfilesize/readsize*100, (unsigned long long)readsize, (unsigned long long) compressedfilesize,
dstFileName);
return 0; return 0;
} }
@ -384,36 +387,38 @@ static int FIO_compressFilename_srcFile(cRess_t ress,
int result; int result;
/* File check */ /* File check */
if (UTIL_isDirectory(srcFileName)) {
DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
return 1;
}
ress.srcFile = FIO_openSrcFile(srcFileName); ress.srcFile = FIO_openSrcFile(srcFileName);
if (!ress.srcFile) return 1; /* srcFile could not be opened */ if (!ress.srcFile) return 1; /* srcFile could not be opened */
result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel); result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel);
fclose(ress.srcFile); fclose(ress.srcFile);
if ((g_removeSrcFile) && (!result)) remove(srcFileName);
return result; return result;
} }
/*! FIO_compressFilename_extRess() : /*! FIO_compressFilename_dstFile() :
* @return : 0 : compression completed correctly, * @return : 0 : compression completed correctly,
* 1 : missing or pb opening srcFileName * 1 : pb
*/ */
static int FIO_compressFilename_extRess(cRess_t ress, static int FIO_compressFilename_dstFile(cRess_t ress,
const char* dstFileName, const char* srcFileName, const char* dstFileName, const char* srcFileName,
int cLevel) int cLevel)
{ {
int result; int result;
ress.srcFile = FIO_openSrcFile(srcFileName);
if (ress.srcFile==0) return 1;
ress.dstFile = FIO_openDstFile(dstFileName); ress.dstFile = FIO_openDstFile(dstFileName);
if (ress.dstFile==0) { fclose(ress.srcFile); return 1; } if (ress.dstFile==0) { fclose(ress.srcFile); return 1; }
result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel); result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, cLevel);
if (result!=0) remove(dstFileName); /* remove operation artefact */
fclose(ress.srcFile); /* no pb to expect : only reading */
if (fclose(ress.dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName); if (fclose(ress.dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName);
if (result!=0) remove(dstFileName); /* remove operation artefact */
return result; return result;
} }
@ -422,14 +427,12 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
const char* dictFileName, int compressionLevel) const char* dictFileName, int compressionLevel)
{ {
clock_t const start = clock(); clock_t const start = clock();
cRess_t const ress = FIO_createCResources(dictFileName); cRess_t const ress = FIO_createCResources(dictFileName);
int issueWithSrcFile = 0; int const issueWithSrcFile = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel);
issueWithSrcFile += FIO_compressFilename_extRess(ress, dstFileName, srcFileName, compressionLevel);
FIO_freeCResources(ress); FIO_freeCResources(ress);
{ double seconds = (double)(clock() - start) / CLOCKS_PER_SEC; { double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds); DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds);
} }
return issueWithSrcFile; return issueWithSrcFile;
@ -465,7 +468,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
if (dfnSize <= ifnSize+suffixSize+1) { free(dstFileName); dfnSize = ifnSize + 20; dstFileName = (char*)malloc(dfnSize); } if (dfnSize <= ifnSize+suffixSize+1) { free(dstFileName); dfnSize = ifnSize + 20; dstFileName = (char*)malloc(dfnSize); }
strcpy(dstFileName, inFileNamesTable[u]); strcpy(dstFileName, inFileNamesTable[u]);
strcat(dstFileName, suffix); strcat(dstFileName, suffix);
missed_files += FIO_compressFilename_extRess(ress, dstFileName, missed_files += FIO_compressFilename_dstFile(ress, dstFileName,
inFileNamesTable[u], compressionLevel); inFileNamesTable[u], compressionLevel);
} } } }
@ -679,7 +682,13 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
{ {
unsigned long long filesize = 0; unsigned long long filesize = 0;
FILE* const dstFile = ress.dstFile; FILE* const dstFile = ress.dstFile;
FILE* const srcFile = FIO_openSrcFile(srcFileName); FILE* srcFile;
if (UTIL_isDirectory(srcFileName)) {
DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
return 1;
}
srcFile = FIO_openSrcFile(srcFileName);
if (srcFile==0) return 1; if (srcFile==0) return 1;
/* for each frame */ /* for each frame */
@ -712,6 +721,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
/* Close */ /* Close */
fclose(srcFile); fclose(srcFile);
if (g_removeSrcFile) remove(srcFileName);
return 0; return 0;
} }
@ -721,7 +731,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
@return : 0 : OK @return : 0 : OK
1 : operation aborted (src not available, dst already taken, etc.) 1 : operation aborted (src not available, dst already taken, etc.)
*/ */
static int FIO_decompressFile_extRess(dRess_t ress, static int FIO_decompressDstFile(dRess_t ress,
const char* dstFileName, const char* srcFileName) const char* dstFileName, const char* srcFileName)
{ {
int result; int result;
@ -729,9 +739,9 @@ static int FIO_decompressFile_extRess(dRess_t ress,
if (ress.dstFile==0) return 1; if (ress.dstFile==0) return 1;
result = FIO_decompressSrcFile(ress, srcFileName); result = FIO_decompressSrcFile(ress, srcFileName);
if (result != 0) remove(dstFileName);
if (fclose(ress.dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName); if (fclose(ress.dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName);
if (result != 0) remove(dstFileName);
return result; return result;
} }
@ -742,7 +752,7 @@ int FIO_decompressFilename(const char* dstFileName, const char* srcFileName,
int missingFiles = 0; int missingFiles = 0;
dRess_t ress = FIO_createDResources(dictFileName); dRess_t ress = FIO_createDResources(dictFileName);
missingFiles += FIO_decompressFile_extRess(ress, dstFileName, srcFileName); missingFiles += FIO_decompressDstFile(ress, dstFileName, srcFileName);
FIO_freeDResources(ress); FIO_freeDResources(ress);
return missingFiles; return missingFiles;
@ -789,7 +799,7 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
memcpy(dstFileName, srcFileName, sfnSize - suffixSize); memcpy(dstFileName, srcFileName, sfnSize - suffixSize);
dstFileName[sfnSize-suffixSize] = '\0'; dstFileName[sfnSize-suffixSize] = '\0';
missingFiles += FIO_decompressFile_extRess(ress, dstFileName, srcFileName); missingFiles += FIO_decompressDstFile(ress, dstFileName, srcFileName);
} }
free(dstFileName); free(dstFileName);
} }

1
programs/fileio.h
View File

@ -50,6 +50,7 @@ void FIO_setMaxWLog(unsigned maxWLog); /**< if `maxWLog` == 0, no max enforc
void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */ void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
void FIO_setDictIDFlag(unsigned dictIDFlag); void FIO_setDictIDFlag(unsigned dictIDFlag);
void FIO_setChecksumFlag(unsigned checksumFlag); void FIO_setChecksumFlag(unsigned checksumFlag);
void FIO_setRemoveSrcFile(unsigned flag);
/*-************************************* /*-*************************************

8
programs/playTests.sh
View File

@ -66,6 +66,14 @@ $ZSTD -q tmp && die "overwrite check failed!"
$ZSTD -q -f tmp $ZSTD -q -f tmp
$ZSTD -q --force tmp $ZSTD -q --force tmp
$ZSTD -df tmp && die "should have refused : wrong extension" $ZSTD -df tmp && die "should have refused : wrong extension"
$ECHO "test : file removal"
$ZSTD -f --rm tmp
ls tmp && die "tmp should no longer be present"
$ZSTD -f -d --rm tmp.zst
ls tmp.zst && die "tmp.zst should no longer be present"
rm tmp
$ZSTD -f tmp && die "tmp not present : should have failed"
ls tmp.zst && die "tmp.zst should not be created"
$ECHO "\n**** Pass-Through mode **** " $ECHO "\n**** Pass-Through mode **** "

12
programs/zstdcli.c
View File

@ -132,9 +132,10 @@ static int usage_advanced(const char* programName)
#ifdef UTIL_HAS_CREATEFILELIST #ifdef UTIL_HAS_CREATEFILELIST
DISPLAY( " -r : operate recursively on directories\n"); DISPLAY( " -r : operate recursively on directories\n");
#endif #endif
DISPLAY( "--rm : remove source files after successful de/compression \n");
#ifndef ZSTD_NOCOMPRESS #ifndef ZSTD_NOCOMPRESS
DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n"); DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n");
DISPLAY( "--no-dictID:don't write dictID into header (dictionary compression)\n"); DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n");
DISPLAY( "--check : enable integrity check\n"); DISPLAY( "--check : enable integrity check\n");
#endif #endif
#ifndef ZSTD_NODECOMPRESS #ifndef ZSTD_NODECOMPRESS
@ -144,11 +145,11 @@ static int usage_advanced(const char* programName)
#ifndef ZSTD_NODICT #ifndef ZSTD_NODICT
DISPLAY( "\n"); DISPLAY( "\n");
DISPLAY( "Dictionary builder :\n"); DISPLAY( "Dictionary builder :\n");
DISPLAY( "--train : create a dictionary from a training set of files \n"); DISPLAY( "--train ## : create a dictionary from a training set of files \n");
DISPLAY( " -o file: `file` is dictionary name (default: %s) \n", g_defaultDictName); DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName);
DISPLAY( "--maxdict:limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize); DISPLAY( "--maxdict ## : limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize);
DISPLAY( " -s# : dictionary selectivity level (default: %u)\n", g_defaultSelectivityLevel); DISPLAY( " -s# : dictionary selectivity level (default: %u)\n", g_defaultSelectivityLevel);
DISPLAY( "--dictID: force dictionary ID to specified value (default: random)\n"); DISPLAY( "--dictID ## : force dictionary ID to specified value (default: random)\n");
#endif #endif
#ifndef ZSTD_NOBENCH #ifndef ZSTD_NOBENCH
DISPLAY( "\n"); DISPLAY( "\n");
@ -264,6 +265,7 @@ int main(int argCount, const char** argv)
if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; } if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; }
if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; continue; } if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; continue; }
if (!strcmp(argument, "--keep")) { continue; } /* does nothing, since preserving input is default; for gzip/xz compatibility */ if (!strcmp(argument, "--keep")) { continue; } /* does nothing, since preserving input is default; for gzip/xz compatibility */
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
/* '-' means stdin/stdout */ /* '-' means stdin/stdout */
if (!strcmp(argument, "-")){ if (!strcmp(argument, "-")){