From b09b12ce10bfbd50c7af3b6d4aae643d3f8f6325 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 9 Jun 2016 22:59:51 +0200 Subject: [PATCH] Added command `--rm` : remove source file after successful de/compression --- NEWS | 1 + programs/fileio.c | 52 ++++++++++++++++++++++++++----------------- programs/fileio.h | 1 + programs/playTests.sh | 8 +++++++ programs/zstdcli.c | 14 +++++++----- 5 files changed, 49 insertions(+), 27 deletions(-) diff --git a/NEWS b/NEWS index 767fe5c0..c7aeae98 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ v0.7.0 New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski +New : Command `--rm`, to remove source file after successful de/compression New : Visual build scripts, by Christophe Chevalier New : Support for Sparse File-systems (do not use space for zero-filled sectors) New : Frame checksum support diff --git a/programs/fileio.c b/programs/fileio.c index 47302f9c..decc971c 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -139,6 +139,8 @@ static U32 g_dictIDFlag = 1; void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; } static U32 g_checksumFlag = 0; void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; } +static U32 g_removeSrcFile = 0; +void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); } /*-************************************* @@ -365,8 +367,9 @@ static int FIO_compressFilename_internal(cRess_t ress, /* Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); - DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n", - (unsigned long long)readsize, (unsigned long long) compressedfilesize, (double)compressedfilesize/readsize*100); + DISPLAYLEVEL(2,"%-20.20s :%6.2f%% (%6llu =>%6llu bytes, %s) \n", srcFileName, + (double)compressedfilesize/readsize*100, (unsigned long long)readsize, (unsigned long long) compressedfilesize, + dstFileName); return 0; } @@ -384,36 +387,38 @@ static int FIO_compressFilename_srcFile(cRess_t ress, int result; /* File check */ + if (UTIL_isDirectory(srcFileName)) { + DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); + return 1; + } ress.srcFile = FIO_openSrcFile(srcFileName); if (!ress.srcFile) return 1; /* srcFile could not be opened */ result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel); fclose(ress.srcFile); + if ((g_removeSrcFile) && (!result)) remove(srcFileName); return result; } -/*! FIO_compressFilename_extRess() : +/*! FIO_compressFilename_dstFile() : * @return : 0 : compression completed correctly, - * 1 : missing or pb opening srcFileName + * 1 : pb */ -static int FIO_compressFilename_extRess(cRess_t ress, +static int FIO_compressFilename_dstFile(cRess_t ress, const char* dstFileName, const char* srcFileName, int cLevel) { int result; - ress.srcFile = FIO_openSrcFile(srcFileName); - if (ress.srcFile==0) return 1; ress.dstFile = FIO_openDstFile(dstFileName); if (ress.dstFile==0) { fclose(ress.srcFile); return 1; } - result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, cLevel); - if (result!=0) remove(dstFileName); /* remove operation artefact */ + result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, cLevel); - fclose(ress.srcFile); /* no pb to expect : only reading */ if (fclose(ress.dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName); + if (result!=0) remove(dstFileName); /* remove operation artefact */ return result; } @@ -422,14 +427,12 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, const char* dictFileName, int compressionLevel) { clock_t const start = clock(); + cRess_t const ress = FIO_createCResources(dictFileName); - int issueWithSrcFile = 0; - - issueWithSrcFile += FIO_compressFilename_extRess(ress, dstFileName, srcFileName, compressionLevel); - + int const issueWithSrcFile = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel); FIO_freeCResources(ress); - { double seconds = (double)(clock() - start) / CLOCKS_PER_SEC; + { double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC; DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds); } return issueWithSrcFile; @@ -465,7 +468,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile if (dfnSize <= ifnSize+suffixSize+1) { free(dstFileName); dfnSize = ifnSize + 20; dstFileName = (char*)malloc(dfnSize); } strcpy(dstFileName, inFileNamesTable[u]); strcat(dstFileName, suffix); - missed_files += FIO_compressFilename_extRess(ress, dstFileName, + missed_files += FIO_compressFilename_dstFile(ress, dstFileName, inFileNamesTable[u], compressionLevel); } } @@ -679,7 +682,13 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) { unsigned long long filesize = 0; FILE* const dstFile = ress.dstFile; - FILE* const srcFile = FIO_openSrcFile(srcFileName); + FILE* srcFile; + + if (UTIL_isDirectory(srcFileName)) { + DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); + return 1; + } + srcFile = FIO_openSrcFile(srcFileName); if (srcFile==0) return 1; /* for each frame */ @@ -712,6 +721,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) /* Close */ fclose(srcFile); + if (g_removeSrcFile) remove(srcFileName); return 0; } @@ -721,7 +731,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) @return : 0 : OK 1 : operation aborted (src not available, dst already taken, etc.) */ -static int FIO_decompressFile_extRess(dRess_t ress, +static int FIO_decompressDstFile(dRess_t ress, const char* dstFileName, const char* srcFileName) { int result; @@ -729,9 +739,9 @@ static int FIO_decompressFile_extRess(dRess_t ress, if (ress.dstFile==0) return 1; result = FIO_decompressSrcFile(ress, srcFileName); - if (result != 0) remove(dstFileName); if (fclose(ress.dstFile)) EXM_THROW(38, "Write error : cannot properly close %s", dstFileName); + if (result != 0) remove(dstFileName); return result; } @@ -742,7 +752,7 @@ int FIO_decompressFilename(const char* dstFileName, const char* srcFileName, int missingFiles = 0; dRess_t ress = FIO_createDResources(dictFileName); - missingFiles += FIO_decompressFile_extRess(ress, dstFileName, srcFileName); + missingFiles += FIO_decompressDstFile(ress, dstFileName, srcFileName); FIO_freeDResources(ress); return missingFiles; @@ -789,7 +799,7 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles memcpy(dstFileName, srcFileName, sfnSize - suffixSize); dstFileName[sfnSize-suffixSize] = '\0'; - missingFiles += FIO_decompressFile_extRess(ress, dstFileName, srcFileName); + missingFiles += FIO_decompressDstFile(ress, dstFileName, srcFileName); } free(dstFileName); } diff --git a/programs/fileio.h b/programs/fileio.h index 01e30834..4a4f3d22 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -50,6 +50,7 @@ void FIO_setMaxWLog(unsigned maxWLog); /**< if `maxWLog` == 0, no max enforc void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */ void FIO_setDictIDFlag(unsigned dictIDFlag); void FIO_setChecksumFlag(unsigned checksumFlag); +void FIO_setRemoveSrcFile(unsigned flag); /*-************************************* diff --git a/programs/playTests.sh b/programs/playTests.sh index f51d28ea..3be4c777 100755 --- a/programs/playTests.sh +++ b/programs/playTests.sh @@ -66,6 +66,14 @@ $ZSTD -q tmp && die "overwrite check failed!" $ZSTD -q -f tmp $ZSTD -q --force tmp $ZSTD -df tmp && die "should have refused : wrong extension" +$ECHO "test : file removal" +$ZSTD -f --rm tmp +ls tmp && die "tmp should no longer be present" +$ZSTD -f -d --rm tmp.zst +ls tmp.zst && die "tmp.zst should no longer be present" +rm tmp +$ZSTD -f tmp && die "tmp not present : should have failed" +ls tmp.zst && die "tmp.zst should not be created" $ECHO "\n**** Pass-Through mode **** " diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 75ce129f..b59c6ebd 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -132,23 +132,24 @@ static int usage_advanced(const char* programName) #ifdef UTIL_HAS_CREATEFILELIST DISPLAY( " -r : operate recursively on directories\n"); #endif + DISPLAY( "--rm : remove source files after successful de/compression \n"); #ifndef ZSTD_NOCOMPRESS DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n"); - DISPLAY( "--no-dictID:don't write dictID into header (dictionary compression)\n"); + DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n"); DISPLAY( "--check : enable integrity check\n"); #endif #ifndef ZSTD_NODECOMPRESS DISPLAY( "--test : test compressed file integrity \n"); - DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); + DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n"); #endif #ifndef ZSTD_NODICT DISPLAY( "\n"); DISPLAY( "Dictionary builder :\n"); - DISPLAY( "--train : create a dictionary from a training set of files \n"); - DISPLAY( " -o file: `file` is dictionary name (default: %s) \n", g_defaultDictName); - DISPLAY( "--maxdict:limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize); + DISPLAY( "--train ## : create a dictionary from a training set of files \n"); + DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName); + DISPLAY( "--maxdict ## : limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize); DISPLAY( " -s# : dictionary selectivity level (default: %u)\n", g_defaultSelectivityLevel); - DISPLAY( "--dictID: force dictionary ID to specified value (default: random)\n"); + DISPLAY( "--dictID ## : force dictionary ID to specified value (default: random)\n"); #endif #ifndef ZSTD_NOBENCH DISPLAY( "\n"); @@ -264,6 +265,7 @@ int main(int argCount, const char** argv) if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; } if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; continue; } if (!strcmp(argument, "--keep")) { continue; } /* does nothing, since preserving input is default; for gzip/xz compatibility */ + if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } /* '-' means stdin/stdout */ if (!strcmp(argument, "-")){