added : -m : compress multiple files in a single command

programs/fileio.c

@@ -236,6 +236,212 @@ static U64 FIO_getFileSize(const char* infilename)
 }
 
 
+static int FIO_getFiles(const char* input_filename, const char* output_filename,
+                        FILE** pfinput, FILE** pfoutput)
+{
+
+    if (!strcmp (input_filename, stdinmark))
+    {
+        DISPLAYLEVEL(4,"Using stdin for input\n");
+        *pfinput = stdin;
+        SET_BINARY_MODE(stdin);
+    }
+    else
+    {
+        *pfinput = fopen(input_filename, "rb");
+    }
+
+    if ( *pfinput==0 )
+    {
+        DISPLAYLEVEL(1, "Unable to access file for processing: %s\n", input_filename);
+        return 1;
+    }
+
+    if (!strcmp (output_filename, stdoutmark))
+    {
+        DISPLAYLEVEL(4,"Using stdout for output\n");
+        *pfoutput = stdout;
+        SET_BINARY_MODE(stdout);
+    }
+    else
+    {
+        /* Check if destination file already exists */
+        *pfoutput=0;
+        *pfoutput = fopen( output_filename, "rb" );
+        if (*pfoutput!=0)
+        {
+            fclose(*pfoutput);
+            if (!g_overwrite)
+            {
+                int ch = 'Y';
+                DISPLAYLEVEL(2, "Warning : %s already exists\n", output_filename);
+                if ((g_displayLevel <= 1) || (*pfinput == stdin))
+                    EXM_THROW(11, "Operation aborted : %s already exists", output_filename);   /* No interaction possible */
+                DISPLAYLEVEL(2, "Overwrite ? (Y/n) : ");
+                while((ch = getchar()) != '\n' && ch != EOF)   /* flush integrated */
+                if ((ch!='Y') && (ch!='y')) EXM_THROW(12, "No. Operation aborted : %s already exists", output_filename);
+            }
+        }
+        *pfoutput = fopen( output_filename, "wb" );
+    }
+
+    if (*pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename);
+
+    return 0;
+}
+
+
+/* **********************************************************************
+*  Compression
+************************************************************************/
+typedef struct {
+    void*  srcBuffer;
+    size_t srcBufferSize;
+    void*  dstBuffer;
+    size_t dstBufferSize;
+    void*  dictBuffer;
+    size_t dictBufferSize;
+    ZBUFF_CCtx* ctx;
+} cRess_t;
+
+static cRess_t FIO_createCResources(const char* dictFileName)
+{
+    cRess_t ress;
+
+    ress.ctx = ZBUFF_createCCtx();
+    if (ress.ctx == NULL) EXM_THROW(30, "Allocation error : can't create ZBUFF context");
+
+    /* Allocate Memory */
+    ress.srcBufferSize = ZBUFF_recommendedCInSize();
+    ress.srcBuffer = malloc(ress.srcBufferSize);
+    ress.dstBufferSize = ZBUFF_recommendedCOutSize();
+    ress.dstBuffer = malloc(ress.dstBufferSize);
+    if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(31, "Allocation error : not enough memory");
+
+    /* dictionary */
+    ress.dictBuffer = NULL;
+    ress.dictBufferSize = 0;
+    if (dictFileName)
+    {
+        FILE* dictHandle;
+        size_t readSize, dictSize;
+        DISPLAYLEVEL(4,"Using %s as dictionary \n", dictFileName);
+        dictHandle = fopen(dictFileName, "rb");
+        if (dictHandle==0) EXM_THROW(31, "Error opening dictionary file %s", dictFileName);
+        dictSize = FIO_getFileSize(dictFileName);
+        if (dictSize > MAX_DICT_SIZE)
+        {
+            int seekResult;
+            if (dictSize > 1 GB) EXM_THROW(32, "Dictionary file %s is too large", dictFileName);   /* avoid extreme cases */
+            DISPLAYLEVEL(2,"Dictionary %s is too large : using last %u bytes only \n", dictFileName, MAX_DICT_SIZE);
+            seekResult = fseek(dictHandle, (long int)(dictSize-MAX_DICT_SIZE), SEEK_SET);   /* use end of file */
+            if (seekResult != 0) EXM_THROW(33, "Error seeking into dictionary file %s", dictFileName);
+            dictSize = MAX_DICT_SIZE;
+        }
+        ress.dictBuffer = (BYTE*)malloc((size_t)dictSize);
+        if (ress.dictBuffer==NULL) EXM_THROW(34, "Allocation error : not enough memory for dictBuffer");
+        readSize = fread(ress.dictBuffer, 1, (size_t)dictSize, dictHandle);
+        if (readSize!=dictSize) EXM_THROW(35, "Error reading dictionary file %s", dictFileName);
+        fclose(dictHandle);
+        ress.dictBufferSize = dictSize;
+    }
+
+    return ress;
+}
+
+static void FIO_freeCResources(cRess_t ress)
+{
+    size_t errorCode;
+    free(ress.srcBuffer);
+    free(ress.dstBuffer);
+    free(ress.dictBuffer);
+    errorCode = ZBUFF_freeCCtx(ress.ctx);
+    if (ZBUFF_isError(errorCode)) EXM_THROW(38, "Error : can't release ZBUFF context resource : %s", ZBUFF_getErrorName(errorCode));
+}
+
+
+/*
+ * FIO_compressFilename_extRess()
+ * result : 0 : compression completed correctly
+ *          1 : missing or pb opening srcFileName
+ */
+static int FIO_compressFilename_extRess(cRess_t ress,
+                                        const char* srcFileName, const char* dstFileName,
+                                        int cLevel)
+{
+    FILE* srcFile;
+    FILE* dstFile;
+    U64 filesize = 0;
+    U64 compressedfilesize = 0;
+    size_t dictSize = ress.dictBufferSize;
+    size_t sizeCheck, errorCode;
+
+    /* File check */
+    if (FIO_getFiles(srcFileName, dstFileName, &srcFile, &dstFile)) return 1;
+
+    /* init */
+    filesize = FIO_getFileSize(srcFileName) + dictSize;
+    errorCode = ZBUFF_compressInit_advanced(ress.ctx, ZSTD_getParams(cLevel, filesize));
+    if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression");
+    errorCode = ZBUFF_compressWithDictionary(ress.ctx, ress.dictBuffer, ress.dictBufferSize);
+    if (ZBUFF_isError(errorCode)) EXM_THROW(22, "Error initializing dictionary");
+
+    /* Main compression loop */
+    filesize = 0;
+    while (1)
+    {
+        size_t inSize;
+
+        /* Fill input Buffer */
+        inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile);
+        if (inSize==0) break;
+        filesize += inSize;
+        DISPLAYUPDATE(2, "\rRead : %u MB  ", (U32)(filesize>>20));
+
+        {
+            /* Compress (buffered streaming ensures appropriate formatting) */
+            size_t usedInSize = inSize;
+            size_t cSize = ress.dstBufferSize;
+            size_t result = ZBUFF_compressContinue(ress.ctx, ress.dstBuffer, &cSize, ress.srcBuffer, &usedInSize);
+            if (ZBUFF_isError(result))
+                EXM_THROW(23, "Compression error : %s ", ZBUFF_getErrorName(result));
+            if (inSize != usedInSize)
+                /* inBuff should be entirely consumed since buffer sizes are recommended ones */
+                EXM_THROW(24, "Compression error : input block not fully consumed");
+
+            /* Write cBlock */
+            sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile);
+            if (sizeCheck!=cSize) EXM_THROW(25, "Write error : cannot write compressed block into %s", dstFileName);
+            compressedfilesize += cSize;
+        }
+
+        DISPLAYUPDATE(2, "\rRead : %u MB  ==> %.2f%%   ", (U32)(filesize>>20), (double)compressedfilesize/filesize*100);
+    }
+
+    /* End of Frame */
+    {
+        size_t cSize = ress.dstBufferSize;
+        size_t result = ZBUFF_compressEnd(ress.ctx, ress.dstBuffer, &cSize);
+        if (result!=0) EXM_THROW(26, "Compression error : cannot create frame end");
+
+        sizeCheck = fwrite(ress.dstBuffer, 1, cSize, dstFile);
+        if (sizeCheck!=cSize) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName);
+        compressedfilesize += cSize;
+    }
+
+    /* Status */
+    DISPLAYLEVEL(2, "\r%79s\r", "");
+    DISPLAYLEVEL(2,"Compressed %llu bytes into %llu bytes ==> %.2f%%\n",
+        (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100);
+
+    /* clean */
+    fclose(srcFile);
+    if (fclose(dstFile)) EXM_THROW(28, "Write error : cannot properly close %s", dstFileName);
+
+    return 0;
+}
+
+
 unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename,
                                         const char* dictFileName, int cLevel)
 {
@@ -349,6 +555,47 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
 }
 
 
+#define FNSPACE 30
+int FIO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize,
+                                  const char* suffix,
+                                  const char* dictFileName, int compressionLevel)
+{
+    int i;
+    int missed_files = 0;
+    char* dstFileName = (char*)malloc(FNSPACE);
+    size_t dfnSize = FNSPACE;
+    const size_t suffixSize = strlen(suffix);
+    cRess_t ress;
+
+    /* init */
+    ress = FIO_createCResources(dictFileName);
+
+    /* loop on each file */
+    for (i=0; i<ifntSize; i++)
+    {
+        size_t ifnSize = strlen(inFileNamesTable[i]);
+        if (dfnSize <= ifnSize+suffixSize+1) { free(dstFileName); dfnSize = ifnSize + 20; dstFileName = (char*)malloc(dfnSize); }
+        strcpy(dstFileName, inFileNamesTable[i]);
+        strcat(dstFileName, suffix);
+
+        missed_files += FIO_compressFilename_extRess(ress, inFileNamesTable[i], dstFileName, compressionLevel);
+    }
+
+    /* Close & Free */
+    FIO_freeCResources(ress);
+    free(dstFileName);
+
+    return missed_files;
+}
+
+
+
+
+/* **************************************************************************
+*  Decompression
+****************************************************************************/
+
+
 unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
                                        BYTE* inBuff, size_t inBuffSize, size_t alreadyLoaded,
                                        BYTE* outBuff, size_t outBuffSize,

programs/fileio.h

@@ -50,7 +50,7 @@ void FIO_setNotificationLevel(unsigned level);
 
 
 /* *************************************
-*  Stream/File functions
+*  Single File functions
 ***************************************/
 unsigned long long FIO_compressFilename (const char* outfilename, const char* infilename, const char* dictFileName, int compressionLevel);
 unsigned long long FIO_decompressFilename (const char* outfilename, const char* infilename, const char* dictFileName);
@@ -63,6 +63,17 @@ FIO_decompressFilename :
 */
 
 
+/* *************************************
+*  Multiple File functions
+***************************************/
+int FIO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize,
+                                  const char* suffix,
+                                  const char* dictFileName, int compressionLevel);
+/**
+FIO_compressMultipleFilenames :
+    @result : nb of missing files
+*/
+
 
 #if defined (__cplusplus)
 }

programs/zstdcli.c

@@ -137,6 +137,7 @@ static int usage_advanced(const char* programName)
     DISPLAY( " -V     : display Version number and exit\n");
     DISPLAY( " -v     : verbose mode\n");
     DISPLAY( " -q     : suppress warnings; specify twice to suppress errors too\n");
+    DISPLAY( " -m     : multiple input filenames mode");
     DISPLAY( " -c     : force write to standard output, even if it is the console\n");
     DISPLAY( " -D file: use file content as Dictionary \n");
     //DISPLAY( " -t     : test compressed file integrity\n");
@@ -174,7 +175,9 @@ int main(int argCount, const char** argv)
         decode=0,
         forceStdout=0,
         main_pause=0,
-        nextEntryIsDictionary=0;
+        nextEntryIsDictionary=0,
+        multiple=0,
+        operationResult=0;
     unsigned cLevel = 1;
     const char* programName = argv[0];
     const char* inFileName = NULL;
@@ -250,6 +253,9 @@ int main(int argCount, const char** argv)
                     /* Decoding */
                 case 'd': decode=1; argument++; break;
 
+                    /* Multiple input files */
+                case 'm': multiple=1; argument++; break;
+
                     /* Force stdout, even if stdout==console */
                 case 'c': forceStdout=1; outFileName=stdoutmark; displayLevel=1; argument++; break;
 
@@ -390,18 +396,24 @@ int main(int argCount, const char** argv)
     /* Check if output is defined as console; trigger an error in this case */
     if (!strcmp(outFileName,stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) return badusage(programName);
 
-    /* No warning message in pure pipe mode (stdin + stdout) */
+    /* No warning message in pure pipe mode (stdin + stdout) or multiple mode */
     if (!strcmp(inFileName, stdinmark) && !strcmp(outFileName,stdoutmark) && (displayLevel==2)) displayLevel=1;
+    if (multiple && (displayLevel==2)) displayLevel=1;
 
     /* IO Stream/File */
     FIO_setNotificationLevel(displayLevel);
     if (decode)
         FIO_decompressFilename(outFileName, inFileName, dictFileName);
+    else
+    {
+        if (multiple)
+          operationResult = FIO_compressMultipleFilenames(argv+fileNameStart, nbFiles, ZSTD_EXTENSION, dictFileName, cLevel);
         else
           FIO_compressFilename(outFileName, inFileName, dictFileName, cLevel);
+    }
 
 _end:
     if (main_pause) waitEnter();
     free(dynNameSpace);
-    return 0;
+    return operationResult;
 }