improved --file=FILE implementation

pass basic tests

doc/zstd_manual.html

@@ -947,7 +947,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
      * to evolve and should be considered only in the context of extremely
      * advanced performance tuning.
      *
-     * Zstd currently supports the use of a CDict in two ways:
+     * Zstd currently supports the use of a CDict in three ways:
      *
      * - The contents of the CDict can be copied into the working context. This
      *   means that the compression can search both the dictionary and input
@@ -962,6 +962,12 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
      *   tables. However, this model incurs no start-up cost (as long as the
      *   working context's tables can be reused). For small inputs, this can be
      *   faster than copying the CDict's tables.
+     * 
+     * - The CDict's tables are not used at all, and instead we use the working
+     *   context alone to reload the dictionary and use params based on the source
+     *   size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict().
+     *   This method is effective when the dictionary sizes are very small relative
+     *   to the input size, and the input size is fairly large to begin with.
      *
      * Zstd has a simple internal heuristic that selects which strategy to use
      * at the beginning of a compression. However, if experimentation shows that
@@ -970,7 +976,8 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
      */
     ZSTD_dictDefaultAttach = 0, </b>/* Use the default heuristic. */<b>
     ZSTD_dictForceAttach   = 1, </b>/* Never copy the dictionary. */<b>
-    ZSTD_dictForceCopy     = 2  </b>/* Always copy the dictionary. */<b>
+    ZSTD_dictForceCopy     = 2, </b>/* Always copy the dictionary. */<b>
+    ZSTD_dictForceLoad     = 3  </b>/* Always reload the dictionary */<b>
 } ZSTD_dictAttachPref_e;
 </b></pre><BR>
 <pre><b>typedef enum {

programs/util.c

@@ -24,6 +24,32 @@ extern "C" {
 #include <direct.h>     /* needed for _mkdir in windows */
 #endif
 
+
+/*-****************************************
+*  Internal Macros
+******************************************/
+
+/* CONTROL is like an assert(), but is never disabled.
+ * Since it's always active, it can trigger side effects.
+ */
+#define CONTROL(c)  {         \
+    if (!(c)) {               \
+        UTIL_DISPLAYLEVEL(1, "Error : %s, %i : %s",  \
+                          __FILE__, __LINE__, #c);   \
+        abort();              \
+}   }
+
+
+/*-****************************************
+*  Console log
+******************************************/
+int g_utilDisplayLevel;
+
+
+/*-****************************************
+*  Public API
+******************************************/
+
 int UTIL_fileExist(const char* filename)
 {
     stat_t statbuf;
@@ -188,220 +214,174 @@ U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbF
 }
 
 
-int UTIL_readLineFromFile(char* buf, size_t len, FILE* file) {
+/* condition : @file must be valid, and not have reached its end.
-    char* fgetsCheck = NULL;
+ * @return : length of line written into buf, without the final '\n',
-
+ *           or 0, if there is no new line */
-    if (feof(file)) {
+static size_t readLineFromFile(char* buf, size_t len, FILE* file)
-      UTIL_DISPLAYLEVEL(1, "[ERROR] end of file reached and need to read\n");
+{
-      return -1;
+    fprintf(stderr, "readLineFromFile \n");
-    }
+    assert(!feof(file));
-
+    CONTROL( fgets(buf, (int) len, file) == buf );  /* requires success */
-    fgetsCheck = fgets(buf, (int) len, file);
+    fprintf(stderr, "line = %s \n", buf);
-
+    if (strlen(buf)==0) return 0;
-    if(fgetsCheck == NULL || fgetsCheck != buf) {
+    return strlen(buf) - (buf[strlen(buf)-1] == '\n');   /* -1 to ignore final '\n' character */
-      UTIL_DISPLAYLEVEL(1, "[ERROR][UTIL_readLineFromFile] fgets has a problem check: %s buf: %s \n",
-        fgetsCheck == NULL ? "NULL" : fgetsCheck, buf);
-      return -1;
-    }
-
-    return (int) strlen(buf)-1;   /* -1 to ignore '\n' character */
 }
 
-/* Warning: inputFileSize should be less than or equal buf capacity and buf should be initialized*/
+/* Conditions :
-static int readFromFile(char* buf, size_t inputFileSize, const char* inputFileName) {
+ *   size of @inputFileName file must be < @dstCapacity
+ *   @dst must be initialized
+ * @return : nb of lines
+ *       or -1 if there's an error
+ */
+static int
+readLinesFromFile(void* dst, size_t dstCapacity,
+            const char* inputFileName)
+{
+    int nbFiles = 0;
+    unsigned pos = 0;
+    char* const buf = (char*)dst;
+    FILE* const inputFile = fopen(inputFileName, "r");
 
-  FILE* inputFile = fopen(inputFileName, "r");
+    assert(dst != NULL);
-  int nbFiles = -1;
-  unsigned pos = 0;
 
+    fprintf(stderr, "readLinesFromFile %s \n", inputFileName);
 
-  if(!buf) {
+    if(!inputFile) {
-    UTIL_DISPLAYLEVEL(1, "[ERROR][UTIL_readFileNamesTableFromFile] Can't create buffer.\n");
+        if (g_utilDisplayLevel >= 1) perror("zstd:util:readLinesFromFile");
-    return -1;
+        return -1;
-  }
-
-  if(!inputFile) {
-    UTIL_DISPLAYLEVEL(1, "[ERROR][UTIL_readFileNamesTableFromFile] Can't open file to read input file names.\n");
-    return -1;
-  }
-
-  for(nbFiles=0; !feof(inputFile) ; ) {
-    if(UTIL_readLineFromFile(buf+pos, inputFileSize, inputFile) > 0) {
-      int len = (int) strlen(buf+pos);
-      buf[pos+len-1] = '\0'; /* replace '\n' with '\0'*/
-      pos += len;
-      ++nbFiles;
     }
-  }
 
-  fclose(inputFile);
+    while ( !feof(inputFile) ) {
+        size_t const lineLength = readLineFromFile(buf+pos, dstCapacity-pos, inputFile);
+        if (lineLength == 0) break;
+        assert(pos + lineLength < dstCapacity);
+        buf[pos+lineLength] = '\0'; /* replace '\n' with '\0'*/
+        pos += lineLength + 1;
+        ++nbFiles;
+        fprintf(stderr, "nbFiles = %i \n", nbFiles);
+    }
 
-  if(pos > inputFileSize) return -1;
+    CONTROL( fclose(inputFile) == 0 );
 
-  return nbFiles;
+    return nbFiles;
 }
 
 /*Note: buf is not freed in case function successfully created table because filesTable->fileNames[0] = buf*/
 FileNamesTable*
-UTIL_createFileNamesTable_fromFileName(const char* inputFileName) {
+UTIL_createFileNamesTable_fromFileName(const char* inputFileName)
-    U64 inputFileSize = 0;
+{
-    unsigned nbFiles = 0;
+    size_t nbFiles = 0;
-    int ret_nbFiles = -1;
+    char* buf;
-    char* buf = NULL;
+    size_t bufSize;
-    size_t i = 0, pos = 0;
+    size_t pos = 0;
 
-    FileNamesTable* filesTable = NULL;
+    if (!UTIL_fileExist(inputFileName) || !UTIL_isRegularFile(inputFileName))
+        return NULL;
 
-    if(!UTIL_fileExist(inputFileName) || !UTIL_isRegularFile(inputFileName))
+    {   U64 const inputFileSize = UTIL_getFileSize(inputFileName);
-      return NULL;
+        if(inputFileSize > MAX_FILE_OF_FILE_NAMES_SIZE)
-
+            return NULL;
-    inputFileSize = UTIL_getFileSize(inputFileName) + 1; /* (+1) to add '\0' at the end of last filename */
+        bufSize = inputFileSize + 1; /* (+1) to add '\0' at the end of last filename */
-
-    if(inputFileSize > MAX_FILE_OF_FILE_NAMES_SIZE)
-      return NULL;
-
-    buf = (char*) malloc((size_t) inputFileSize * sizeof(char));
-    if(!buf) {
-      UTIL_DISPLAYLEVEL(1, "[ERROR][UTIL_readFileNamesTableFromFile] Can't create buffer.\n");
-      return NULL;
     }
 
-    ret_nbFiles = readFromFile(buf, (size_t) inputFileSize, inputFileName);
+    buf = (char*) malloc(bufSize);
+    CONTROL( buf != NULL );
 
-    if(ret_nbFiles <= 0) {
+    {   int const ret_nbFiles = readLinesFromFile(buf, bufSize, inputFileName);
-      free(buf);
-      return NULL;
-    }
-    nbFiles = ret_nbFiles;
 
-    filesTable = UTIL_createFileNamesTable(NULL, NULL, 0);
+        if (ret_nbFiles <= 0) {
-    if(!filesTable) {
+          free(buf);
-      free(buf);
+          return NULL;
-      UTIL_DISPLAYLEVEL(1, "[ERROR][UTIL_readFileNamesTableFromFile] Can't create table for files.\n");
+        }
-      return NULL;
+        nbFiles = (size_t)ret_nbFiles;
     }
 
-    filesTable->tableSize = nbFiles;
+    {   const char** filenamesTable = (const char**) malloc(nbFiles * sizeof(*filenamesTable));
-    filesTable->fileNames = (const char**) malloc((nbFiles+1) * sizeof(char*));
+        CONTROL(filenamesTable != NULL);
 
+        {   size_t fnb;
+            for (fnb = 0, pos = 0; fnb < nbFiles; fnb++) {
+                filenamesTable[fnb] = buf+pos;
+                pos += strlen(buf+pos)+1;  /* +1 for the finishing `\0` */
+        }   }
+        assert(pos <= bufSize);
 
-
+        return UTIL_createFileNamesTable(filenamesTable, nbFiles, buf);
-    for(i = 0, pos = 0; i < nbFiles; ++i) {
-      filesTable->fileNames[i] = buf+pos;
-      pos += strlen(buf+pos)+1;
     }
-
-
-    if(pos > inputFileSize){
-      UTIL_freeFileNamesTable(filesTable);
-      if(buf) free(buf);
-      return NULL;
-    }
-
-    filesTable->buf = buf;
-
-    return filesTable;
 }
 
 FileNamesTable*
-UTIL_createFileNamesTable(const char** filenames, char* buf, size_t tableSize){
+UTIL_createFileNamesTable(const char** filenames, size_t tableSize, char* buf)
-  FileNamesTable* table = (FileNamesTable*) malloc(sizeof(FileNamesTable));
+{
-  if(!table) {
+    FileNamesTable* const table = (FileNamesTable*) malloc(sizeof(*table));
-    return NULL;
+    if(!table) return NULL;
-  }
+    table->fileNames = filenames;
-  table->fileNames = filenames;
+    table->buf = buf;
-  table->buf = buf;
+    table->tableSize = tableSize;
-  table->tableSize = tableSize;
+    return table;
-  return table;
 }
 
-void UTIL_freeFileNamesTable(FileNamesTable* table) {
+void UTIL_freeFileNamesTable(FileNamesTable* table)
-  if(table) {
+{
-    if(table->fileNames) {
+    if (table==NULL) return;
-      free((void*)table->fileNames);
+    free((void*)table->fileNames);
-    }
+    free(table->buf);
-
-    if(table && table->buf) {
-      free(table->buf);
-    }
-
     free(table);
-  }
 }
 
-static size_t getTotalTableSize(FileNamesTable* table) {
+static size_t getTotalTableSize(FileNamesTable* table)
-  size_t i = 0, totalSize = 0;
+{
-  for(i = 0 ; i < table->tableSize && table->fileNames[i] ; ++i) {
+    size_t fnb = 0, totalSize = 0;
-    totalSize += strlen(table->fileNames[i]) + 1; /* +1 to add '\0' at the end of each fileName */
+    for(fnb = 0 ; fnb < table->tableSize && table->fileNames[fnb] ; ++fnb) {
-  }
+        totalSize += strlen(table->fileNames[fnb]) + 1; /* +1 to add '\0' at the end of each fileName */
-
+    }
-  return totalSize;
+    return totalSize;
 }
 
 FileNamesTable*
-UTIL_concatenateTwoTables(FileNamesTable* table1, FileNamesTable* table2) {
+UTIL_concatenateTwoTables(FileNamesTable* table1, FileNamesTable* table2)
-    unsigned newTableIdx = 0, idx1 = 0, idx2 = 0;
+{
-    size_t i = 0, pos = 0;
+    unsigned newTableIdx = 0;
-    size_t newTotalTableSize = 0;
+    size_t pos = 0;
+    size_t newTotalTableSize;
+    char* buf;
 
-    FileNamesTable* newTable = NULL;
+    fprintf(stderr, "UTIL_concatenateTwoTables \n");
 
-    char* buf = NULL;
+    FileNamesTable* const newTable = UTIL_createFileNamesTable(NULL, 0, NULL);
-
+    CONTROL( newTable != NULL );
-
-    newTable = UTIL_createFileNamesTable(NULL, NULL, 0);
-
-    if(!newTable) {
-      UTIL_DISPLAYLEVEL(1, "[ERROR][UTIL_concatenateTwoTables] Can't create new table for concatenation output.\n");
-      return NULL;
-    }
 
     newTotalTableSize = getTotalTableSize(table1) + getTotalTableSize(table2);
 
-    buf = (char*) malloc(newTotalTableSize * sizeof(char));
+    buf = (char*) calloc(newTotalTableSize, sizeof(*buf));
-    if(!buf) {
+    CONTROL ( buf != NULL );
-      UTIL_freeFileNamesTable(newTable);
-      UTIL_DISPLAYLEVEL(1, "[ERROR][UTIL_concatenateTwoTables] Can't create buf for concatenation output.\n");
-      return NULL;
-    }
-
-    for(i = 0; i < newTotalTableSize ; ++i) buf[i] = '\0';
-
-    newTable->tableSize = table1->tableSize + table2->tableSize;
-    newTable->fileNames = (const char **) malloc(newTable->tableSize * sizeof(char*));
-
-    if(!newTable->fileNames) {
-      UTIL_freeFileNamesTable(newTable);
-      if(buf) free(buf);
-      UTIL_DISPLAYLEVEL(1, "[ERROR][UTIL_concatenateTwoTables] Can't create new table for concatenation output.\n");
-      return NULL;
-    }
-
-    for (i = 0; i < newTable->tableSize; ++i)
-      newTable->fileNames[i] = NULL;
-
-    for( ; idx1 < table1->tableSize && table1->fileNames[idx1] && pos < newTotalTableSize; ++idx1, ++newTableIdx) {
-      size_t curLen = strlen(table1->fileNames[idx1]);
-      memcpy(buf+pos, table1->fileNames[idx1], curLen);
-      newTable->fileNames[newTableIdx] = buf+pos;
-      pos += curLen+1;
-    }
-
-
-    for( ; idx2 < table2->tableSize && table2->fileNames[idx2] && pos < newTotalTableSize ; ++idx2, ++newTableIdx) {
-      size_t curLen = strlen(table2->fileNames[idx2]);
-      memcpy(buf+pos, table2->fileNames[idx2], curLen);
-      newTable->fileNames[newTableIdx] = buf+pos;
-      pos += curLen+1;
-    }
-
-    if(pos > newTotalTableSize) {
-      UTIL_freeFileNamesTable(newTable);
-      if(buf) free(buf);
-      return NULL;
-    }
 
     newTable->buf = buf;
+    fprintf(stderr, "Size table1 = %u ,  table2 = %u \n", (unsigned)table1->tableSize, (unsigned)table2->tableSize);
+    newTable->tableSize = table1->tableSize + table2->tableSize;
+    newTable->fileNames = (const char **) calloc(newTable->tableSize, sizeof(*(newTable->fileNames)));
+    CONTROL ( newTable->fileNames != NULL );
+
+    {   unsigned idx1;
+        for( idx1=0 ; (idx1 < table1->tableSize) && table1->fileNames[idx1] && (pos < newTotalTableSize); ++idx1, ++newTableIdx) {
+            size_t const curLen = strlen(table1->fileNames[idx1]);
+            memcpy(buf+pos, table1->fileNames[idx1], curLen);
+            assert(newTableIdx <= newTable->tableSize);
+            newTable->fileNames[newTableIdx] = buf+pos;
+            pos += curLen+1;
+    }   }
+
+    {   unsigned idx2;
+        for( idx2=0 ; (idx2 < table2->tableSize) && table2->fileNames[idx2] && (pos < newTotalTableSize) ; ++idx2, ++newTableIdx) {
+            size_t const curLen = strlen(table2->fileNames[idx2]);
+            memcpy(buf+pos, table2->fileNames[idx2], curLen);
+            assert(newTableIdx <= newTable->tableSize);
+            newTable->fileNames[newTableIdx] = buf+pos;
+            pos += curLen+1;
+    }   }
+    assert(pos <= newTotalTableSize);
+    fprintf(stderr, "newTableIdx = %u ,  newTable->tableSize = %u \n", newTableIdx, (unsigned)newTable->tableSize);
+    newTable->tableSize = newTableIdx;
 
     UTIL_freeFileNamesTable(table1);
     UTIL_freeFileNamesTable(table2);
@@ -410,14 +390,17 @@ UTIL_concatenateTwoTables(FileNamesTable* table1, FileNamesTable* table2) {
 }
 
 #ifdef _WIN32
-int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
+int UTIL_prepareFileList(const char* dirName,
+                         char** bufStart, size_t* pos,
+                         char** bufEnd, int followLinks)
 {
     char* path;
-    int dirLength, fnameLength, pathLength, nbFiles = 0;
+    size_t dirLength;
+    int pathLength, nbFiles = 0;
     WIN32_FIND_DATAA cFile;
     HANDLE hFile;
 
-    dirLength = (int)strlen(dirName);
+    dirLength = strlen(dirName);
     path = (char*) malloc(dirLength + 3);
     if (!path) return 0;
 
@@ -434,7 +417,7 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
     free(path);
 
     do {
-        fnameLength = (int)strlen(cFile.cFileName);
+        size_t const fnameLength = strlen(cFile.cFileName);
         path = (char*) malloc(dirLength + fnameLength + 2);
         if (!path) { FindClose(hFile); return 0; }
         memcpy(path, dirName, dirLength);
@@ -462,8 +445,7 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
                 memcpy(*bufStart + *pos, path, pathLength+1 /* include final \0 */);
                 *pos += pathLength + 1;
                 nbFiles++;
-            }
+        }   }
-        }
         free(path);
     } while (FindNextFileA(hFile, &cFile));
 
@@ -473,12 +455,13 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
 
 #elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L)  /* opendir, readdir require POSIX.1-2001 */
 
-int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
+int UTIL_prepareFileList(const char *dirName,
+                         char** bufStart, size_t* pos,
+                         char** bufEnd, int followLinks)
 {
-    DIR *dir;
+    DIR* dir;
-    struct dirent *entry;
+    struct dirent * entry;
-    char* path;
+    size_t dirLength;
-    size_t dirLength, fnameLength, pathLength;
     int nbFiles = 0;
 
     if (!(dir = opendir(dirName))) {
@@ -489,6 +472,8 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
     dirLength = strlen(dirName);
     errno = 0;
     while ((entry = readdir(dir)) != NULL) {
+        char* path;
+        size_t fnameLength, pathLength;
         if (strcmp (entry->d_name, "..") == 0 ||
             strcmp (entry->d_name, ".") == 0) continue;
         fnameLength = strlen(entry->d_name);
@@ -522,8 +507,7 @@ int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char
                 memcpy(*bufStart + *pos, path, pathLength + 1);  /* with final \0 */
                 *pos += pathLength + 1;
                 nbFiles++;
-            }
+        }   }
-        }
         free(path);
         errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */
     }
@@ -605,11 +589,6 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
 }
 
 
-/*-****************************************
-*  Console log
-******************************************/
-int g_utilDisplayLevel;
-
 
 
 /*-****************************************

programs/util.h

@@ -142,12 +142,6 @@ U32 UTIL_isLink(const char* infilename);
 U64 UTIL_getFileSize(const char* infilename);
 
 U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles);
-/*! UTIL_readLineFromFile(char* buf, size_t len, File* file):
- * @return : int. size next line in file or -1 in case of file ends
- * function reads next line in the file
- * Will also modify `*file`, advancing it to position where it stopped reading.
- */
-int UTIL_readLineFromFile(char* buf, size_t len, FILE* file);
 
 /*Note: tableSize is denotes the total capacity of table*/
 typedef struct
@@ -171,7 +165,7 @@ FileNamesTable* UTIL_createFileNamesTable_fromFileName(const char* inputFileName
  */
 
 FileNamesTable*
-UTIL_createFileNamesTable(const char** filenames, char* buf, size_t tableSize);
+UTIL_createFileNamesTable(const char** filenames, size_t tableSize, char* buf);
 
 
 /*!  UTIL_freeFileNamesTable(FileNamesTable* table) :

programs/zstdcli.c

@@ -38,8 +38,7 @@
 #ifndef ZSTD_NODICT
 #  include "dibio.h"  /* ZDICT_cover_params_t, DiB_trainFromFiles() */
 #endif
-#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_minCLevel */
+#include "zstd.h"     /* ZSTD_VERSION_STRING, ZSTD_minCLevel, ZSTD_maxCLevel */
-#include "zstd.h"     /* ZSTD_VERSION_STRING, ZSTD_maxCLevel */
 
 
 /*-************************************
@@ -587,9 +586,6 @@ int main(int argCount, const char* argv[])
     unsigned memLimit = 0;
     size_t filenameTableSize = argCount;
     const char** filenameTable = (const char**)malloc(filenameTableSize * sizeof(const char*));   /* argCount >= 1 */
-    FileNamesTable* extendedTable = NULL;
-    FileNamesTable* concatenatedTables = NULL;
-    FileNamesTable* curTable = NULL;
     char* tableBuf = NULL;
     unsigned filenameIdx = 0;
     const char* programName = argv[0];
@@ -804,40 +800,49 @@ int main(int argCount, const char* argv[])
 #endif
 
                     if (longCommandWArg(&argument, "--file=")) {
+                        FileNamesTable* extendedTable;
+                        FileNamesTable* curTable;
+                        FileNamesTable* concatenatedTables;
 
-                        if(!UTIL_fileExist(argument) || !UTIL_isRegularFile(argument)){
+                        if (!UTIL_fileExist(argument) || !UTIL_isRegularFile(argument)){
-                          DISPLAYLEVEL(1, "[ERROR] wrong fileName: %s\n", argument);
+                            DISPLAYLEVEL(1, "[ERROR] wrong fileName: %s\n", argument);
-                          CLEAN_RETURN(badusage(programName));
+                            CLEAN_RETURN(badusage(programName));
                         }
 
                         extendedTable = UTIL_createFileNamesTable_fromFileName(argument);
-                        if(!extendedTable) {
+                        if (!extendedTable) {
-                          CLEAN_RETURN(badusage(programName));
+                            CLEAN_RETURN(badusage(programName));
                         }
 
 
                         filenameTable[filenameIdx] = NULL; // marking end of table
+                        filenameIdx += (unsigned) extendedTable->tableSize;
 
-                        curTable = UTIL_createFileNamesTable(filenameTable, tableBuf, filenameTableSize);
+                        curTable = UTIL_createFileNamesTable(filenameTable, filenameTableSize, tableBuf);
 
-                        if(!curTable) {
+                        if (!curTable) {
-                          UTIL_freeFileNamesTable(extendedTable);
+                            UTIL_freeFileNamesTable(extendedTable);
-                          CLEAN_RETURN(badusage(programName));
+                            CLEAN_RETURN(badusage(programName));
                         }
 
                         concatenatedTables = UTIL_concatenateTwoTables(curTable, extendedTable);
-                        if(!concatenatedTables) {
+                        if (!concatenatedTables) {
-                          UTIL_freeFileNamesTable(curTable);
+                            if (!isTableBufferBased) curTable->buf = NULL;
-                          UTIL_freeFileNamesTable(extendedTable);
+                            UTIL_freeFileNamesTable(curTable);
-                          CLEAN_RETURN(badusage(programName));
+                            UTIL_freeFileNamesTable(extendedTable);
+                            CLEAN_RETURN(badusage(programName));
                         }
 
+                        /* transfer ownership */
                         filenameTable = concatenatedTables->fileNames;
                         filenameTableSize = concatenatedTables->tableSize;
                         tableBuf = concatenatedTables->buf;
+                        concatenatedTables->fileNames = NULL;
+                        concatenatedTables->tableSize = 0;
+                        concatenatedTables->buf = NULL;
+                        UTIL_freeFileNamesTable(concatenatedTables);
 
-                        filenameIdx += (unsigned) extendedTable->tableSize;
+                        isTableBufferBased = 1;   /* file names are now in heap */
-                        isTableBufferBased = 1;
 
                         continue;
                     }
@@ -1106,7 +1111,7 @@ int main(int argCount, const char* argv[])
         if (cLevelLast < cLevel) cLevelLast = cLevel;
         if (cLevelLast > cLevel)
             DISPLAYLEVEL(3, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast);
-        if(filenameIdx) {
+        if (filenameIdx) {
             if(separateFiles) {
                 unsigned i;
                 for(i = 0; i < filenameIdx; i++) {
@@ -1114,18 +1119,15 @@ int main(int argCount, const char* argv[])
                     DISPLAYLEVEL(3, "Benchmarking %s \n", filenameTable[i]);
                     for(c = cLevel; c <= cLevelLast; c++) {
                         BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams);
-                    }
+                }   }
-                }
             } else {
                 for(; cLevel <= cLevelLast; cLevel++) {
                     BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams);
-                }
+            }   }
-            }
         } else {
             for(; cLevel <= cLevelLast; cLevel++) {
                 BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams);
-            }
+        }   }
-        }
 
 #else
         (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility;
@@ -1238,10 +1240,11 @@ int main(int argCount, const char* argv[])
             }
         }
         FIO_setMemLimit(prefs, memLimit);
-        if (filenameIdx==1 && outFileName)
+        if (filenameIdx==1 && outFileName) {
             operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName);
-        else
+        } else {
             operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outDirName, outFileName, dictFileName);
+        }
 #else
         DISPLAY("Decompression not supported \n");
 #endif
@@ -1255,9 +1258,6 @@ _end:
          free(tableBuf);
        }
     }
-    UTIL_freeFileNamesTable(curTable);
-    UTIL_freeFileNamesTable(extendedTable);
-    UTIL_freeFileNamesTable(concatenatedTables);
 
     if (main_pause) waitEnter();
 #ifdef UTIL_HAS_CREATEFILELIST

tests/playTests.sh

@@ -293,49 +293,45 @@ test -f tmpOutDirDecomp/tmp2
 test -f tmpOutDirDecomp/tmp1
 rm -rf tmp*
 
+
 println "test : compress multiple files reading them from a file, --file=FILE"
-mkdir tmpInputTestDir
+println "Hello world!, file1" > tmp1
-println "Hello world!, file1" > tmpInputTestDir/file1
+println "Hello world!, file2" > tmp2
-println "Hello world!, file2" > tmpInputTestDir/file2
+println tmp1 > tmp_fileList
-println tmpInputTestDir/file1 > tmp
+println tmp2 >> tmp_fileList
-println tmpInputTestDir/file2 >> tmp
+$ZSTD -f --file=tmp_fileList
-$ZSTD -f --file=tmp
+test -f tmp2.zst
-test -f tmpInputTestDir/file2.zst
+test -f tmp1.zst
-test -f tmpInputTestDir/file1.zst
+rm -f *.zst
-rm tmpInputTestDir/*.zst
 
 println "test : compress multiple files reading them from multiple files, --file=FILE"
-println "Hello world!, file3" > tmpInputTestDir/file3
+println "Hello world!, file3" > tmp3
-println "Hello world!, file4" > tmpInputTestDir/file4
+println "Hello world!, file4" > tmp4
-println tmpInputTestDir/file3 > tmp1
+println tmp3 > tmp_fileList2
-println tmpInputTestDir/file4 >> tmp1
+println tmp4 >> tmp_fileList2
-$ZSTD -f --file=tmp --file=tmp1
+$ZSTD -f --file=tmp_fileList --file=tmp_fileList2
-test -f tmpInputTestDir/file1.zst
+test -f tmp1.zst
-test -f tmpInputTestDir/file2.zst
+test -f tmp2.zst
-test -f tmpInputTestDir/file3.zst
+test -f tmp3.zst
-test -f tmpInputTestDir/file4.zst
+test -f tmp4.zst
 
 println "test : decompress multiple files reading them from a file, --file=FILE"
-rm tmpInputTestDir/file1
+rm -f tmp1 tmp2
-rm tmpInputTestDir/file2
+println tmp1.zst > tmpZst
-println tmpInputTestDir/file1.zst > tmpZst
+println tmp2.zst >> tmpZst
-println tmpInputTestDir/file2.zst >> tmpZst
 $ZSTD -d -f --file=tmpZst
-test -f tmpInputTestDir/file2
+test -f tmp1
-test -f tmpInputTestDir/file1
+test -f tmp2
 
 println "test : decompress multiple files reading them from multiple files, --file=FILE"
-rm tmpInputTestDir/file1
+rm -f tmp1 tmp2 tmp3 tmp4
-rm tmpInputTestDir/file2
+println tmp3.zst > tmpZst2
-rm tmpInputTestDir/file3
+println tmp4.zst >> tmpZst2
-rm tmpInputTestDir/file4
+$ZSTD -d -f --file=tmpZst --file=tmpZst2
-println tmpInputTestDir/file3.zst > tmpZst1
+test -f tmp1
-println tmpInputTestDir/file4.zst >> tmpZst1
+test -f tmp2
-$ZSTD -d -f --file=tmpZst --file=tmpZst1
+test -f tmp3
-test -f tmpInputTestDir/file1
+test -f tmp4
-test -f tmpInputTestDir/file2
-test -f tmpInputTestDir/file3
-test -f tmpInputTestDir/file4
 
 rm -rf tmp*