Added : ability to create compressed frames without dictID

programs/bench.h

@@ -21,10 +21,11 @@
     You can contact the author at :
     - ZSTD homepage : http://www.zstd.net/
 */
-#pragma once
+#ifndef BENCH_H_121279284357
+#define BENCH_H_121279284357
 
+#include <stddef.h>
 
-/* Main function */
 int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
                    const char* dictFileName, int cLevel, int cLevelLast);
 
@@ -34,3 +35,4 @@ void BMK_SetBlockSize(size_t blockSize);
 void BMK_setAdditionalParam(int additionalParam);
 void BMK_setNotificationLevel(unsigned level);
 
+ #endif   /* BENCH_H_121279284357 */

programs/fileio.c

@@ -133,6 +133,8 @@ static U32 g_maxWLog = 23;
 void FIO_setMaxWLog(unsigned maxWLog) { g_maxWLog = maxWLog; }
 static U32 g_sparseFileSupport = 1;   /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
 void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; }
+static U32 g_dictIDFlag = 1;
+void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; }
 
 
 /*-*************************************
@@ -186,7 +188,7 @@ static FILE* FIO_openDstFile(const char* dstFileName)
             DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
         }
     } else {
-        if (!g_overwrite) {  /* Check if destination file already exists */
+        if (!g_overwrite && strcmp (dstFileName, nulmark)) {  /* Check if destination file already exists */
             f = fopen( dstFileName, "rb" );
             if (f != 0) {  /* dest file exists, prompt for overwrite authorization */
                 fclose(f);
@@ -311,6 +313,7 @@ static int FIO_compressFilename_internal(cRess_t ress,
         memset(&params, 0, sizeof(params));
         params.cParams = ZSTD_getCParams(cLevel, fileSize, ress.dictBufferSize);
         params.fParams.contentSizeFlag = 1;
+        params.fParams.noDictIDFlag = !g_dictIDFlag;
         if (g_maxWLog) if (params.cParams.windowLog > g_maxWLog) params.cParams.windowLog = g_maxWLog;
         { size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
           if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode)); }
@@ -410,14 +413,10 @@ static int FIO_compressFilename_extRess(cRess_t ress,
 int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
                          const char* dictFileName, int compressionLevel)
 {
-    clock_t start;
+    clock_t const start = clock();
-    cRess_t ress;
+    cRess_t const ress = FIO_createCResources(dictFileName);
     int issueWithSrcFile = 0;
 
-    /* Init */
-    start = clock();
-    ress = FIO_createCResources(dictFileName);
-
     issueWithSrcFile += FIO_compressFilename_extRess(ress, dstFileName, srcFileName, compressionLevel);
 
     FIO_freeCResources(ress);
@@ -702,7 +701,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
 
     /* Final Status */
     DISPLAYLEVEL(2, "\r%79s\r", "");
-    DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize);
+    DISPLAYLEVEL(2, "%-20.20s: %llu bytes \n", srcFileName, filesize);
 
     /* Close */
     fclose(srcFile);

programs/fileio.h

@@ -48,6 +48,7 @@ void FIO_overwriteMode(void);
 void FIO_setNotificationLevel(unsigned level);
 void FIO_setMaxWLog(unsigned maxWLog);     /**< if `maxWLog` == 0, no max enforced */
 void FIO_setSparseWrite(unsigned sparse);  /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
+void FIO_setDictIDFlag(unsigned dictIDFlag);
 
 
 /*-*************************************

programs/tests/playTests.sh

@@ -123,14 +123,23 @@ $ECHO "\n**** dictionary tests **** "
 ./datagen -g1M | md5sum > tmp1
 ./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | md5sum > tmp2
 diff -q tmp1 tmp2
+$ECHO "Create first dictionary"
 $ZSTD --train *.c -o tmpDict
-$ZSTD zstdcli.c -D tmpDict -of tmp
+cp zstdcli.c tmp
-$ZSTD -d tmp -D tmpDict -of result
+$ZSTD -f tmp -D tmpDict
+$ZSTD -d tmp.zst -D tmpDict -of result
 diff zstdcli.c result
+$ECHO "Create second (different) dictionary"
 $ZSTD --train *.c *.h -o tmpDictC
-$ZSTD -d tmp -D tmpDictC -of result && die "wrong dictionary not detected!"
+$ZSTD -d tmp.zst -D tmpDictC -of result && die "wrong dictionary not detected!"
+$ECHO "Create dictionary with short dictID"
 $ZSTD --train *.c --dictID 1 -o tmpDict1
 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
+$ECHO "Compress without dictID"
+$ZSTD -f tmp -D tmpDict1 --no-dictID
+$ZSTD -d tmp.zst -D tmpDict -of result
+diff zstdcli.c result
+rm tmp*
 
 
 $ECHO "\n**** multiple files tests **** "

programs/zstdcli.c

@@ -125,7 +125,6 @@ static int usage_advanced(const char* programName)
     DISPLAY( "\n");
     DISPLAY( "Advanced arguments :\n");
     DISPLAY( " -V     : display Version number and exit\n");
-    DISPLAY( " -t     : test compressed file integrity \n");
     DISPLAY( " -v     : verbose mode\n");
     DISPLAY( " -q     : suppress warnings; specify twice to suppress errors too\n");
     DISPLAY( " -c     : force write to standard output, even if it is the console\n");
@@ -134,8 +133,12 @@ static int usage_advanced(const char* programName)
 #endif
 #ifndef ZSTD_NOCOMPRESS
     DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n");
+    DISPLAY( "--no-dictID:don't write dictID into header (dictionary compression)\n");
 #endif
+#ifndef ZSTD_NODECOMPRESS
+    DISPLAY( " -t     : test compressed file integrity \n");
     DISPLAY( "--[no-]sparse  : sparse mode (default:enabled on file, disabled on stdout)\n");
+#endif
 #ifndef ZSTD_NODICT
     DISPLAY( "\n");
     DISPLAY( "Dictionary builder :\n");
@@ -236,14 +239,15 @@ int main(int argCount, const char** argv)
         if (!strcmp(argument, "--verbose")) { displayLevel=4; continue; }
         if (!strcmp(argument, "--quiet")) { displayLevel--; continue; }
         if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel=1; continue; }
+        if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; }
+        if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
+        if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
+        if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
         if (!strcmp(argument, "--test")) { decode=1; outFileName=nulmark; FIO_overwriteMode(); continue; }
         if (!strcmp(argument, "--train")) { dictBuild=1; outFileName=g_defaultDictName; continue; }
         if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; }
         if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; continue; }
         if (!strcmp(argument, "--keep")) { continue; }   /* does nothing, since preserving input is default; for gzip/xz compatibility */
-        if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; }
-        if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
-        if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
 
         /* '-' means stdin/stdout */
         if (!strcmp(argument, "-")){
@@ -300,7 +304,7 @@ int main(int argCount, const char** argv)
                 case 'k': argument++; break;
 
                     /* test compressed file */
-                case 't': decode=1; outFileName=nulmark; FIO_overwriteMode(); argument++; break;
+                case 't': decode=1; outFileName=nulmark; argument++; break;
 
                     /* dictionary name */
                 case 'o': nextArgumentIsOutFileName=1; argument++; break;