Pass-Through mode support (using -df), for compatibility with gzip

NEWS

@@ -1,3 +1,7 @@
+v0.6.2
+New : Support for Sparse File-systems (do not use space for zero-filled sectors)
+New : Support pass-through mode, when using `-df`
+
 v0.6.1
 New : zlib wrapper API, thanks to Przemyslaw Skibinski
 New : Ability to compile compressor / decompressor separately

programs/fileio.c

@@ -639,6 +639,28 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
 }
 
 
+/** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode
+    @return : 0 (no error) */
+static unsigned FIO_passThrough(FILE* foutput, FILE* finput, unsigned char* buffer, size_t bufferSize)
+{
+    size_t const blockSize = MIN (64 KB, bufferSize);
+    size_t read = 1;
+    unsigned storedSkips = 0;
+
+    /* assumption : first 4 bytes already loaded (magic number detection), and stored within buffer */
+    { size_t const sizeCheck = fwrite(buffer, 1, 4, foutput);
+      if (sizeCheck != 4) EXM_THROW(50, "Pass-through write error"); }
+
+    while (read) {
+        read = fread(buffer, 1, blockSize, finput);
+        storedSkips = FIO_fwriteSparse(foutput, buffer, read, storedSkips);
+    }
+
+    FIO_fwriteSparseEnd(foutput, storedSkips);
+    return 0;
+}
+
+
 /** FIO_decompressSrcFile() :
     Decompression `srcFileName` into `ress.dstFile`
     @return : 0 : OK
@@ -666,9 +688,12 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
             }
 #endif
             if (magic !=  ZSTD_MAGICNUMBER) {
-                DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName);
+                if (g_overwrite)   /* -df : pass-through mode */
-                return 1;
+                    return FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize);
-        }   }
+                else {
+                    DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName);
+                    return 1;
+        }   }   }
         filesize += FIO_decompressFrame(ress, dstFile, srcFile, toRead);
     }
 

programs/playTests.sh

@@ -24,6 +24,7 @@ roundTripTest() {
 
 
 echo "\n**** simple tests **** "
+
 ./datagen > tmp
 $ZSTD -f tmp                             # trivial compression case, creates tmp.zst
 $ZSTD -df tmp.zst                        # trivial decompression case (overwrites tmp)
@@ -47,9 +48,12 @@ $ZSTD -q tmp && die "overwrite check failed!"
 $ZSTD -q -f tmp
 $ZSTD -q --force tmp
 $ZSTD -df tmp && die "should have refused : wrong extension"
-cp tmp tmp2.zst
+
-$ZSTD -df tmp2.zst && die "should have failed : wrong format"
+
-rm tmp2.zst
+echo "\n**** Pass-Through mode **** "
+echo "Hello world !" | $ZSTD -df
+echo "Hello world !" | $ZSTD -dcf
+
 
 echo "\n**** frame concatenation **** "
 
@@ -63,8 +67,7 @@ $ZSTD -dc helloworld.zstd > result.tmp
 cat result.tmp
 sdiff helloworld.tmp result.tmp
 rm ./*.tmp ./*.zstd
-
+echo "frame concatenation tests completed"
-echo frame concatenation test completed
 
 
 echo "\n**** flush write error test **** "
@@ -136,7 +139,9 @@ ls -ls tmp*
 echo "compress multiple files including a missing one (notHere) : "
 $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
 
+
 echo "\n**** integrity tests **** "
+
 echo "test one file (tmp1.zst) "
 $ZSTD -t tmp1.zst
 $ZSTD --test tmp1.zst
@@ -145,6 +150,7 @@ $ZSTD -t *.zst
 echo "test good and bad files (*) "
 $ZSTD -t * && die "bad files not detected !"
 
+
 echo "\n**** zstd round-trip tests **** "
 
 roundTripTest

programs/zstdcli.c

@@ -279,7 +279,7 @@ int main(int argCount, const char** argv)
                 case 'D': nextEntryIsDictionary = 1; argument++; break;
 
                     /* Overwrite */
-                case 'f': FIO_overwriteMode(); argument++; break;
+                case 'f': FIO_overwriteMode(); forceStdout=1; argument++; break;
 
                     /* Verbose mode */
                 case 'v': displayLevel=4; argument++; break;
@@ -422,7 +422,8 @@ int main(int argCount, const char** argv)
 
     /* Check if input/output defined as console; trigger an error in this case */
     if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) CLEAN_RETURN(badusage(programName));
-    if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && !forceStdout) CLEAN_RETURN(badusage(programName));
+    if (outFileName && !strcmp(outFileName, stdoutmark) && IS_CONSOLE(stdout) && !(forceStdout && decode))
+        CLEAN_RETURN(badusage(programName));
 
     /* user-selected output filename, only possible with a single file */
     if (outFileName && strcmp(outFileName,stdoutmark) && strcmp(outFileName,nulmark) && (filenameIdx>1)) {