Added : Sparse write support

--[no-]sparse command

programs/fileio.c

@@ -130,6 +130,8 @@ static U32 g_overwrite = 0;
 void FIO_overwriteMode(void) { g_overwrite=1; }
 static U32 g_maxWLog = 23;
 void FIO_setMaxWLog(unsigned maxWLog) { g_maxWLog = maxWLog; }
+static U32 g_sparseFileSupport = 1;   /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
+void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; }
 
 
 /*-*************************************
@@ -178,6 +180,10 @@ static FILE* FIO_openDstFile(const char* dstFileName)
         DISPLAYLEVEL(4,"Using stdout for output\n");
         f = stdout;
         SET_BINARY_MODE(stdout);
+        if (g_sparseFileSupport==1) {
+            g_sparseFileSupport = 0;
+            DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
+        }
     } else {
         if (!g_overwrite) {  /* Check if destination file already exists */
             f = fopen( dstFileName, "rb" );
@@ -189,8 +195,7 @@ static FILE* FIO_openDstFile(const char* dstFileName)
                     return 0;
                 }
                 DISPLAY("zstd: %s already exists; do you wish to overwrite (y/N) ? ", dstFileName);
-                {
+                {   int ch = getchar();
-                    int ch = getchar();
                     if ((ch!='Y') && (ch!='y')) {
                         DISPLAY("    not overwritten  \n");
                         return 0;
@@ -512,6 +517,81 @@ static void FIO_freeDResources(dRess_t ress)
 }
 
 
+/** FIO_fwriteSparse() :
+*   @return : storedSkips, to be provided to next call to FIO_fwriteSparse() of LZ4IO_fwriteSparseEnd() */
+static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips)
+{
+    const size_t* const bufferT = (const size_t*)buffer;   /* Buffer is supposed malloc'ed, hence aligned on size_t */
+    size_t bufferSizeT = bufferSize / sizeof(size_t);
+    const size_t* const bufferTEnd = bufferT + bufferSizeT;
+    const size_t* ptrT = bufferT;
+    static const size_t segmentSizeT = (32 KB) / sizeof(size_t);   /* 0-test re-attempted every 32 KB */
+
+    if (!g_sparseFileSupport) {  /* normal write */
+        size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
+        if (sizeCheck != bufferSize) EXM_THROW(70, "Write error : cannot write decoded block");
+        return 0;
+    }
+
+    /* avoid int overflow */
+    if (storedSkips > 1 GB) {
+        int const seekResult = fseek(file, 1 GB, SEEK_CUR);
+        if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)");
+        storedSkips -= 1 GB;
+    }
+
+    while (ptrT < bufferTEnd) {
+        size_t seg0SizeT = segmentSizeT;
+        size_t nb0T;
+
+        /* count leading zeros */
+        if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT;
+        bufferSizeT -= seg0SizeT;
+        for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ;
+        storedSkips += (unsigned)(nb0T * sizeof(size_t));
+
+        if (nb0T != seg0SizeT) {   /* not all 0s */
+            int const seekResult = fseek(file, storedSkips, SEEK_CUR);
+            if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse");
+            storedSkips = 0;
+            seg0SizeT -= nb0T;
+            ptrT += nb0T;
+            {   size_t const sizeCheck = fwrite(ptrT, sizeof(size_t), seg0SizeT, file);
+                if (sizeCheck != seg0SizeT) EXM_THROW(73, "Write error : cannot write decoded block");
+        }   }
+        ptrT += seg0SizeT;
+    }
+
+    {   static size_t const maskT = sizeof(size_t)-1;
+        if (bufferSize & maskT) {   /* size not multiple of sizeof(size_t) : implies end of block */
+            const char* const restStart = (const char*)bufferTEnd;
+            const char* restPtr = restStart;
+            size_t restSize =  bufferSize & maskT;
+            const char* const restEnd = restStart + restSize;
+            for ( ; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ;
+            storedSkips += (unsigned) (restPtr - restStart);
+            if (restPtr != restEnd) {
+                int seekResult = fseek(file, storedSkips, SEEK_CUR);
+                if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse");
+                storedSkips = 0;
+                {   size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file);
+                    if (sizeCheck != (size_t)(restEnd - restPtr)) EXM_THROW(75, "Write error : cannot write decoded end of block");
+    }   }   }   }
+
+    return storedSkips;
+}
+
+static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
+{
+    if (storedSkips-->0) {   /* implies g_sparseFileSupport>0 */
+        int const seekResult = fseek(file, storedSkips, SEEK_CUR);
+        if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n");
+        {   const char lastZeroByte[1] = { 0 };
+            size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file);
+            if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n");
+    }   }
+}
+
 /** FIO_decompressFrame() :
     @return : size of decoded frame
 */
@@ -520,6 +600,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
 {
     U64    frameSize = 0;
     size_t readSize;
+    U32 storedSkips = 0;
 
     ZBUFF_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
 
@@ -538,8 +619,7 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
         readSize -= inSize;
 
         /* Write block */
-        { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput);
+        storedSkips = FIO_fwriteSparse(foutput, ress.dstBuffer, decodedSize, storedSkips);
-          if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block into destination"); }
         frameSize += decodedSize;
         DISPLAYUPDATE(2, "\rDecoded : %u MB...     ", (U32)(frameSize>>20) );
 
@@ -553,6 +633,8 @@ unsigned long long FIO_decompressFrame(dRess_t ress,
             EXM_THROW(35, "Read error");
     }
 
+    FIO_fwriteSparseEnd(foutput, storedSkips);
+
     return frameSize;
 }
 

programs/fileio.h

@@ -46,7 +46,8 @@ extern "C" {
 ***************************************/
 void FIO_overwriteMode(void);
 void FIO_setNotificationLevel(unsigned level);
-void FIO_setMaxWLog(unsigned maxWLog);   /**< if `maxWLog` == 0, no max enforced */
+void FIO_setMaxWLog(unsigned maxWLog);     /**< if `maxWLog` == 0, no max enforced */
+void FIO_setSparseWrite(unsigned sparse);  /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
 
 
 /*-*************************************

programs/zstdcli.c

@@ -132,6 +132,7 @@ static int usage_advanced(const char* programName)
 #ifndef ZSTD_NOCOMPRESS
     DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n");
 #endif
+    DISPLAY( "--[no-]sparse  : sparse mode (default:enabled on file, disabled on stdout)\n");
 #ifndef ZSTD_NODICT
     DISPLAY( "\n");
     DISPLAY( "Dictionary builder :\n");
@@ -229,6 +230,8 @@ int main(int argCount, const char** argv)
         if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; }
         if (!strcmp(argument, "--keep")) { continue; }   /* does nothing, since preserving input is default; for gzip/xz compatibility */
         if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; }
+        if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
+        if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
 
         /* '-' means stdin/stdout */
         if (!strcmp(argument, "-")){