Added : Frame concatenation ability

lib/zstd.c

@@ -1668,14 +1668,20 @@ size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t src
 *  Streaming Decompression API
 *******************************/
 
-ZSTD_Dctx* ZSTD_createDCtx(void)
+size_t ZSTD_resetDCtx(ZSTD_Dctx* dctx)
 {
-    ZSTD_Dctx* dctx = (ZSTD_Dctx*)malloc(sizeof(ZSTD_Dctx));
-    if (dctx==NULL) return NULL;
     dctx->expected = ZSTD_frameHeaderSize;
     dctx->phase = 0;
     dctx->previousDstEnd = NULL;
     dctx->base = NULL;
+    return 0;
+}
+
+ZSTD_Dctx* ZSTD_createDCtx(void)
+{
+    ZSTD_Dctx* dctx = (ZSTD_Dctx*)malloc(sizeof(ZSTD_Dctx));
+    if (dctx==NULL) return NULL;
+    ZSTD_resetDCtx(dctx);
     return dctx;
 }
 
@@ -1685,7 +1691,6 @@ size_t ZSTD_freeDCtx(ZSTD_Dctx* dctx)
     return 0;
 }
 
-
 size_t ZSTD_nextSrcSizeToDecompress(ZSTD_Dctx* dctx)
 {
     return ((dctx_t*)dctx)->expected;

lib/zstd_static.h

@@ -56,6 +56,7 @@ size_t ZSTD_compressEnd(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize);
 
 typedef struct ZSTD_Dctx_s ZSTD_Dctx;
 ZSTD_Dctx* ZSTD_createDCtx(void);
+size_t     ZSTD_resetDCtx(ZSTD_Dctx* dctx);
 size_t     ZSTD_freeDCtx(ZSTD_Dctx* dctx);
 
 size_t ZSTD_nextSrcSizeToDecompress(ZSTD_Dctx* dctx);

programs/Makefile

@@ -120,10 +120,22 @@ test32: test-zstd32 test-fullbench32 test-fuzzer32
 test-all: test test32 memtest
 
 test-zstd: zstd datagen
-	@echo "*** flush write error test ***"
+	@echo "\n**** frame concatenation **** "
+	@echo "hello " > hello.tmp
+	@echo "world!" > world.tmp
+	@cat hello.tmp world.tmp > helloworld.tmp
+	./zstd hello.tmp > hello.zstd
+	./zstd world.tmp > world.zstd
+	@cat hello.zstd world.zstd > helloworld.zstd
+	./zstd -d helloworld.zstd > result.tmp
+	cat result.tmp
+	sdiff helloworld.tmp result.tmp
+	@rm *.tmp *.zstd
+	@echo frame concatenation test completed
+	@echo "**** flush write error test **** "
 	echo foo | ./zstd > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
 	echo foo | ./zstd | ./zstd -d > /dev/full; if [ $$? -eq 0 ] ; then echo "write error not detected!"; false; fi
-	@echo "*** zstd round-trip tests *** "
+	@echo "**** zstd round-trip tests **** "
 	./datagen          | ./zstd -v    | ./zstd -d > $(VOID)
 	./datagen -g256MB  | ./zstd -v    | ./zstd -d > $(VOID)
 	./datagen -g6GB -P99 | ./zstd -vq | ./zstd -d > $(VOID)

programs/fileio.c

@@ -309,45 +309,18 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
 
 
 #define MAXHEADERSIZE FIO_FRAMEHEADERSIZE+3
-unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
+unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
+                                       BYTE* inBuff, size_t inBuffSize,
+                                       BYTE* outBuff, size_t outBuffSize,
+                                       ZSTD_Dctx* dctx)
 {
-    FILE* finput, *foutput;
+    BYTE* op = outBuff;
-    BYTE* inBuff;
+    BYTE* const oend = outBuff + outBuffSize;
-    size_t inBuffSize;
-    BYTE* outBuff, *op, *oend;
-    size_t outBuffSize;
-    U32   blockSize = 128 KB;
-    U32   wNbBlocks = 4;
     U64   filesize = 0;
-    BYTE* header[MAXHEADERSIZE];
-    ZSTD_Dctx* dctx;
     size_t toRead;
     size_t sizeCheck;
 
 
-    /* Init */
-    FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
-    dctx = ZSTD_createDCtx();
-
-    /* check header */
-    toRead = ZSTD_nextSrcSizeToDecompress(dctx);
-    if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
-    sizeCheck = fread(header, (size_t)1, toRead, finput);
-    if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
-    sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead);   // Decode frame header
-    if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
-
-    /* Here later : blockSize determination */
-
-    /* Allocate Memory */
-    inBuffSize = blockSize + FIO_blockHeaderSize;
-    inBuff  = (BYTE*)malloc(inBuffSize);
-    outBuffSize = wNbBlocks * blockSize;
-    outBuff = (BYTE*)malloc(outBuffSize);
-    op = outBuff;
-    oend = outBuff + outBuffSize;
-    if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
-
     /* Main decompression Loop */
     toRead = ZSTD_nextSrcSizeToDecompress(dctx);
     while (toRead)
@@ -380,6 +353,67 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
         toRead = ZSTD_nextSrcSizeToDecompress(dctx);
     }
 
+    return filesize;
+}
+
+
+unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
+{
+    FILE* finput, *foutput;
+    BYTE* inBuff=NULL;
+    size_t inBuffSize = 0;
+    BYTE* outBuff=NULL;
+    size_t outBuffSize = 0;
+    U32   blockSize = 128 KB;
+    U32   wNbBlocks = 4;
+    U64   filesize = 0;
+    BYTE* header[MAXHEADERSIZE];
+    ZSTD_Dctx* dctx;
+    size_t toRead;
+    size_t sizeCheck;
+
+
+    /* Init */
+    FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
+    dctx = ZSTD_createDCtx();
+
+    /* for each frame */
+    for ( ; ; )
+    {
+        /* check header */
+        ZSTD_resetDCtx(dctx);
+        toRead = ZSTD_nextSrcSizeToDecompress(dctx);
+        if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
+        sizeCheck = fread(header, (size_t)1, toRead, finput);
+        if (sizeCheck==0) break;   /* no more input */
+        if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
+        sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, toRead);   // Decode frame header
+        if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
+
+        /* Here later : blockSize determination */
+
+        /* Allocate Memory (if needed) */
+        {
+            size_t newInBuffSize = blockSize + FIO_blockHeaderSize;
+            size_t newOutBuffSize = wNbBlocks * blockSize;
+            if (newInBuffSize > inBuffSize)
+            {
+                free(inBuff);
+                inBuffSize = newInBuffSize;
+                inBuff  = (BYTE*)malloc(inBuffSize);
+            }
+            if (newOutBuffSize > outBuffSize)
+            {
+                free(outBuff);
+                outBuffSize = newOutBuffSize;
+                outBuff  = (BYTE*)malloc(outBuffSize);
+            }
+        }
+        if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
+
+        filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, outBuff, outBuffSize, dctx);
+    }
+
     DISPLAYLEVEL(2, "\r%79s\r", "");
     DISPLAYLEVEL(2,"Decoded %llu bytes   \n", (long long unsigned)filesize);