AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

support for legacy 0.3 format

Browse Source
This commit is contained in:
Yann Collet 2015-11-26 16:02:04 +01:00
parent 661f843637
commit 29a2c838c5
6 changed files with 3910 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lib/Makefile
View File

@ -70,7 +70,8 @@ default: clean libzstd
all: clean libzstd
libzstd: zstd_compress.c zstd_decompress.c huff0.c fse.c legacy/zstd_v01.c legacy/zstd_v02.c
libzstd: zstd_compress.c zstd_decompress.c huff0.c fse.c \
legacy/zstd_v01.c legacy/zstd_v02.c legacy/zstd_v03.c
@echo compiling static library
@$(CC) $(FLAGS) -c $^
@$(AR) rcs libzstd.a *.o

8
lib/legacy/zstd_legacy.h
View File

@ -1,5 +1,5 @@
/*
zstd_v02 - decoder for 0.2 format
zstd_legacy - decoder for legacy format
Header File
Copyright (C) 2015, Yann Collet.
@ -44,13 +44,15 @@ extern "C" {
#include "error.h" /* ERROR */
#include "zstd_v01.h"
#include "zstd_v02.h"
#include "zstd_v03.h"
MEM_STATIC unsigned ZSTD_isLegacy (U32 magicNumberLE)
{
switch(magicNumberLE)
{
case ZSTDv01_magicNumberLE :
case ZSTDv02_magicNumber : return 1;
case ZSTDv02_magicNumber :
case ZSTDv03_magicNumber : return 1;
default : return 0;
}
}
@ -67,6 +69,8 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
return ZSTDv01_decompress(dst, maxOriginalSize, src, compressedSize);
case ZSTDv02_magicNumber :
return ZSTDv02_decompress(dst, maxOriginalSize, src, compressedSize);
case ZSTDv03_magicNumber :
return ZSTDv03_decompress(dst, maxOriginalSize, src, compressedSize);
default :
return ERROR(prefix_unknown);
}

3730
lib/legacy/zstd_v03.c Normal file
View File

File diff suppressed because it is too large Load Diff

99
lib/legacy/zstd_v03.h Normal file
View File

@ -0,0 +1,99 @@
/*
zstd_v03 - decoder for 0.3 format
Header File
Copyright (C) 2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- zstd source repository : https://github.com/Cyan4973/zstd
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
*/
#pragma once
#if defined (__cplusplus)
extern "C" {
#endif
/* *************************************
* Includes
***************************************/
#include <stddef.h> /* size_t */
/* *************************************
* Simple one-step function
***************************************/
/**
ZSTDv03_decompress() : decompress ZSTD frames compliant with v0.3.x format
compressedSize : is the exact source size
maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated.
It must be equal or larger than originalSize, otherwise decompression will fail.
return : the number of bytes decompressed into destination buffer (originalSize)
or an errorCode if it fails (which can be tested using ZSTDv01_isError())
*/
size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
const void* src, size_t compressedSize);
/**
ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error
*/
unsigned ZSTDv03_isError(size_t code);
/* *************************************
* Advanced functions
***************************************/
typedef struct ZSTDv03_Dctx_s ZSTDv03_Dctx;
ZSTDv03_Dctx* ZSTDv03_createDCtx(void);
size_t ZSTDv03_freeDCtx(ZSTDv03_Dctx* dctx);
size_t ZSTDv03_decompressDCtx(void* ctx,
void* dst, size_t maxOriginalSize,
const void* src, size_t compressedSize);
/* *************************************
* Streaming functions
***************************************/
size_t ZSTDv03_resetDCtx(ZSTDv03_Dctx* dctx);
size_t ZSTDv03_nextSrcSizeToDecompress(ZSTDv03_Dctx* dctx);
size_t ZSTDv03_decompressContinue(ZSTDv03_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
/**
Use above functions alternatively.
ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block.
Result is the number of bytes regenerated within 'dst'.
It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
*/
/* *************************************
* Prefix - version detection
***************************************/
#define ZSTDv03_magicNumber 0xFD2FB523 /* v0.3 */
#if defined (__cplusplus)
}
#endif

20
programs/Makefile
View File

@ -62,48 +62,48 @@ default: zstd
all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 zbufftest zbufftest32 paramgrill datagen
zstd: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/zstd_buffered.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c $(ZSTDDIR)/legacy/zstd_v03.c \
xxhash.c bench.c fileio.c zstdcli.c legacy/fileio_legacy.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
zstd32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/zstd_buffered.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c $(ZSTDDIR)/legacy/zstd_v03.c \
xxhash.c bench.c fileio.c zstdcli.c legacy/fileio_legacy.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
fullbench : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c $(ZSTDDIR)/legacy/zstd_v03.c \
datagen.c fullbench.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fullbench32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c $(ZSTDDIR)/legacy/zstd_v03.c \
datagen.c fullbench.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
fuzzer : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c $(ZSTDDIR)/legacy/zstd_v03.c \
datagen.c xxhash.c fuzzer.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fuzzer32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c $(ZSTDDIR)/legacy/zstd_v03.c \
datagen.c xxhash.c fuzzer.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
zbufftest : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/zstd_buffered.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c $(ZSTDDIR)/legacy/zstd_v03.c \
datagen.c xxhash.c zbufftest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
zbufftest32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/zstd_buffered.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c $(ZSTDDIR)/legacy/zstd_v03.c \
datagen.c xxhash.c zbufftest.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
paramgrill : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \
datagen.c xxhash.c paramgrill.c
$(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c $(ZSTDDIR)/legacy/zstd_v03.c \
datagen.c xxhash.c paramgrill.c
$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
datagen : datagen.c datagencli.c

65
programs/legacy/fileio_legacy.c
View File

@ -222,7 +222,6 @@ unsigned long long FIOv02_decompressFrame(FILE* foutput, FILE* finput)
size_t sizeCheck;
ZSTDv02_Dctx* dctx = ZSTDv02_createDCtx();
/* init */
if (outBuff==NULL) EXM_THROW(41, "Error : not enough memory to decode legacy frame");
@ -246,7 +245,7 @@ unsigned long long FIOv02_decompressFrame(FILE* foutput, FILE* finput)
/* Decode block */
decodedSize = ZSTDv02_decompressContinue(dctx, op, oend-op, inBuff, readSize);
if (ZSTDv01_isError(decodedSize)) EXM_THROW(45, "Decoding error : input corrupted");
if (ZSTDv02_isError(decodedSize)) EXM_THROW(45, "Decoding error : input corrupted");
if (decodedSize) /* not a header */
{
@ -270,6 +269,66 @@ unsigned long long FIOv02_decompressFrame(FILE* foutput, FILE* finput)
}
unsigned long long FIOv03_decompressFrame(FILE* foutput, FILE* finput)
{
size_t outBuffSize = 512 KB;
BYTE* outBuff = (BYTE*)malloc(outBuffSize);
size_t inBuffSize = 128 KB + 8;
BYTE inBuff[128 KB + 8];
BYTE* op = outBuff;
BYTE* const oend = outBuff + outBuffSize;
U64 filesize = 0;
size_t toRead;
size_t sizeCheck;
ZSTDv03_Dctx* dctx = ZSTDv03_createDCtx();
/* init */
if (outBuff==NULL) EXM_THROW(41, "Error : not enough memory to decode legacy frame");
/* restore header, already read from input */
MEM_writeLE32(inBuff, ZSTDv03_magicNumber);
sizeCheck = ZSTDv03_decompressContinue(dctx, NULL, 0, inBuff, sizeof(ZSTDv03_magicNumber)); /* Decode frame header */
if (ZSTDv03_isError(sizeCheck)) EXM_THROW(42, "Error decoding legacy header");
/* Main decompression Loop */
toRead = ZSTDv03_nextSrcSizeToDecompress(dctx);
while (toRead)
{
size_t readSize, decodedSize;
/* Fill input buffer */
if (toRead > inBuffSize)
EXM_THROW(43, "too large block");
readSize = fread(inBuff, 1, toRead, finput);
if (readSize != toRead)
EXM_THROW(44, "Read error");
/* Decode block */
decodedSize = ZSTDv03_decompressContinue(dctx, op, oend-op, inBuff, readSize);
if (ZSTDv03_isError(decodedSize)) EXM_THROW(45, "Decoding error : input corrupted");
if (decodedSize) /* not a header */
{
/* Write block */
sizeCheck = fwrite(op, 1, decodedSize, foutput);
if (sizeCheck != decodedSize) EXM_THROW(46, "Write error : unable to write data block to destination file");
filesize += decodedSize;
op += decodedSize;
if (op==oend) op = outBuff;
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(filesize>>20) );
}
/* prepare for next Block */
toRead = ZSTDv03_nextSrcSizeToDecompress(dctx);
}
/* release resources */
free(outBuff);
free(dctx);
return filesize;
}
unsigned long long FIO_decompressLegacyFrame(FILE* foutput, FILE* finput, U32 magicNumberLE)
{
switch(magicNumberLE)
@ -278,6 +337,8 @@ unsigned long long FIO_decompressLegacyFrame(FILE* foutput, FILE* finput, U32 ma
return FIOv01_decompressFrame(foutput, finput);
case ZSTDv02_magicNumber :
return FIOv02_decompressFrame(foutput, finput);
case ZSTDv03_magicNumber :
return FIOv03_decompressFrame(foutput, finput);
default :
return ERROR(prefix_unknown);
}