update huff0
This commit is contained in:
Yann Collet
parent
0fef5be1cc
commit
b1f3f4b565
2
Makefile
2
Makefile
@ -32,7 +32,7 @@
|
||||
# ################################################################
|
||||
|
||||
# Version number
|
||||
export VERSION := 0.1.2
|
||||
export VERSION := 0.2.0
|
||||
|
||||
PRGDIR = programs
|
||||
ZSTDDIR = lib
|
||||
|
||||
384
lib/bitstream.h
Normal file
384
lib/bitstream.h
Normal file
@ -0,0 +1,384 @@
|
||||
/* ******************************************************************
|
||||
bitstream
|
||||
Part of NewGen Entropy library
|
||||
header file (to include)
|
||||
Copyright (C) 2013-2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
****************************************************************** */
|
||||
#ifndef BITSTREAM_H_MODULE
|
||||
#define BITSTREAM_H_MODULE
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/*
|
||||
* This API consists of small unitary functions, which highly benefit from being inlined.
|
||||
* Since link-time-optimization is not available for all compilers,
|
||||
* these functions are defined into a .h to be included.
|
||||
*/
|
||||
|
||||
/******************************************
|
||||
* Includes
|
||||
******************************************/
|
||||
#include "mem.h" /* unaligned access routines */
|
||||
#include "error.h" /* error codes and messages */
|
||||
|
||||
|
||||
/********************************************
|
||||
* bitStream compression API (write forward)
|
||||
********************************************/
|
||||
/*
|
||||
* bitStream can mix input from multiple sources.
|
||||
* A critical property of these streams is that they encode and decode in **reverse** direction.
|
||||
* So the first bit sequence you add will be the last to be read, like a LIFO stack.
|
||||
*/
|
||||
typedef struct
|
||||
{
|
||||
size_t bitContainer;
|
||||
int bitPos;
|
||||
char* startPtr;
|
||||
char* ptr;
|
||||
char* endPtr;
|
||||
} BIT_CStream_t;
|
||||
|
||||
MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t maxDstSize);
|
||||
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
|
||||
MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC);
|
||||
MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
|
||||
|
||||
/*
|
||||
* Start by initCStream, providing the maximum size of write buffer to write into.
|
||||
* bitStream will never write outside of this buffer.
|
||||
* buffer must be at least as large as a size_t, otherwise function result will be an error code.
|
||||
*
|
||||
* bits are first added to a local register.
|
||||
* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
|
||||
* Writing data into memory is a manual operation, performed by the flushBits function.
|
||||
* Hence keep track how many bits are potentially stored into local register to avoid register overflow.
|
||||
* After a flushBits, a maximum of 7 bits might still be stored into local register.
|
||||
*
|
||||
* Avoid storing elements of more than 25 bits if you want compatibility with 32-bits bitstream readers.
|
||||
*
|
||||
* Last operation is to close the bitStream.
|
||||
* The function returns the final size of CStream in bytes.
|
||||
* If data couldn't fit into dstBuffer, it will return a 0 ( == not storable)
|
||||
*/
|
||||
|
||||
|
||||
/**********************************************
|
||||
* bitStream decompression API (read backward)
|
||||
**********************************************/
|
||||
typedef struct
|
||||
{
|
||||
size_t bitContainer;
|
||||
unsigned bitsConsumed;
|
||||
const char* ptr;
|
||||
const char* start;
|
||||
} BIT_DStream_t;
|
||||
|
||||
typedef enum { BIT_DStream_unfinished = 0,
|
||||
BIT_DStream_endOfBuffer = 1,
|
||||
BIT_DStream_completed = 2,
|
||||
BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */
|
||||
/* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */
|
||||
|
||||
MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
|
||||
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
|
||||
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
|
||||
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
|
||||
|
||||
|
||||
/*
|
||||
* Start by invoking BIT_initDStream().
|
||||
* A chunk of the bitStream is then stored into a local register.
|
||||
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
||||
* You can then retrieve bitFields stored into the local register, **in reverse order**.
|
||||
* Local register is manually filled from memory by the BIT_reloadDStream() method.
|
||||
* A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BIT_DStream_unfinished.
|
||||
* Otherwise, it can be less than that, so proceed accordingly.
|
||||
* Checking if DStream has reached its end can be performed with BIT_endOfDStream()
|
||||
*/
|
||||
|
||||
|
||||
/******************************************
|
||||
* unsafe API
|
||||
******************************************/
|
||||
MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
|
||||
/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */
|
||||
|
||||
MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC);
|
||||
/* unsafe version; does not check buffer overflow */
|
||||
|
||||
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
|
||||
/* faster, but works only if nbBits >= 1 */
|
||||
|
||||
|
||||
|
||||
/****************************************************************
|
||||
* Helper functions
|
||||
****************************************************************/
|
||||
MEM_STATIC unsigned BIT_highbit32 (register U32 val)
|
||||
{
|
||||
# if defined(_MSC_VER) /* Visual */
|
||||
unsigned long r;
|
||||
_BitScanReverse ( &r, val );
|
||||
return (unsigned) r;
|
||||
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
|
||||
return 31 - __builtin_clz (val);
|
||||
# else /* Software version */
|
||||
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
|
||||
U32 v = val;
|
||||
unsigned r;
|
||||
v |= v >> 1;
|
||||
v |= v >> 2;
|
||||
v |= v >> 4;
|
||||
v |= v >> 8;
|
||||
v |= v >> 16;
|
||||
r = DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
|
||||
return r;
|
||||
# endif
|
||||
}
|
||||
|
||||
|
||||
/****************************************************************
|
||||
* bitStream encoding
|
||||
****************************************************************/
|
||||
|
||||
MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* startPtr, size_t maxSize)
|
||||
{
|
||||
bitC->bitContainer = 0;
|
||||
bitC->bitPos = 0;
|
||||
bitC->startPtr = (char*)startPtr;
|
||||
bitC->ptr = bitC->startPtr;
|
||||
bitC->endPtr = bitC->startPtr + maxSize - sizeof(bitC->ptr);
|
||||
if (maxSize < sizeof(bitC->ptr)) return ERROR(dstSize_tooSmall);
|
||||
return 0;
|
||||
}
|
||||
|
||||
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits)
|
||||
{
|
||||
static const unsigned mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF }; /* up to 25 bits */
|
||||
bitC->bitContainer |= (value & mask[nbBits]) << bitC->bitPos;
|
||||
bitC->bitPos += nbBits;
|
||||
}
|
||||
|
||||
/*! BIT_addBitsFast
|
||||
* works only if `value` is _clean_, meaning all high bits above nbBits are 0 */
|
||||
MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits)
|
||||
{
|
||||
bitC->bitContainer |= value << bitC->bitPos;
|
||||
bitC->bitPos += nbBits;
|
||||
}
|
||||
|
||||
/*! BIT_flushBitsFast
|
||||
* unsafe version; does not check buffer overflow */
|
||||
MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
|
||||
{
|
||||
size_t nbBytes = bitC->bitPos >> 3;
|
||||
MEM_writeLEST(bitC->ptr, bitC->bitContainer);
|
||||
bitC->ptr += nbBytes;
|
||||
bitC->bitPos &= 7;
|
||||
bitC->bitContainer >>= nbBytes*8;
|
||||
}
|
||||
|
||||
MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC)
|
||||
{
|
||||
size_t nbBytes = bitC->bitPos >> 3;
|
||||
MEM_writeLEST(bitC->ptr, bitC->bitContainer);
|
||||
bitC->ptr += nbBytes;
|
||||
if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;
|
||||
bitC->bitPos &= 7;
|
||||
bitC->bitContainer >>= nbBytes*8;
|
||||
}
|
||||
|
||||
/*! BIT_closeCStream
|
||||
* @result : size of CStream, in bytes, or 0 if it cannot fit into dstBuffer */
|
||||
MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
|
||||
{
|
||||
char* endPtr;
|
||||
|
||||
BIT_addBitsFast(bitC, 1, 1); /* endMark */
|
||||
BIT_flushBits(bitC);
|
||||
|
||||
if (bitC->ptr >= bitC->endPtr) /* too close to buffer's end */
|
||||
return 0; /* not storable */
|
||||
|
||||
endPtr = bitC->ptr;
|
||||
endPtr += bitC->bitPos > 0; /* remaining bits (incomplete byte) */
|
||||
|
||||
return (endPtr - bitC->startPtr);
|
||||
}
|
||||
|
||||
|
||||
/**********************************************************
|
||||
* bitStream decoding
|
||||
**********************************************************/
|
||||
|
||||
/*!BIT_initDStream
|
||||
* Initialize a BIT_DStream_t.
|
||||
* @bitD : a pointer to an already allocated BIT_DStream_t structure
|
||||
* @srcBuffer must point at the beginning of a bitStream
|
||||
* @srcSize must be the exact size of the bitStream
|
||||
* @result : size of stream (== srcSize) or an errorCode if a problem is detected
|
||||
*/
|
||||
MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize)
|
||||
{
|
||||
if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); }
|
||||
|
||||
if (srcSize >= sizeof(size_t)) /* normal case */
|
||||
{
|
||||
U32 contain32;
|
||||
bitD->start = (const char*)srcBuffer;
|
||||
bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(size_t);
|
||||
bitD->bitContainer = MEM_readLEST(bitD->ptr);
|
||||
contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
|
||||
if (contain32 == 0) return ERROR(GENERIC); /* endMark not present */
|
||||
bitD->bitsConsumed = 8 - BIT_highbit32(contain32);
|
||||
}
|
||||
else
|
||||
{
|
||||
U32 contain32;
|
||||
bitD->start = (const char*)srcBuffer;
|
||||
bitD->ptr = bitD->start;
|
||||
bitD->bitContainer = *(const BYTE*)(bitD->start);
|
||||
switch(srcSize)
|
||||
{
|
||||
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16);
|
||||
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24);
|
||||
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32);
|
||||
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24;
|
||||
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16;
|
||||
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8;
|
||||
default:;
|
||||
}
|
||||
contain32 = ((const BYTE*)srcBuffer)[srcSize-1];
|
||||
if (contain32 == 0) return ERROR(GENERIC); /* endMark not present */
|
||||
bitD->bitsConsumed = 8 - BIT_highbit32(contain32);
|
||||
bitD->bitsConsumed += (U32)(sizeof(size_t) - srcSize)*8;
|
||||
}
|
||||
|
||||
return srcSize;
|
||||
}
|
||||
|
||||
/*!BIT_lookBits
|
||||
* Provides next n bits from local register
|
||||
* local register is not modified (bits are still present for next read/look)
|
||||
* On 32-bits, maxNbBits==25
|
||||
* On 64-bits, maxNbBits==57
|
||||
* @return : value extracted
|
||||
*/
|
||||
MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
||||
return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);
|
||||
}
|
||||
|
||||
/*! BIT_lookBitsFast :
|
||||
* unsafe version; only works only if nbBits >= 1 */
|
||||
MEM_STATIC size_t BIT_lookBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1;
|
||||
return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask);
|
||||
}
|
||||
|
||||
MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
bitD->bitsConsumed += nbBits;
|
||||
}
|
||||
|
||||
/*!BIT_readBits
|
||||
* Read next n bits from local register.
|
||||
* pay attention to not read more than nbBits contained into local register.
|
||||
* @return : extracted value.
|
||||
*/
|
||||
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
size_t value = BIT_lookBits(bitD, nbBits);
|
||||
BIT_skipBits(bitD, nbBits);
|
||||
return value;
|
||||
}
|
||||
|
||||
/*!BIT_readBitsFast :
|
||||
* unsafe version; only works only if nbBits >= 1 */
|
||||
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
|
||||
{
|
||||
size_t value = BIT_lookBitsFast(bitD, nbBits);
|
||||
BIT_skipBits(bitD, nbBits);
|
||||
return value;
|
||||
}
|
||||
|
||||
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
|
||||
{
|
||||
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */
|
||||
return BIT_DStream_overflow;
|
||||
|
||||
if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer))
|
||||
{
|
||||
bitD->ptr -= bitD->bitsConsumed >> 3;
|
||||
bitD->bitsConsumed &= 7;
|
||||
bitD->bitContainer = MEM_readLEST(bitD->ptr);
|
||||
return BIT_DStream_unfinished;
|
||||
}
|
||||
if (bitD->ptr == bitD->start)
|
||||
{
|
||||
if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer;
|
||||
return BIT_DStream_completed;
|
||||
}
|
||||
{
|
||||
U32 nbBytes = bitD->bitsConsumed >> 3;
|
||||
BIT_DStream_status result = BIT_DStream_unfinished;
|
||||
if (bitD->ptr - nbBytes < bitD->start)
|
||||
{
|
||||
nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */
|
||||
result = BIT_DStream_endOfBuffer;
|
||||
}
|
||||
bitD->ptr -= nbBytes;
|
||||
bitD->bitsConsumed -= nbBytes*8;
|
||||
bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
/*! BIT_endOfDStream
|
||||
* @return Tells if DStream has reached its exact end
|
||||
*/
|
||||
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
|
||||
{
|
||||
return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));
|
||||
}
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* BITSTREAM_H_MODULE */
|
||||
91
lib/error.h
Normal file
91
lib/error.h
Normal file
@ -0,0 +1,91 @@
|
||||
/* ******************************************************************
|
||||
Error codes and messages
|
||||
Copyright (C) 2013-2015, Yann Collet
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
****************************************************************** */
|
||||
#ifndef ERROR_H_MODULE
|
||||
#define ERROR_H_MODULE
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/******************************************
|
||||
* Compiler-specific
|
||||
******************************************/
|
||||
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
||||
# define ERR_STATIC static inline
|
||||
#elif defined(_MSC_VER)
|
||||
# define ERR_STATIC static __inline
|
||||
#elif defined(__GNUC__)
|
||||
# define ERR_STATIC static __attribute__((unused))
|
||||
#else
|
||||
# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
|
||||
#endif
|
||||
|
||||
|
||||
/******************************************
|
||||
* Error Management
|
||||
******************************************/
|
||||
#define PREFIX(name) ZSTD_error_##name
|
||||
|
||||
#define ERROR(name) (size_t)-PREFIX(name)
|
||||
|
||||
#define ERROR_LIST(ITEM) \
|
||||
ITEM(PREFIX(No_Error)) ITEM(PREFIX(GENERIC)) \
|
||||
ITEM(PREFIX(dstSize_tooSmall)) ITEM(PREFIX(srcSize_wrong)) \
|
||||
ITEM(PREFIX(prefix_unknown)) ITEM(PREFIX(corruption_detected)) \
|
||||
ITEM(PREFIX(tableLog_tooLarge)) ITEM(PREFIX(maxSymbolValue_tooLarge)) ITEM(PREFIX(maxSymbolValue_tooSmall)) \
|
||||
ITEM(PREFIX(maxCode))
|
||||
|
||||
#define ERROR_GENERATE_ENUM(ENUM) ENUM,
|
||||
typedef enum { ERROR_LIST(ERROR_GENERATE_ENUM) } ERR_codes; /* enum is exposed, to detect & handle specific errors; compare function result to -enum value */
|
||||
|
||||
#define ERROR_CONVERTTOSTRING(STRING) #STRING,
|
||||
#define ERROR_GENERATE_STRING(EXPR) ERROR_CONVERTTOSTRING(EXPR)
|
||||
static const char* ERR_strings[] = { ERROR_LIST(ERROR_GENERATE_STRING) };
|
||||
|
||||
ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
|
||||
|
||||
ERR_STATIC const char* ERR_getErrorName(size_t code)
|
||||
{
|
||||
static const char* codeError = "Unspecified error code";
|
||||
if (ERR_isError(code)) return ERR_strings[-(int)(code)];
|
||||
return codeError;
|
||||
}
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ERROR_H_MODULE */
|
||||
28
lib/fse.h
28
lib/fse.h
@ -73,32 +73,6 @@ FSE_decompress():
|
||||
*/
|
||||
|
||||
|
||||
/******************************************
|
||||
* Huff0 simple functions
|
||||
******************************************/
|
||||
size_t HUF_compress(void* dst, size_t maxDstSize,
|
||||
const void* src, size_t srcSize);
|
||||
size_t HUF_decompress(void* dst, size_t maxDstSize,
|
||||
const void* cSrc, size_t cSrcSize);
|
||||
/*
|
||||
HUF_compress():
|
||||
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
|
||||
'dst' buffer must be already allocated. Compression runs faster is maxDstSize >= HUF_compressBound(srcSize)
|
||||
return : size of compressed data (<= maxDstSize)
|
||||
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
|
||||
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
|
||||
if FSE_isError(return), compression failed (more details using FSE_getErrorName())
|
||||
|
||||
HUF_decompress():
|
||||
Decompress Huff0 data from buffer 'cSrc', of size 'cSrcSize',
|
||||
into already allocated destination buffer 'dst', of size 'maxDstSize'.
|
||||
return : size of regenerated data (<= maxDstSize)
|
||||
or an error code, which can be tested using FSE_isError()
|
||||
|
||||
** Important ** : HUF_decompress() doesn't decompress non-compressible nor RLE data !!!
|
||||
*/
|
||||
|
||||
|
||||
/******************************************
|
||||
* Tool functions
|
||||
******************************************/
|
||||
@ -123,8 +97,6 @@ FSE_compress2():
|
||||
*/
|
||||
size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
|
||||
/******************************************
|
||||
* FSE detailed API
|
||||
|
||||
193
lib/fse_static.h
193
lib/fse_static.h
@ -43,6 +43,7 @@ extern "C" {
|
||||
* FSE API compatible with DLL
|
||||
******************************************/
|
||||
#include "fse.h"
|
||||
#include "bitstream.h"
|
||||
|
||||
|
||||
/******************************************
|
||||
@ -57,30 +58,6 @@ extern "C" {
|
||||
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
|
||||
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
|
||||
|
||||
/* Huff0 buffer bounds */
|
||||
#define HUF_CTABLEBOUND 129
|
||||
#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if pre-filtered with fast heuristic */
|
||||
#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
|
||||
|
||||
/* You can statically allocate Huff0 DTable as a table of unsigned short using below macro */
|
||||
#define HUF_DTABLE_SIZE_U16(maxTableLog) (1 + (1<<maxTableLog))
|
||||
#define HUF_CREATE_STATIC_DTABLE(DTable, maxTableLog) \
|
||||
unsigned short DTable[HUF_DTABLE_SIZE_U16(maxTableLog)] = { maxTableLog }
|
||||
|
||||
|
||||
/******************************************
|
||||
* Error Management
|
||||
******************************************/
|
||||
#define FSE_LIST_ERRORS(ITEM) \
|
||||
ITEM(FSE_OK_NoError) ITEM(FSE_ERROR_GENERIC) \
|
||||
ITEM(FSE_ERROR_tableLog_tooLarge) ITEM(FSE_ERROR_maxSymbolValue_tooLarge) ITEM(FSE_ERROR_maxSymbolValue_tooSmall) \
|
||||
ITEM(FSE_ERROR_dstSize_tooSmall) ITEM(FSE_ERROR_srcSize_wrong)\
|
||||
ITEM(FSE_ERROR_corruptionDetected) \
|
||||
ITEM(FSE_ERROR_maxCode)
|
||||
|
||||
#define FSE_GENERATE_ENUM(ENUM) ENUM,
|
||||
typedef enum { FSE_LIST_ERRORS(FSE_GENERATE_ENUM) } FSE_errorCodes; /* enum is exposed, to detect & handle specific errors; compare function result to -enum value */
|
||||
|
||||
|
||||
/******************************************
|
||||
* FSE advanced API
|
||||
@ -111,16 +88,6 @@ size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
|
||||
For gcc or clang, you'll need to add -flto flag at compilation and linking stages.
|
||||
If none of these solutions is applicable, include "fse.c" directly.
|
||||
*/
|
||||
|
||||
typedef struct
|
||||
{
|
||||
size_t bitContainer;
|
||||
int bitPos;
|
||||
char* startPtr;
|
||||
char* ptr;
|
||||
char* endPtr;
|
||||
} FSE_CStream_t;
|
||||
|
||||
typedef struct
|
||||
{
|
||||
ptrdiff_t value;
|
||||
@ -129,15 +96,11 @@ typedef struct
|
||||
unsigned stateLog;
|
||||
} FSE_CState_t;
|
||||
|
||||
size_t FSE_initCStream(FSE_CStream_t* bitC, void* dstBuffer, size_t maxDstSize);
|
||||
void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct);
|
||||
static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct);
|
||||
|
||||
void FSE_encodeSymbol(FSE_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol);
|
||||
void FSE_addBits(FSE_CStream_t* bitC, size_t value, unsigned nbBits);
|
||||
void FSE_flushBits(FSE_CStream_t* bitC);
|
||||
static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol);
|
||||
|
||||
void FSE_flushCState(FSE_CStream_t* bitC, const FSE_CState_t* CStatePtr);
|
||||
size_t FSE_closeCStream(FSE_CStream_t* bitC);
|
||||
static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr);
|
||||
|
||||
/*
|
||||
These functions are inner components of FSE_compress_usingCTable().
|
||||
@ -149,15 +112,15 @@ So the first symbol you will encode is the last you will decode, like a LIFO sta
|
||||
You will need a few variables to track your CStream. They are :
|
||||
|
||||
FSE_CTable ct; // Provided by FSE_buildCTable()
|
||||
FSE_CStream_t bitStream; // bitStream tracking structure
|
||||
BIT_CStream_t bitStream; // bitStream tracking structure
|
||||
FSE_CState_t state; // State tracking structure (can have several)
|
||||
|
||||
|
||||
The first thing to do is to init bitStream and state.
|
||||
size_t errorCode = FSE_initCStream(&bitStream, dstBuffer, maxDstSize);
|
||||
size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize);
|
||||
FSE_initCState(&state, ct);
|
||||
|
||||
Note that FSE_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
|
||||
Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
|
||||
You can then encode your input data, byte after byte.
|
||||
FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
|
||||
Remember decoding will be done in reverse direction.
|
||||
@ -165,12 +128,12 @@ Remember decoding will be done in reverse direction.
|
||||
|
||||
At any time, you can also add any bit sequence.
|
||||
Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders
|
||||
FSE_addBits(&bitStream, bitField, nbBits);
|
||||
BIT_addBits(&bitStream, bitField, nbBits);
|
||||
|
||||
The above methods don't commit data to memory, they just store it into local register, for speed.
|
||||
Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
|
||||
Writing data to memory is a manual operation, performed by the flushBits function.
|
||||
FSE_flushBits(&bitStream);
|
||||
BIT_flushBits(&bitStream);
|
||||
|
||||
Your last FSE encoding operation shall be to flush your last state value(s).
|
||||
FSE_flushState(&bitStream, &state);
|
||||
@ -179,21 +142,13 @@ Finally, you must close the bitStream.
|
||||
The function returns the size of CStream in bytes.
|
||||
If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
|
||||
If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
|
||||
size_t size = FSE_closeCStream(&bitStream);
|
||||
size_t size = BIT_closeCStream(&bitStream);
|
||||
*/
|
||||
|
||||
|
||||
/******************************************
|
||||
* FSE symbol decompression API
|
||||
******************************************/
|
||||
typedef struct
|
||||
{
|
||||
size_t bitContainer;
|
||||
unsigned bitsConsumed;
|
||||
const char* ptr;
|
||||
const char* start;
|
||||
} FSE_DStream_t;
|
||||
|
||||
typedef struct
|
||||
{
|
||||
size_t state;
|
||||
@ -201,21 +156,11 @@ typedef struct
|
||||
} FSE_DState_t;
|
||||
|
||||
|
||||
size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
|
||||
void FSE_initDState(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD, const FSE_DTable* dt);
|
||||
static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt);
|
||||
|
||||
unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD);
|
||||
size_t FSE_readBits(FSE_DStream_t* bitD, unsigned nbBits);
|
||||
unsigned int FSE_reloadDStream(FSE_DStream_t* bitD);
|
||||
static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
|
||||
|
||||
unsigned FSE_endOfDStream(const FSE_DStream_t* bitD);
|
||||
unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
|
||||
|
||||
typedef enum { FSE_DStream_unfinished = 0,
|
||||
FSE_DStream_endOfBuffer = 1,
|
||||
FSE_DStream_completed = 2,
|
||||
FSE_DStream_tooFar = 3 } FSE_DStream_status; /* result of FSE_reloadDStream() */
|
||||
/* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... ?! */
|
||||
static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
|
||||
|
||||
/*
|
||||
Let's now decompose FSE_decompress_usingDTable() into its unitary components.
|
||||
@ -224,12 +169,12 @@ and also any other bitFields you put in, **in reverse order**.
|
||||
|
||||
You will need a few variables to track your bitStream. They are :
|
||||
|
||||
FSE_DStream_t DStream; // Stream context
|
||||
BIT_DStream_t DStream; // Stream context
|
||||
FSE_DState_t DState; // State context. Multiple ones are possible
|
||||
FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
|
||||
|
||||
The first thing to do is to init the bitStream.
|
||||
errorCode = FSE_initDStream(&DStream, srcBuffer, srcSize);
|
||||
errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
|
||||
|
||||
You should then retrieve your initial state(s)
|
||||
(in reverse flushing order if you have several ones) :
|
||||
@ -242,39 +187,119 @@ Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last
|
||||
|
||||
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
|
||||
Note : maximum allowed nbBits is 25, for 32-bits compatibility
|
||||
size_t bitField = FSE_readBits(&DStream, nbBits);
|
||||
size_t bitField = BIT_readBits(&DStream, nbBits);
|
||||
|
||||
All above operations only read from local register (which size depends on size_t).
|
||||
Refueling the register from memory is manually performed by the reload method.
|
||||
endSignal = FSE_reloadDStream(&DStream);
|
||||
|
||||
FSE_reloadDStream() result tells if there is still some more data to read from DStream.
|
||||
FSE_DStream_unfinished : there is still some data left into the DStream.
|
||||
FSE_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
||||
FSE_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
||||
FSE_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
||||
BIT_reloadDStream() result tells if there is still some more data to read from DStream.
|
||||
BIT_DStream_unfinished : there is still some data left into the DStream.
|
||||
BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
|
||||
BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
|
||||
BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
|
||||
|
||||
When reaching end of buffer (FSE_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
||||
When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
|
||||
to properly detect the exact end of stream.
|
||||
After each decoded symbol, check if DStream is fully consumed using this simple test :
|
||||
FSE_reloadDStream(&DStream) >= FSE_DStream_completed
|
||||
BIT_reloadDStream(&DStream) >= BIT_DStream_completed
|
||||
|
||||
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
|
||||
Checking if DStream has reached its end is performed by :
|
||||
FSE_endOfDStream(&DStream);
|
||||
BIT_endOfDStream(&DStream);
|
||||
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
|
||||
FSE_endOfDState(&DState);
|
||||
*/
|
||||
|
||||
|
||||
/******************************************
|
||||
* FSE unsafe symbol API
|
||||
* FSE unsafe API
|
||||
******************************************/
|
||||
size_t FSE_readBitsFast(FSE_DStream_t* bitD, unsigned nbBits);
|
||||
/* faster, but works only if nbBits >= 1 (otherwise, result will be corrupted) */
|
||||
static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
|
||||
/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
|
||||
|
||||
unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD);
|
||||
/* faster, but works only if allways nbBits >= 1 (otherwise, result will be corrupted) */
|
||||
|
||||
/******************************************
|
||||
* Implementation of inline functions
|
||||
******************************************/
|
||||
typedef struct
|
||||
{
|
||||
int deltaFindState;
|
||||
U32 deltaNbBits;
|
||||
} FSE_symbolCompressionTransform; /* total 8 bytes */
|
||||
|
||||
MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
|
||||
{
|
||||
const U32 tableLog = ( (const U16*) ct) [0];
|
||||
statePtr->value = (ptrdiff_t)1<<tableLog;
|
||||
statePtr->stateTable = ((const U16*) ct) + 2;
|
||||
statePtr->symbolTT = (const void*)((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1));
|
||||
statePtr->stateLog = tableLog;
|
||||
}
|
||||
|
||||
MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol)
|
||||
{
|
||||
const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
|
||||
const U16* const stateTable = (const U16*)(statePtr->stateTable);
|
||||
U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
|
||||
BIT_addBits(bitC, statePtr->value, nbBitsOut);
|
||||
statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
|
||||
}
|
||||
|
||||
MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr)
|
||||
{
|
||||
BIT_addBits(bitC, statePtr->value, statePtr->stateLog);
|
||||
BIT_flushBits(bitC);
|
||||
}
|
||||
|
||||
/* decompression */
|
||||
|
||||
typedef struct {
|
||||
U16 tableLog;
|
||||
U16 fastMode;
|
||||
} FSE_DTableHeader; /* sizeof U32 */
|
||||
|
||||
typedef struct
|
||||
{
|
||||
unsigned short newState;
|
||||
unsigned char symbol;
|
||||
unsigned char nbBits;
|
||||
} FSE_decode_t; /* size == U32 */
|
||||
|
||||
MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt)
|
||||
{
|
||||
const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)dt;
|
||||
DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
|
||||
BIT_reloadDStream(bitD);
|
||||
DStatePtr->table = dt + 1;
|
||||
}
|
||||
|
||||
MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
|
||||
{
|
||||
const FSE_decode_t DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
const U32 nbBits = DInfo.nbBits;
|
||||
BYTE symbol = DInfo.symbol;
|
||||
size_t lowBits = BIT_readBits(bitD, nbBits);
|
||||
|
||||
DStatePtr->state = DInfo.newState + lowBits;
|
||||
return symbol;
|
||||
}
|
||||
|
||||
MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
|
||||
{
|
||||
const FSE_decode_t DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
|
||||
const U32 nbBits = DInfo.nbBits;
|
||||
BYTE symbol = DInfo.symbol;
|
||||
size_t lowBits = BIT_readBitsFast(bitD, nbBits);
|
||||
|
||||
DStatePtr->state = DInfo.newState + lowBits;
|
||||
return symbol;
|
||||
}
|
||||
|
||||
MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
|
||||
{
|
||||
return DStatePtr->state == 0;
|
||||
}
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
|
||||
1705
lib/huff0.c
Normal file
1705
lib/huff0.c
Normal file
File diff suppressed because it is too large
Load Diff
93
lib/huff0.h
Normal file
93
lib/huff0.h
Normal file
@ -0,0 +1,93 @@
|
||||
/* ******************************************************************
|
||||
Huff0 : Huffman coder, part of New Generation Entropy library
|
||||
header file
|
||||
Copyright (C) 2013-2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
****************************************************************** */
|
||||
#pragma once
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/******************************************
|
||||
* Dependency
|
||||
******************************************/
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
|
||||
/******************************************
|
||||
* Huff0 simple functions
|
||||
******************************************/
|
||||
size_t HUF_compress(void* dst, size_t maxDstSize,
|
||||
const void* src, size_t srcSize);
|
||||
size_t HUF_decompress(void* dst, size_t dstSize,
|
||||
const void* cSrc, size_t cSrcSize);
|
||||
/*
|
||||
HUF_compress():
|
||||
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
|
||||
'dst' buffer must be already allocated. Compression runs faster if maxDstSize >= HUF_compressBound(srcSize).
|
||||
Note : srcSize must be <= 128 KB
|
||||
return : size of compressed data (<= maxDstSize)
|
||||
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
|
||||
if return == 1, srcData is a single repeated byte symbol (RLE compression)
|
||||
if HUF_isError(return), compression failed (more details using HUF_getErrorName())
|
||||
|
||||
HUF_decompress():
|
||||
Decompress Huff0 data from buffer 'cSrc', of size 'cSrcSize',
|
||||
into already allocated destination buffer 'dst', of size 'dstSize'.
|
||||
'dstSize' must be the exact size of original (uncompressed) data.
|
||||
Note : in contrast with FSE, HUF_decompress can regenerate RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, because it knows size to regenerate.
|
||||
return : size of regenerated data (== dstSize)
|
||||
or an error code, which can be tested using HUF_isError()
|
||||
*/
|
||||
|
||||
|
||||
/******************************************
|
||||
* Tool functions
|
||||
******************************************/
|
||||
size_t HUF_compressBound(size_t size); /* maximum compressed size */
|
||||
|
||||
/* Error Management */
|
||||
unsigned HUF_isError(size_t code); /* tells if a return value is an error code */
|
||||
const char* HUF_getErrorName(size_t code); /* provides error code string (useful for debugging) */
|
||||
|
||||
|
||||
/******************************************
|
||||
* Advanced functions
|
||||
******************************************/
|
||||
size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
76
lib/huff0_static.h
Normal file
76
lib/huff0_static.h
Normal file
@ -0,0 +1,76 @@
|
||||
/* ******************************************************************
|
||||
Huff0 : Huffman coder, part of New Generation Entropy library
|
||||
header file for static linking (only)
|
||||
Copyright (C) 2013-2015, Yann Collet
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
****************************************************************** */
|
||||
#pragma once
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
/******************************************
|
||||
* Dependency
|
||||
******************************************/
|
||||
#include "huff0.h"
|
||||
|
||||
|
||||
/******************************************
|
||||
* Static allocation macros
|
||||
******************************************/
|
||||
/* Huff0 buffer bounds */
|
||||
#define HUF_CTABLEBOUND 129
|
||||
#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */
|
||||
#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
|
||||
|
||||
/* static allocation of Huff0's DTable */
|
||||
#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<maxTableLog)) /* nb Cells; use unsigned short for X2, unsigned int for X4 */
|
||||
#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
|
||||
unsigned short DTable[HUF_DTABLE_SIZE(maxTableLog)] = { maxTableLog }
|
||||
#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
|
||||
unsigned int DTable[HUF_DTABLE_SIZE(maxTableLog)] = { maxTableLog }
|
||||
#define HUF_CREATE_STATIC_DTABLEX6(DTable, maxTableLog) \
|
||||
unsigned int DTable[HUF_DTABLE_SIZE(maxTableLog) * 3 / 2] = { maxTableLog }
|
||||
|
||||
|
||||
/******************************************
|
||||
* Advanced functions
|
||||
******************************************/
|
||||
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
|
||||
size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */
|
||||
size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder */
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
2171
lib/legacy/zstd_v01.c
Normal file
2171
lib/legacy/zstd_v01.c
Normal file
File diff suppressed because it is too large
Load Diff
100
lib/legacy/zstd_v01.h
Normal file
100
lib/legacy/zstd_v01.h
Normal file
@ -0,0 +1,100 @@
|
||||
/*
|
||||
zstd - standard compression library
|
||||
Header File
|
||||
Copyright (C) 2014-2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd source repository : https://github.com/Cyan4973/zstd
|
||||
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* *************************************
|
||||
* Includes
|
||||
***************************************/
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Simple one-step function
|
||||
***************************************/
|
||||
/**
|
||||
ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format
|
||||
compressedSize : is the exact source size
|
||||
maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated.
|
||||
It must be equal or larger than originalSize, otherwise decompression will fail.
|
||||
return : the number of bytes decompressed into destination buffer (originalSize)
|
||||
or an errorCode if it fails (which can be tested using ZSTDv01_isError())
|
||||
*/
|
||||
size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize,
|
||||
const void* src, size_t compressedSize);
|
||||
|
||||
/**
|
||||
ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error
|
||||
*/
|
||||
unsigned ZSTDv01_isError(size_t code);
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Advanced functions
|
||||
***************************************/
|
||||
typedef struct ZSTDv01_Dctx_s ZSTDv01_Dctx;
|
||||
ZSTDv01_Dctx* ZSTDv01_createDCtx(void);
|
||||
size_t ZSTDv01_freeDCtx(ZSTDv01_Dctx* dctx);
|
||||
|
||||
size_t ZSTDv01_decompressDCtx(void* ctx,
|
||||
void* dst, size_t maxOriginalSize,
|
||||
const void* src, size_t compressedSize);
|
||||
|
||||
/* *************************************
|
||||
* Streaming functions
|
||||
***************************************/
|
||||
size_t ZSTDv01_resetDCtx(ZSTDv01_Dctx* dctx);
|
||||
|
||||
size_t ZSTDv01_nextSrcSizeToDecompress(ZSTDv01_Dctx* dctx);
|
||||
size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
|
||||
/**
|
||||
Use above functions alternatively.
|
||||
ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
|
||||
ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block.
|
||||
Result is the number of bytes regenerated within 'dst'.
|
||||
It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
|
||||
*/
|
||||
|
||||
/* *************************************
|
||||
* Prefix - version detection
|
||||
***************************************/
|
||||
#define ZSTDv01_magicNumber 0xFD2FB51E /* Big Endian version */
|
||||
#define ZSTDv01_magicNumberLE 0x1EB52FFD /* Little Endian version */
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
289
lib/mem.h
Normal file
289
lib/mem.h
Normal file
@ -0,0 +1,289 @@
|
||||
/* ******************************************************************
|
||||
mem.h
|
||||
low-level memory access routines
|
||||
Copyright (C) 2013-2015, Yann Collet.
|
||||
|
||||
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following disclaimer
|
||||
in the documentation and/or other materials provided with the
|
||||
distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
You can contact the author at :
|
||||
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
****************************************************************** */
|
||||
#ifndef MEM_H_MODULE
|
||||
#define MEM_H_MODULE
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/******************************************
|
||||
* Includes
|
||||
******************************************/
|
||||
#include <stddef.h> /* size_t, ptrdiff_t */
|
||||
#include <string.h> /* memcpy */
|
||||
|
||||
|
||||
/******************************************
|
||||
* Compiler-specific
|
||||
******************************************/
|
||||
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
||||
# define MEM_STATIC static inline
|
||||
#elif defined(_MSC_VER)
|
||||
# define MEM_STATIC static __inline
|
||||
#elif defined(__GNUC__)
|
||||
# define MEM_STATIC static __attribute__((unused))
|
||||
#else
|
||||
# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
|
||||
#endif
|
||||
|
||||
|
||||
/****************************************************************
|
||||
* Basic Types
|
||||
*****************************************************************/
|
||||
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
||||
# include <stdint.h>
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef int16_t S16;
|
||||
typedef uint32_t U32;
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
typedef int64_t S64;
|
||||
#else
|
||||
typedef unsigned char BYTE;
|
||||
typedef unsigned short U16;
|
||||
typedef signed short S16;
|
||||
typedef unsigned int U32;
|
||||
typedef signed int S32;
|
||||
typedef unsigned long long U64;
|
||||
typedef signed long long S64;
|
||||
#endif
|
||||
|
||||
|
||||
/****************************************************************
|
||||
* Memory I/O
|
||||
*****************************************************************/
|
||||
/* MEM_FORCE_MEMORY_ACCESS
|
||||
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
|
||||
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
|
||||
* The below switch allow to select different access method for improved performance.
|
||||
* Method 0 (default) : use `memcpy()`. Safe and portable.
|
||||
* Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
|
||||
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
|
||||
* Method 2 : direct access. This method is portable but violate C standard.
|
||||
* It can generate buggy code on targets generating assembly depending on alignment.
|
||||
* But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
|
||||
* See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
|
||||
* Prefer these methods in priority order (0 > 1 > 2)
|
||||
*/
|
||||
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
|
||||
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
|
||||
# define MEM_FORCE_MEMORY_ACCESS 2
|
||||
# elif defined(__INTEL_COMPILER) || \
|
||||
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
|
||||
# define MEM_FORCE_MEMORY_ACCESS 1
|
||||
# endif
|
||||
#endif
|
||||
|
||||
MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; }
|
||||
MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; }
|
||||
|
||||
MEM_STATIC unsigned MEM_isLittleEndian(void)
|
||||
{
|
||||
const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
|
||||
return one.c[0];
|
||||
}
|
||||
|
||||
#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
|
||||
|
||||
/* violates C standard on structure alignment.
|
||||
Only use if no other choice to achieve best performance on target platform */
|
||||
MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
|
||||
MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
|
||||
MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
|
||||
|
||||
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
|
||||
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
|
||||
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
|
||||
|
||||
#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
|
||||
|
||||
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
|
||||
/* currently only defined for gcc and icc */
|
||||
typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign;
|
||||
|
||||
MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
|
||||
MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
|
||||
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
|
||||
|
||||
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
|
||||
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
|
||||
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
|
||||
|
||||
#else
|
||||
|
||||
/* default method, safe and standard.
|
||||
can sometimes prove slower */
|
||||
|
||||
MEM_STATIC U16 MEM_read16(const void* memPtr)
|
||||
{
|
||||
U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
||||
}
|
||||
|
||||
MEM_STATIC U32 MEM_read32(const void* memPtr)
|
||||
{
|
||||
U32 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
||||
}
|
||||
|
||||
MEM_STATIC U64 MEM_read64(const void* memPtr)
|
||||
{
|
||||
U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_write16(void* memPtr, U16 value)
|
||||
{
|
||||
memcpy(memPtr, &value, sizeof(value));
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_write32(void* memPtr, U32 value)
|
||||
{
|
||||
memcpy(memPtr, &value, sizeof(value));
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_write64(void* memPtr, U64 value)
|
||||
{
|
||||
memcpy(memPtr, &value, sizeof(value));
|
||||
}
|
||||
|
||||
#endif // MEM_FORCE_MEMORY_ACCESS
|
||||
|
||||
|
||||
MEM_STATIC U16 MEM_readLE16(const void* memPtr)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
return MEM_read16(memPtr);
|
||||
else
|
||||
{
|
||||
const BYTE* p = (const BYTE*)memPtr;
|
||||
return (U16)(p[0] + (p[1]<<8));
|
||||
}
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
{
|
||||
MEM_write16(memPtr, val);
|
||||
}
|
||||
else
|
||||
{
|
||||
BYTE* p = (BYTE*)memPtr;
|
||||
p[0] = (BYTE)val;
|
||||
p[1] = (BYTE)(val>>8);
|
||||
}
|
||||
}
|
||||
|
||||
MEM_STATIC U32 MEM_readLE32(const void* memPtr)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
return MEM_read32(memPtr);
|
||||
else
|
||||
{
|
||||
const BYTE* p = (const BYTE*)memPtr;
|
||||
return (U32)((U32)p[0] + ((U32)p[1]<<8) + ((U32)p[2]<<16) + ((U32)p[3]<<24));
|
||||
}
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
{
|
||||
MEM_write32(memPtr, val32);
|
||||
}
|
||||
else
|
||||
{
|
||||
BYTE* p = (BYTE*)memPtr;
|
||||
p[0] = (BYTE)val32;
|
||||
p[1] = (BYTE)(val32>>8);
|
||||
p[2] = (BYTE)(val32>>16);
|
||||
p[3] = (BYTE)(val32>>24);
|
||||
}
|
||||
}
|
||||
|
||||
MEM_STATIC U64 MEM_readLE64(const void* memPtr)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
return MEM_read64(memPtr);
|
||||
else
|
||||
{
|
||||
const BYTE* p = (const BYTE*)memPtr;
|
||||
return (U64)((U64)p[0] + ((U64)p[1]<<8) + ((U64)p[2]<<16) + ((U64)p[3]<<24)
|
||||
+ ((U64)p[4]<<32) + ((U64)p[5]<<40) + ((U64)p[6]<<48) + ((U64)p[7]<<56));
|
||||
}
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
|
||||
{
|
||||
if (MEM_isLittleEndian())
|
||||
{
|
||||
MEM_write64(memPtr, val64);
|
||||
}
|
||||
else
|
||||
{
|
||||
BYTE* p = (BYTE*)memPtr;
|
||||
p[0] = (BYTE)val64;
|
||||
p[1] = (BYTE)(val64>>8);
|
||||
p[2] = (BYTE)(val64>>16);
|
||||
p[3] = (BYTE)(val64>>24);
|
||||
p[4] = (BYTE)(val64>>32);
|
||||
p[5] = (BYTE)(val64>>40);
|
||||
p[6] = (BYTE)(val64>>48);
|
||||
p[7] = (BYTE)(val64>>56);
|
||||
}
|
||||
}
|
||||
|
||||
MEM_STATIC size_t MEM_readLEST(const void* memPtr)
|
||||
{
|
||||
if (MEM_32bits())
|
||||
return (size_t)MEM_readLE32(memPtr);
|
||||
else
|
||||
return (size_t)MEM_readLE64(memPtr);
|
||||
}
|
||||
|
||||
MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
|
||||
{
|
||||
if (MEM_32bits())
|
||||
MEM_writeLE32(memPtr, (U32)val);
|
||||
else
|
||||
MEM_writeLE64(memPtr, (U64)val);
|
||||
}
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* MEM_H_MODULE */
|
||||
|
||||
414
lib/zstd.c
414
lib/zstd.c
@ -32,51 +32,39 @@
|
||||
|
||||
/****************************************************************
|
||||
* Tuning parameters
|
||||
*****************************************************************/
|
||||
/* MEMORY_USAGE :
|
||||
****************************************************************/
|
||||
/**MEMORY_USAGE :
|
||||
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
|
||||
* Increasing memory usage improves compression ratio
|
||||
* Reduced memory usage can improve speed, due to cache effect */
|
||||
#define ZSTD_MEMORY_USAGE 17
|
||||
|
||||
|
||||
/**************************************
|
||||
CPU Feature Detection
|
||||
**************************************/
|
||||
/*
|
||||
* Automated efficient unaligned memory access detection
|
||||
* Based on known hardware architectures
|
||||
* This list will be updated thanks to feedbacks
|
||||
*/
|
||||
#if defined(CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS) \
|
||||
|| defined(__ARM_FEATURE_UNALIGNED) \
|
||||
|| defined(__i386__) || defined(__x86_64__) \
|
||||
|| defined(_M_IX86) || defined(_M_X64) \
|
||||
|| defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_8__) \
|
||||
|| (defined(_M_ARM) && (_M_ARM >= 7))
|
||||
# define ZSTD_UNALIGNED_ACCESS 1
|
||||
#else
|
||||
# define ZSTD_UNALIGNED_ACCESS 0
|
||||
#endif
|
||||
#ifndef ZSTD_LEGACY_SUPPORT
|
||||
/**LEGACY_SUPPORT :
|
||||
* decompressor can decode older formats (starting from Zstd 0.1+) */
|
||||
# define ZSTD_LEGACY_SUPPORT 1
|
||||
#endif // ZSTD_LEGACY_SUPPORT
|
||||
|
||||
|
||||
/********************************************************
|
||||
* Includes
|
||||
*********************************************************/
|
||||
********************************************************/
|
||||
#include <stdlib.h> /* calloc */
|
||||
#include <string.h> /* memcpy, memmove */
|
||||
#include <stdio.h> /* debug : printf */
|
||||
#include "mem.h" /* low level memory routines */
|
||||
#include "zstd_static.h"
|
||||
#if defined(__clang__) || defined(__GNUC__)
|
||||
# include "fse.c" /* due to GCC/Clang inlining limitations, including *.c runs noticeably faster */
|
||||
#else
|
||||
# include "fse_static.h"
|
||||
#endif
|
||||
#include "fse_static.h"
|
||||
#include "huff0.h"
|
||||
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
# include "zstd_v01.h"
|
||||
#endif // defined
|
||||
|
||||
|
||||
/********************************************************
|
||||
* Compiler specifics
|
||||
*********************************************************/
|
||||
********************************************************/
|
||||
#ifdef __AVX2__
|
||||
# include <immintrin.h> /* AVX2 intrinsics */
|
||||
#endif
|
||||
@ -96,36 +84,9 @@
|
||||
#endif
|
||||
|
||||
|
||||
#ifndef MEM_ACCESS_MODULE
|
||||
#define MEM_ACCESS_MODULE
|
||||
/********************************************************
|
||||
* Basic Types
|
||||
*********************************************************/
|
||||
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
|
||||
# include <stdint.h>
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef int16_t S16;
|
||||
typedef uint32_t U32;
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
#else
|
||||
typedef unsigned char BYTE;
|
||||
typedef unsigned short U16;
|
||||
typedef signed short S16;
|
||||
typedef unsigned int U32;
|
||||
typedef signed int S32;
|
||||
typedef unsigned long long U64;
|
||||
#endif
|
||||
|
||||
#endif /* MEM_ACCESS_MODULE */
|
||||
|
||||
|
||||
/********************************************************
|
||||
* Constants
|
||||
*********************************************************/
|
||||
static const U32 ZSTD_magicNumber = 0xFD2FB51E; /* 3rd version : seqNb header */
|
||||
|
||||
#define HASH_LOG (ZSTD_MEMORY_USAGE - 2)
|
||||
#define HASH_TABLESIZE (1 << HASH_LOG)
|
||||
#define HASH_MASK (HASH_TABLESIZE - 1)
|
||||
@ -171,21 +132,6 @@ static const size_t ZSTD_frameHeaderSize = 4;
|
||||
/********************************************************
|
||||
* Memory operations
|
||||
*********************************************************/
|
||||
static unsigned ZSTD_32bits(void) { return sizeof(void*)==4; }
|
||||
static unsigned ZSTD_64bits(void) { return sizeof(void*)==8; }
|
||||
|
||||
static unsigned ZSTD_isLittleEndian(void)
|
||||
{
|
||||
const union { U32 i; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
|
||||
return one.c[0];
|
||||
}
|
||||
|
||||
static U16 ZSTD_read16(const void* p) { U16 r; memcpy(&r, p, sizeof(r)); return r; }
|
||||
|
||||
static U32 ZSTD_read32(const void* p) { U32 r; memcpy(&r, p, sizeof(r)); return r; }
|
||||
|
||||
static U64 ZSTD_read64(const void* p) { U64 r; memcpy(&r, p, sizeof(r)); return r; }
|
||||
|
||||
static size_t ZSTD_read_ARCH(const void* p) { size_t r; memcpy(&r, p, sizeof(r)); return r; }
|
||||
|
||||
static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
|
||||
@ -202,54 +148,6 @@ static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
|
||||
while (op < oend) COPY8(op, ip);
|
||||
}
|
||||
|
||||
static U16 ZSTD_readLE16(const void* memPtr)
|
||||
{
|
||||
if (ZSTD_isLittleEndian()) return ZSTD_read16(memPtr);
|
||||
else
|
||||
{
|
||||
const BYTE* p = (const BYTE*)memPtr;
|
||||
return (U16)((U16)p[0] + ((U16)p[1]<<8));
|
||||
}
|
||||
}
|
||||
|
||||
static void ZSTD_writeLE16(void* memPtr, U16 val)
|
||||
{
|
||||
if (ZSTD_isLittleEndian()) memcpy(memPtr, &val, sizeof(val));
|
||||
else
|
||||
{
|
||||
BYTE* p = (BYTE*)memPtr;
|
||||
p[0] = (BYTE)val;
|
||||
p[1] = (BYTE)(val>>8);
|
||||
}
|
||||
}
|
||||
|
||||
static U32 ZSTD_readLE32(const void* memPtr)
|
||||
{
|
||||
if (ZSTD_isLittleEndian())
|
||||
return ZSTD_read32(memPtr);
|
||||
else
|
||||
{
|
||||
const BYTE* p = (const BYTE*)memPtr;
|
||||
return (U32)((U32)p[0] + ((U32)p[1]<<8) + ((U32)p[2]<<16) + ((U32)p[3]<<24));
|
||||
}
|
||||
}
|
||||
|
||||
static void ZSTD_writeLE32(void* memPtr, U32 val32)
|
||||
{
|
||||
if (ZSTD_isLittleEndian())
|
||||
{
|
||||
memcpy(memPtr, &val32, 4);
|
||||
}
|
||||
else
|
||||
{
|
||||
BYTE* p = (BYTE*)memPtr;
|
||||
p[0] = (BYTE)val32;
|
||||
p[1] = (BYTE)(val32>>8);
|
||||
p[2] = (BYTE)(val32>>16);
|
||||
p[3] = (BYTE)(val32>>24);
|
||||
}
|
||||
}
|
||||
|
||||
static U32 ZSTD_readBE32(const void* memPtr)
|
||||
{
|
||||
const BYTE* p = (const BYTE*)memPtr;
|
||||
@ -348,22 +246,13 @@ size_t ZSTD_freeCCtx(ZSTD_Cctx* ctx)
|
||||
/**************************************
|
||||
* Error Management
|
||||
**************************************/
|
||||
/* tells if a return value is an error code */
|
||||
unsigned ZSTD_isError(size_t code)
|
||||
{
|
||||
return (code > (size_t)(-ZSTD_ERROR_maxCode));
|
||||
}
|
||||
/*! ZSTD_isError
|
||||
* tells if a return value is an error code */
|
||||
unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
|
||||
|
||||
#define ZSTD_GENERATE_STRING(STRING) #STRING,
|
||||
static const char* ZSTD_errorStrings[] = { ZSTD_LIST_ERRORS(ZSTD_GENERATE_STRING) };
|
||||
|
||||
/* provides error code string (useful for debugging) */
|
||||
const char* ZSTD_getErrorName(size_t code)
|
||||
{
|
||||
static const char* codeError = "Unspecified error code";
|
||||
if (ZSTD_isError(code)) return ZSTD_errorStrings[-(int)(code)];
|
||||
return codeError;
|
||||
}
|
||||
/*! ZSTD_getErrorName
|
||||
* provides error code string (useful for debugging) */
|
||||
const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); }
|
||||
|
||||
|
||||
/**************************************
|
||||
@ -395,9 +284,9 @@ static unsigned ZSTD_highbit(U32 val)
|
||||
|
||||
static unsigned ZSTD_NbCommonBytes (register size_t val)
|
||||
{
|
||||
if (ZSTD_isLittleEndian())
|
||||
if (MEM_isLittleEndian())
|
||||
{
|
||||
if (ZSTD_64bits())
|
||||
if (MEM_64bits())
|
||||
{
|
||||
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
|
||||
unsigned long r = 0;
|
||||
@ -426,7 +315,7 @@ static unsigned ZSTD_NbCommonBytes (register size_t val)
|
||||
}
|
||||
else /* Big Endian CPU */
|
||||
{
|
||||
if (ZSTD_64bits())
|
||||
if (MEM_32bits())
|
||||
{
|
||||
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
|
||||
unsigned long r = 0;
|
||||
@ -473,8 +362,8 @@ static unsigned ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInL
|
||||
return (unsigned)(pIn - pStart);
|
||||
}
|
||||
|
||||
if (ZSTD_64bits()) if ((pIn<(pInLimit-3)) && (ZSTD_read32(pMatch) == ZSTD_read32(pIn))) { pIn+=4; pMatch+=4; }
|
||||
if ((pIn<(pInLimit-1)) && (ZSTD_read16(pMatch) == ZSTD_read16(pIn))) { pIn+=2; pMatch+=2; }
|
||||
if (MEM_32bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
|
||||
if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
|
||||
if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
|
||||
return (unsigned)(pIn - pStart);
|
||||
}
|
||||
@ -512,7 +401,7 @@ static size_t ZSTD_noCompressBlock (void* dst, size_t maxDstSize, const void* sr
|
||||
{
|
||||
BYTE* const ostart = (BYTE* const)dst;
|
||||
|
||||
if (srcSize + ZSTD_blockHeaderSize > maxDstSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
|
||||
if (srcSize + ZSTD_blockHeaderSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
||||
memcpy(ostart + ZSTD_blockHeaderSize, src, srcSize);
|
||||
|
||||
/* Build header */
|
||||
@ -525,10 +414,7 @@ static size_t ZSTD_noCompressBlock (void* dst, size_t maxDstSize, const void* sr
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_minGain(size_t srcSize)
|
||||
{
|
||||
return (srcSize >> 6) + 1;
|
||||
}
|
||||
size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 1; }
|
||||
|
||||
|
||||
static size_t ZSTD_compressLiterals (void* dst, size_t dstSize,
|
||||
@ -539,7 +425,7 @@ static size_t ZSTD_compressLiterals (void* dst, size_t dstSize,
|
||||
size_t hsize;
|
||||
static const size_t LHSIZE = 5;
|
||||
|
||||
if (dstSize < LHSIZE+1) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* not enough space for compression */
|
||||
if (dstSize < LHSIZE+1) return ERROR(dstSize_tooSmall); /* not enough space for compression */
|
||||
|
||||
hsize = HUF_compress(ostart+LHSIZE, dstSize-LHSIZE, src, srcSize);
|
||||
if (hsize<2) return hsize; /* special cases */
|
||||
@ -615,8 +501,8 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
|
||||
|
||||
/* Sequences Header */
|
||||
if ((oend-op) < 2+3+6) /* nbSeq + dumpsLength + 3*rleCTable*/
|
||||
return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
|
||||
ZSTD_writeLE16(op, (U16)nbSeq); op+=2;
|
||||
return ERROR(dstSize_tooSmall);
|
||||
MEM_writeLE16(op, (U16)nbSeq); op+=2;
|
||||
seqHead = op;
|
||||
|
||||
/* dumps : contains too large lengths */
|
||||
@ -635,7 +521,7 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
|
||||
op[2] = (BYTE)(dumpsLength);
|
||||
op += 3;
|
||||
}
|
||||
if ((size_t)(oend-op) < dumpsLength+6) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
|
||||
if ((size_t)(oend-op) < dumpsLength+6) return ERROR(dstSize_tooSmall);
|
||||
memcpy(op, seqStorePtr->dumpsStart, dumpsLength);
|
||||
op += dumpsLength;
|
||||
}
|
||||
@ -660,7 +546,7 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
|
||||
tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max);
|
||||
FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
|
||||
NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
|
||||
if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
|
||||
if (FSE_isError(NCountSize)) return ERROR(GENERIC);
|
||||
op += NCountSize;
|
||||
FSE_buildCTable(CTable_LitLength, norm, max, tableLog);
|
||||
LLtype = bt_compressed;
|
||||
@ -695,7 +581,7 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
|
||||
tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max);
|
||||
FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
|
||||
NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
|
||||
if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
|
||||
if (FSE_isError(NCountSize)) return ERROR(GENERIC);
|
||||
op += NCountSize;
|
||||
FSE_buildCTable(CTable_OffsetBits, norm, max, tableLog);
|
||||
Offtype = bt_compressed;
|
||||
@ -721,7 +607,7 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
|
||||
tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max);
|
||||
FSE_normalizeCount(norm, tableLog, count, nbSeq, max);
|
||||
NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */
|
||||
if (FSE_isError(NCountSize)) return (size_t)-ZSTD_ERROR_GENERIC;
|
||||
if (FSE_isError(NCountSize)) return ERROR(GENERIC);
|
||||
op += NCountSize;
|
||||
FSE_buildCTable(CTable_MatchLength, norm, max, tableLog);
|
||||
MLtype = bt_compressed;
|
||||
@ -732,14 +618,14 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
|
||||
/* Encoding Sequences */
|
||||
{
|
||||
size_t streamSize, errorCode;
|
||||
FSE_CStream_t blockStream;
|
||||
BIT_CStream_t blockStream;
|
||||
FSE_CState_t stateMatchLength;
|
||||
FSE_CState_t stateOffsetBits;
|
||||
FSE_CState_t stateLitLength;
|
||||
int i;
|
||||
|
||||
errorCode = FSE_initCStream(&blockStream, op, oend-op);
|
||||
if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* not enough space remaining */
|
||||
errorCode = BIT_initCStream(&blockStream, op, oend-op);
|
||||
if (ERR_isError(errorCode)) return ERROR(dstSize_tooSmall); /* not enough space remaining */
|
||||
FSE_initCState(&stateMatchLength, CTable_MatchLength);
|
||||
FSE_initCState(&stateOffsetBits, CTable_OffsetBits);
|
||||
FSE_initCState(&stateLitLength, CTable_LitLength);
|
||||
@ -752,20 +638,20 @@ static size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
|
||||
U32 nbBits = (offCode-1) * (!!offCode);
|
||||
BYTE litLength = llTable[i]; /* (7)*/ /* (7)*/
|
||||
FSE_encodeSymbol(&blockStream, &stateMatchLength, matchLength); /* 17 */ /* 17 */
|
||||
if (ZSTD_32bits()) FSE_flushBits(&blockStream); /* 7 */
|
||||
FSE_addBits(&blockStream, offset, nbBits); /* 32 */ /* 42 */
|
||||
if (ZSTD_32bits()) FSE_flushBits(&blockStream); /* 7 */
|
||||
if (MEM_32bits()) BIT_flushBits(&blockStream); /* 7 */
|
||||
BIT_addBits(&blockStream, offset, nbBits); /* 32 */ /* 42 */
|
||||
if (MEM_32bits()) BIT_flushBits(&blockStream); /* 7 */
|
||||
FSE_encodeSymbol(&blockStream, &stateOffsetBits, offCode); /* 16 */ /* 51 */
|
||||
FSE_encodeSymbol(&blockStream, &stateLitLength, litLength); /* 26 */ /* 61 */
|
||||
FSE_flushBits(&blockStream); /* 7 */ /* 7 */
|
||||
BIT_flushBits(&blockStream); /* 7 */ /* 7 */
|
||||
}
|
||||
|
||||
FSE_flushCState(&blockStream, &stateMatchLength);
|
||||
FSE_flushCState(&blockStream, &stateOffsetBits);
|
||||
FSE_flushCState(&blockStream, &stateLitLength);
|
||||
|
||||
streamSize = FSE_closeCStream(&blockStream);
|
||||
if (streamSize==0) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* not enough space */
|
||||
streamSize = BIT_closeCStream(&blockStream);
|
||||
if (streamSize==0) return ERROR(dstSize_tooSmall); /* not enough space */
|
||||
op += streamSize;
|
||||
}
|
||||
|
||||
@ -794,7 +680,7 @@ static void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE*
|
||||
else
|
||||
{
|
||||
*(seqStorePtr->dumps++) = 255;
|
||||
ZSTD_writeLE32(seqStorePtr->dumps, (U32)litLength); seqStorePtr->dumps += 3;
|
||||
MEM_writeLE32(seqStorePtr->dumps, (U32)litLength); seqStorePtr->dumps += 3;
|
||||
}
|
||||
}
|
||||
else *(seqStorePtr->litLength++) = (BYTE)litLength;
|
||||
@ -811,7 +697,7 @@ static void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE*
|
||||
else
|
||||
{
|
||||
*(seqStorePtr->dumps++) = 255;
|
||||
ZSTD_writeLE32(seqStorePtr->dumps, (U32)matchLength); seqStorePtr->dumps+=3;
|
||||
MEM_writeLE32(seqStorePtr->dumps, (U32)matchLength); seqStorePtr->dumps+=3;
|
||||
}
|
||||
}
|
||||
else *(seqStorePtr->matchLength++) = (BYTE)matchLength;
|
||||
@ -830,7 +716,7 @@ static const U64 prime7bytes = 58295818150454627ULL;
|
||||
//static U32 ZSTD_hashPtr(const void* p) { return ( ((*(U64*)p & 0xFFFFFFFFFFFFFF) * prime7bytes) >> (64-HASH_LOG)); }
|
||||
|
||||
//static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime8bytes) >> (64-HASH_LOG)); }
|
||||
static U32 ZSTD_hashPtr(const void* p) { return ( (ZSTD_read64(p) * prime7bytes) >> (56-HASH_LOG)) & HASH_MASK; }
|
||||
static U32 ZSTD_hashPtr(const void* p) { return ( (MEM_read64(p) * prime7bytes) >> (56-HASH_LOG)) & HASH_MASK; }
|
||||
//static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime6bytes) >> (48-HASH_LOG)) & HASH_MASK; }
|
||||
//static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime5bytes) >> (40-HASH_LOG)) & HASH_MASK; }
|
||||
//static U32 ZSTD_hashPtr(const void* p) { return ( (*(U32*)p * KNUTH) >> (32-HASH_LOG)); }
|
||||
@ -848,7 +734,7 @@ static const BYTE* ZSTD_updateMatch(U32* table, const BYTE* p, const BYTE* start
|
||||
|
||||
static int ZSTD_checkMatch(const BYTE* match, const BYTE* ip)
|
||||
{
|
||||
return ZSTD_read32(match) == ZSTD_read32(ip);
|
||||
return MEM_read32(match) == MEM_read32(ip);
|
||||
}
|
||||
|
||||
|
||||
@ -916,7 +802,7 @@ static size_t ZSTD_compressBlock(void* cctx, void* dst, size_t maxDstSize, const
|
||||
size_t ZSTD_compressBegin(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize)
|
||||
{
|
||||
/* Sanity check */
|
||||
if (maxDstSize < ZSTD_frameHeaderSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
|
||||
if (maxDstSize < ZSTD_frameHeaderSize) return ERROR(dstSize_tooSmall);
|
||||
|
||||
/* Init */
|
||||
ZSTD_resetCCtx(ctx);
|
||||
@ -1024,7 +910,7 @@ size_t ZSTD_compressContinue(ZSTD_Cctx* cctx, void* dst, size_t maxDstSize, con
|
||||
if (blockSize > srcSize) blockSize = srcSize;
|
||||
|
||||
if (maxDstSize < 2*ZSTD_blockHeaderSize+1) /* one RLE block + endMark */
|
||||
return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
|
||||
return ERROR(dstSize_tooSmall);
|
||||
|
||||
/* update hash table */
|
||||
if (g_maxDistance <= BLOCKSIZE) /* static test ; yes == blocks are independent */
|
||||
@ -1071,7 +957,7 @@ size_t ZSTD_compressEnd(ZSTD_Cctx* ctx, void* dst, size_t maxDstSize)
|
||||
|
||||
/* Sanity check */
|
||||
(void)ctx;
|
||||
if (maxDstSize < ZSTD_blockHeaderSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
|
||||
if (maxDstSize < ZSTD_blockHeaderSize) return ERROR(dstSize_tooSmall);
|
||||
|
||||
/* End of frame */
|
||||
op[0] = (BYTE)(bt_end << 6);
|
||||
@ -1120,7 +1006,7 @@ size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSi
|
||||
size_t r;
|
||||
|
||||
ctx = ZSTD_createCCtx();
|
||||
if (ctx==NULL) return (size_t)-ZSTD_ERROR_GENERIC;
|
||||
if (ctx==NULL) return ERROR(GENERIC);
|
||||
r = ZSTD_compressCCtx(ctx, dst, maxDstSize, src, srcSize);
|
||||
ZSTD_freeCCtx(ctx);
|
||||
return r;
|
||||
@ -1130,6 +1016,22 @@ size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSi
|
||||
/**************************************************************
|
||||
* Decompression code
|
||||
**************************************************************/
|
||||
typedef struct ZSTD_Dctx_s
|
||||
{
|
||||
U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
|
||||
U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
|
||||
U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
|
||||
BYTE litBuffer[BLOCKSIZE];
|
||||
const BYTE* litPtr;
|
||||
size_t litBufSize;
|
||||
size_t litSize;
|
||||
void* previousDstEnd;
|
||||
void* base;
|
||||
size_t expected;
|
||||
blockType_t bType;
|
||||
U32 phase;
|
||||
} ZSTD_Dctx;
|
||||
|
||||
|
||||
size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr)
|
||||
{
|
||||
@ -1137,7 +1039,7 @@ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bp
|
||||
BYTE headerFlags;
|
||||
U32 cSize;
|
||||
|
||||
if (srcSize < 3) return (size_t)-ZSTD_ERROR_SrcSize;
|
||||
if (srcSize < 3) return ERROR(srcSize_wrong);
|
||||
|
||||
headerFlags = *in;
|
||||
cSize = in[2] + (in[1]<<8) + ((in[0] & 7)<<16);
|
||||
@ -1153,7 +1055,7 @@ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bp
|
||||
|
||||
static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
if (srcSize > maxDstSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
|
||||
if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
||||
memcpy(dst, src, srcSize);
|
||||
return srcSize;
|
||||
}
|
||||
@ -1164,70 +1066,69 @@ static size_t ZSTD_decompressLiterals(void* ctx,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
BYTE* op = (BYTE*)dst;
|
||||
BYTE* const oend = op + maxDstSize;
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
size_t errorCode;
|
||||
size_t litSize;
|
||||
|
||||
/* check : minimum 2, for litSize, +1, for content */
|
||||
if (srcSize <= 3) return (size_t)-ZSTD_ERROR_corruption;
|
||||
if (srcSize <= 3) return ERROR(corruption_detected);
|
||||
|
||||
litSize = ip[1] + (ip[0]<<8);
|
||||
litSize += ((ip[-3] >> 3) & 7) << 16; // mmmmh....
|
||||
op = oend - litSize;
|
||||
|
||||
(void)ctx;
|
||||
if (litSize > maxDstSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
|
||||
if (litSize > maxDstSize) return ERROR(dstSize_tooSmall);
|
||||
errorCode = HUF_decompress(op, litSize, ip+2, srcSize-2);
|
||||
if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_GENERIC;
|
||||
if (HUF_isError(errorCode)) return ERROR(GENERIC);
|
||||
return litSize;
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_decodeLiteralsBlock(void* ctx,
|
||||
void* dst, size_t maxDstSize,
|
||||
const BYTE** litStart, size_t* litSize,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
ZSTD_Dctx* dctx = (ZSTD_Dctx*)ctx;
|
||||
const BYTE* const istart = (const BYTE* const)src;
|
||||
const BYTE* ip = istart;
|
||||
BYTE* const ostart = (BYTE* const)dst;
|
||||
BYTE* const oend = ostart + maxDstSize;
|
||||
blockProperties_t litbp;
|
||||
|
||||
size_t litcSize = ZSTD_getcBlockSize(src, srcSize, &litbp);
|
||||
if (ZSTD_isError(litcSize)) return litcSize;
|
||||
if (litcSize > srcSize - ZSTD_blockHeaderSize) return (size_t)-ZSTD_ERROR_SrcSize;
|
||||
if (litcSize > srcSize - ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
ip += ZSTD_blockHeaderSize;
|
||||
|
||||
switch(litbp.blockType)
|
||||
{
|
||||
case bt_raw:
|
||||
*litStart = ip;
|
||||
dctx->litPtr = ip;
|
||||
dctx->litBufSize = srcSize - ZSTD_blockHeaderSize;
|
||||
dctx->litSize = litcSize;
|
||||
ip += litcSize;
|
||||
*litSize = litcSize;
|
||||
break;
|
||||
case bt_rle:
|
||||
{
|
||||
size_t rleSize = litbp.origSize;
|
||||
if (rleSize>maxDstSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
|
||||
memset(oend - rleSize, *ip, rleSize);
|
||||
*litStart = oend - rleSize;
|
||||
*litSize = rleSize;
|
||||
if (rleSize>BLOCKSIZE) return ERROR(dstSize_tooSmall);
|
||||
memset(dctx->litBuffer, *ip, rleSize);
|
||||
dctx->litPtr = dctx->litBuffer;
|
||||
dctx->litBufSize = BLOCKSIZE;
|
||||
dctx->litSize = rleSize;
|
||||
ip++;
|
||||
break;
|
||||
}
|
||||
case bt_compressed:
|
||||
{
|
||||
size_t decodedLitSize = ZSTD_decompressLiterals(ctx, dst, maxDstSize, ip, litcSize);
|
||||
size_t decodedLitSize = ZSTD_decompressLiterals(ctx, dctx->litBuffer, BLOCKSIZE, ip, litcSize);
|
||||
if (ZSTD_isError(decodedLitSize)) return decodedLitSize;
|
||||
*litStart = oend - decodedLitSize;
|
||||
*litSize = decodedLitSize;
|
||||
dctx->litPtr = dctx->litBuffer;
|
||||
dctx->litBufSize = BLOCKSIZE;
|
||||
dctx->litSize = decodedLitSize;
|
||||
ip += litcSize;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
return (size_t)-ZSTD_ERROR_GENERIC;
|
||||
/* unknown blockType (impossible) */
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
|
||||
return ip-istart;
|
||||
@ -1246,10 +1147,10 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLen
|
||||
size_t dumpsLength;
|
||||
|
||||
/* check */
|
||||
if (srcSize < 5) return (size_t)-ZSTD_ERROR_SrcSize;
|
||||
if (srcSize < 5) return ERROR(srcSize_wrong);
|
||||
|
||||
/* SeqHead */
|
||||
*nbSeq = ZSTD_readLE16(ip); ip+=2;
|
||||
*nbSeq = MEM_readLE16(ip); ip+=2;
|
||||
LLtype = *ip >> 6;
|
||||
Offtype = (*ip >> 4) & 3;
|
||||
MLtype = (*ip >> 2) & 3;
|
||||
@ -1270,7 +1171,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLen
|
||||
*dumpsLengthPtr = dumpsLength;
|
||||
|
||||
/* check */
|
||||
if (ip > iend-3) return (size_t)-ZSTD_ERROR_SrcSize; /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
|
||||
if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
|
||||
|
||||
/* sequences */
|
||||
{
|
||||
@ -1290,8 +1191,8 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLen
|
||||
default :
|
||||
max = MaxLL;
|
||||
headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
|
||||
if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
|
||||
if (LLlog > LLFSELog) return (size_t)-ZSTD_ERROR_corruption;
|
||||
if (FSE_isError(headerSize)) return ERROR(GENERIC);
|
||||
if (LLlog > LLFSELog) return ERROR(corruption_detected);
|
||||
ip += headerSize;
|
||||
FSE_buildDTable(DTableLL, norm, max, LLlog);
|
||||
}
|
||||
@ -1301,7 +1202,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLen
|
||||
U32 max;
|
||||
case bt_rle :
|
||||
Offlog = 0;
|
||||
if (ip > iend-2) return (size_t)-ZSTD_ERROR_SrcSize; /* min : "raw", hence no header, but at least xxLog bits */
|
||||
if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */
|
||||
FSE_buildDTable_rle(DTableOffb, *ip++); break;
|
||||
case bt_raw :
|
||||
Offlog = Offbits;
|
||||
@ -1309,8 +1210,8 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLen
|
||||
default :
|
||||
max = MaxOff;
|
||||
headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
|
||||
if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
|
||||
if (Offlog > OffFSELog) return (size_t)-ZSTD_ERROR_corruption;
|
||||
if (FSE_isError(headerSize)) return ERROR(GENERIC);
|
||||
if (Offlog > OffFSELog) return ERROR(corruption_detected);
|
||||
ip += headerSize;
|
||||
FSE_buildDTable(DTableOffb, norm, max, Offlog);
|
||||
}
|
||||
@ -1320,7 +1221,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLen
|
||||
U32 max;
|
||||
case bt_rle :
|
||||
MLlog = 0;
|
||||
if (ip > iend-2) return (size_t)-ZSTD_ERROR_SrcSize; /* min : "raw", hence no header, but at least xxLog bits */
|
||||
if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */
|
||||
FSE_buildDTable_rle(DTableML, *ip++); break;
|
||||
case bt_raw :
|
||||
MLlog = MLbits;
|
||||
@ -1328,8 +1229,8 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLen
|
||||
default :
|
||||
max = MaxML;
|
||||
headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
|
||||
if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
|
||||
if (MLlog > MLFSELog) return (size_t)-ZSTD_ERROR_corruption;
|
||||
if (FSE_isError(headerSize)) return ERROR(GENERIC);
|
||||
if (MLlog > MLFSELog) return ERROR(corruption_detected);
|
||||
ip += headerSize;
|
||||
FSE_buildDTable(DTableML, norm, max, MLlog);
|
||||
}
|
||||
@ -1346,7 +1247,7 @@ typedef struct {
|
||||
} seq_t;
|
||||
|
||||
typedef struct {
|
||||
FSE_DStream_t DStream;
|
||||
BIT_DStream_t DStream;
|
||||
FSE_DState_t stateLL;
|
||||
FSE_DState_t stateOffb;
|
||||
FSE_DState_t stateML;
|
||||
@ -1377,7 +1278,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
||||
{
|
||||
if (dumps<=(de-3))
|
||||
{
|
||||
litLength = ZSTD_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
|
||||
litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
|
||||
dumps += 3;
|
||||
}
|
||||
}
|
||||
@ -1387,11 +1288,11 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
||||
{
|
||||
U32 offsetCode, nbBits;
|
||||
offsetCode = FSE_decodeSymbol(&(seqState->stateOffb), &(seqState->DStream));
|
||||
if (ZSTD_32bits()) FSE_reloadDStream(&(seqState->DStream));
|
||||
if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream));
|
||||
nbBits = offsetCode - 1;
|
||||
if (offsetCode==0) nbBits = 0; /* cmove */
|
||||
offset = ((size_t)1 << (nbBits & ((sizeof(offset)*8)-1))) + FSE_readBits(&(seqState->DStream), nbBits);
|
||||
if (ZSTD_32bits()) FSE_reloadDStream(&(seqState->DStream));
|
||||
offset = ((size_t)1 << (nbBits & ((sizeof(offset)*8)-1))) + BIT_readBits(&(seqState->DStream), nbBits);
|
||||
if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream));
|
||||
if (offsetCode==0) offset = prevOffset;
|
||||
}
|
||||
|
||||
@ -1405,7 +1306,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
|
||||
{
|
||||
if (dumps<=(de-3))
|
||||
{
|
||||
matchLength = ZSTD_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
|
||||
matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
|
||||
dumps += 3;
|
||||
}
|
||||
}
|
||||
@ -1433,9 +1334,9 @@ static size_t ZSTD_execSequence(BYTE* op,
|
||||
const BYTE* const litEnd = *litPtr + litLength;
|
||||
|
||||
/* check */
|
||||
if (endMatch > oend) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* overwrite beyond dst buffer */
|
||||
if (litEnd > litLimit) return (size_t)-ZSTD_ERROR_corruption;
|
||||
if (sequence.matchLength > (size_t)(*litPtr-op)) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall; /* overwrite literal segment */
|
||||
if (endMatch > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
|
||||
if (litEnd > litLimit) return ERROR(corruption_detected);
|
||||
if (sequence.matchLength > (size_t)(*litPtr-op)) return ERROR(dstSize_tooSmall); /* overwrite literal segment */
|
||||
|
||||
/* copy Literals */
|
||||
if (((size_t)(*litPtr - op) < 8) || ((size_t)(oend-litEnd) < 8) || (op+litLength > oend-8))
|
||||
@ -1446,26 +1347,17 @@ static size_t ZSTD_execSequence(BYTE* op,
|
||||
*litPtr = litEnd; /* update for next sequence */
|
||||
|
||||
/* check : last match must be at a minimum distance of 8 from end of dest buffer */
|
||||
if (oend-op < 8) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
|
||||
if (oend-op < 8) return ERROR(dstSize_tooSmall);
|
||||
|
||||
/* copy Match */
|
||||
{
|
||||
const U32 overlapRisk = (((size_t)(litEnd - endMatch)) < 12);
|
||||
const BYTE* match = op - sequence.offset; /* possible underflow at op - offset ? */
|
||||
size_t qutt = 12;
|
||||
U64 saved[2];
|
||||
|
||||
/* check */
|
||||
if (match < base) return (size_t)-ZSTD_ERROR_corruption;
|
||||
if (sequence.offset > (size_t)base) return (size_t)-ZSTD_ERROR_corruption;
|
||||
|
||||
/* save beginning of literal sequence, in case of write overlap */
|
||||
if (overlapRisk)
|
||||
{
|
||||
if ((endMatch + qutt) > oend) qutt = oend-endMatch;
|
||||
memcpy(saved, endMatch, qutt);
|
||||
}
|
||||
if (match < base) return ERROR(corruption_detected);
|
||||
if (sequence.offset > (size_t)base) return ERROR(corruption_detected);
|
||||
|
||||
/* close range match, overlap */
|
||||
if (sequence.offset < 8)
|
||||
{
|
||||
const int dec64 = dec64table[sequence.offset];
|
||||
@ -1491,42 +1383,26 @@ static size_t ZSTD_execSequence(BYTE* op,
|
||||
}
|
||||
else
|
||||
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
|
||||
|
||||
/* restore, in case of overlap */
|
||||
if (overlapRisk) memcpy(endMatch, saved, qutt);
|
||||
}
|
||||
|
||||
return endMatch-ostart;
|
||||
}
|
||||
|
||||
typedef struct ZSTD_Dctx_s
|
||||
{
|
||||
U32 LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
|
||||
U32 OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
|
||||
U32 MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
|
||||
void* previousDstEnd;
|
||||
void* base;
|
||||
size_t expected;
|
||||
blockType_t bType;
|
||||
U32 phase;
|
||||
} dctx_t;
|
||||
|
||||
|
||||
static size_t ZSTD_decompressSequences(
|
||||
void* ctx,
|
||||
void* dst, size_t maxDstSize,
|
||||
const void* seqStart, size_t seqSize,
|
||||
const BYTE* litStart, size_t litSize)
|
||||
const void* seqStart, size_t seqSize)
|
||||
{
|
||||
dctx_t* dctx = (dctx_t*)ctx;
|
||||
ZSTD_Dctx* dctx = (ZSTD_Dctx*)ctx;
|
||||
const BYTE* ip = (const BYTE*)seqStart;
|
||||
const BYTE* const iend = ip + seqSize;
|
||||
BYTE* const ostart = (BYTE* const)dst;
|
||||
BYTE* op = ostart;
|
||||
BYTE* const oend = ostart + maxDstSize;
|
||||
size_t errorCode, dumpsLength;
|
||||
const BYTE* litPtr = litStart;
|
||||
const BYTE* const litEnd = litStart + litSize;
|
||||
const BYTE* litPtr = dctx->litPtr;
|
||||
const BYTE* const litMax = litPtr + dctx->litBufSize;
|
||||
const BYTE* const litEnd = litPtr + dctx->litSize;
|
||||
int nbSeq;
|
||||
const BYTE* dumps;
|
||||
U32* DTableLL = dctx->LLTable;
|
||||
@ -1550,30 +1426,31 @@ static size_t ZSTD_decompressSequences(
|
||||
seqState.dumps = dumps;
|
||||
seqState.dumpsEnd = dumps + dumpsLength;
|
||||
seqState.prevOffset = 1;
|
||||
errorCode = FSE_initDStream(&(seqState.DStream), ip, iend-ip);
|
||||
if (FSE_isError(errorCode)) return (size_t)-ZSTD_ERROR_corruption;
|
||||
errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip);
|
||||
if (ERR_isError(errorCode)) return ERROR(corruption_detected);
|
||||
FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL);
|
||||
FSE_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb);
|
||||
FSE_initDState(&(seqState.stateML), &(seqState.DStream), DTableML);
|
||||
|
||||
for ( ; (FSE_reloadDStream(&(seqState.DStream)) <= FSE_DStream_completed) && (nbSeq>0) ; )
|
||||
for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && (nbSeq>0) ; )
|
||||
{
|
||||
size_t oneSeqSize;
|
||||
nbSeq--;
|
||||
ZSTD_decodeSequence(&sequence, &seqState);
|
||||
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend);
|
||||
oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litMax, base, oend);
|
||||
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
|
||||
op += oneSeqSize;
|
||||
}
|
||||
|
||||
/* check if reached exact end */
|
||||
if ( !FSE_endOfDStream(&(seqState.DStream)) ) return (size_t)-ZSTD_ERROR_corruption; /* requested too much : data is corrupted */
|
||||
if (nbSeq<0) return (size_t)-ZSTD_ERROR_corruption; /* requested too many sequences : data is corrupted */
|
||||
if ( !BIT_endOfDStream(&(seqState.DStream)) ) return ERROR(corruption_detected); /* requested too much : data is corrupted */
|
||||
if (nbSeq<0) return ERROR(corruption_detected); /* requested too many sequences : data is corrupted */
|
||||
|
||||
/* last literal segment */
|
||||
{
|
||||
size_t lastLLSize = litEnd - litPtr;
|
||||
if (op+lastLLSize > oend) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
|
||||
if (litPtr > litEnd) return ERROR(corruption_detected);
|
||||
if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall);
|
||||
if (op != litPtr) memmove(op, litPtr, lastLLSize);
|
||||
op += lastLLSize;
|
||||
}
|
||||
@ -1588,19 +1465,17 @@ static size_t ZSTD_decompressBlock(
|
||||
void* dst, size_t maxDstSize,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
/* blockType == blockCompressed, srcSize is trusted */
|
||||
/* blockType == blockCompressed; srcSize is trusted */
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
const BYTE* litPtr;
|
||||
size_t litSize;
|
||||
size_t errorCode;
|
||||
|
||||
/* Decode literals sub-block */
|
||||
errorCode = ZSTD_decodeLiteralsBlock(ctx, dst, maxDstSize, &litPtr, &litSize, src, srcSize);
|
||||
errorCode = ZSTD_decodeLiteralsBlock(ctx, src, srcSize);
|
||||
if (ZSTD_isError(errorCode)) return errorCode;
|
||||
ip += errorCode;
|
||||
srcSize -= errorCode;
|
||||
|
||||
return ZSTD_decompressSequences(ctx, dst, maxDstSize, ip, srcSize, litPtr, litSize);
|
||||
return ZSTD_decompressSequences(ctx, dst, maxDstSize, ip, srcSize);
|
||||
}
|
||||
|
||||
|
||||
@ -1617,9 +1492,12 @@ static size_t ZSTD_decompressDCtx(void* ctx, void* dst, size_t maxDstSize, const
|
||||
blockProperties_t blockProperties;
|
||||
|
||||
/* Frame Header */
|
||||
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return (size_t)-ZSTD_ERROR_SrcSize;
|
||||
if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
|
||||
magicNumber = ZSTD_readBE32(src);
|
||||
if (magicNumber != ZSTD_magicNumber) return (size_t)-ZSTD_ERROR_MagicNumber;
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
if (magicNumber != ZSTDv01_magicNumber) return ZSTDv01_decompressDCtx(ctx, dst, maxDstSize, src, srcSize);
|
||||
#endif // defined
|
||||
if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
|
||||
ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize;
|
||||
|
||||
/* Loop on each block */
|
||||
@ -1630,7 +1508,7 @@ static size_t ZSTD_decompressDCtx(void* ctx, void* dst, size_t maxDstSize, const
|
||||
|
||||
ip += ZSTD_blockHeaderSize;
|
||||
remainingSize -= ZSTD_blockHeaderSize;
|
||||
if (blockSize > remainingSize) return (size_t)-ZSTD_ERROR_SrcSize;
|
||||
if (blockSize > remainingSize) return ERROR(srcSize_wrong);
|
||||
|
||||
switch(blockProperties.blockType)
|
||||
{
|
||||
@ -1641,14 +1519,14 @@ static size_t ZSTD_decompressDCtx(void* ctx, void* dst, size_t maxDstSize, const
|
||||
errorCode = ZSTD_copyUncompressedBlock(op, oend-op, ip, blockSize);
|
||||
break;
|
||||
case bt_rle :
|
||||
return (size_t)-ZSTD_ERROR_GENERIC; /* not yet supported */
|
||||
return ERROR(GENERIC); /* not yet supported */
|
||||
break;
|
||||
case bt_end :
|
||||
/* end of frame */
|
||||
if (remainingSize) return (size_t)-ZSTD_ERROR_SrcSize;
|
||||
if (remainingSize) return ERROR(srcSize_wrong);
|
||||
break;
|
||||
default:
|
||||
return (size_t)-ZSTD_ERROR_GENERIC;
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
if (blockSize == 0) break; /* bt_end */
|
||||
|
||||
@ -1663,7 +1541,7 @@ static size_t ZSTD_decompressDCtx(void* ctx, void* dst, size_t maxDstSize, const
|
||||
|
||||
size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
dctx_t ctx;
|
||||
ZSTD_Dctx ctx;
|
||||
ctx.base = dst;
|
||||
return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
|
||||
}
|
||||
@ -1698,15 +1576,15 @@ size_t ZSTD_freeDCtx(ZSTD_Dctx* dctx)
|
||||
|
||||
size_t ZSTD_nextSrcSizeToDecompress(ZSTD_Dctx* dctx)
|
||||
{
|
||||
return ((dctx_t*)dctx)->expected;
|
||||
return ((ZSTD_Dctx*)dctx)->expected;
|
||||
}
|
||||
|
||||
size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
|
||||
{
|
||||
dctx_t* ctx = (dctx_t*)dctx;
|
||||
ZSTD_Dctx* ctx = (ZSTD_Dctx*)dctx;
|
||||
|
||||
/* Sanity check */
|
||||
if (srcSize != ctx->expected) return (size_t)-ZSTD_ERROR_SrcSize;
|
||||
if (srcSize != ctx->expected) return ERROR(srcSize_wrong);
|
||||
if (dst != ctx->previousDstEnd) /* not contiguous */
|
||||
ctx->base = dst;
|
||||
|
||||
@ -1715,7 +1593,7 @@ size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, co
|
||||
{
|
||||
/* Check frame magic header */
|
||||
U32 magicNumber = ZSTD_readBE32(src);
|
||||
if (magicNumber != ZSTD_magicNumber) return (size_t)-ZSTD_ERROR_MagicNumber;
|
||||
if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown);
|
||||
ctx->phase = 1;
|
||||
ctx->expected = ZSTD_blockHeaderSize;
|
||||
return 0;
|
||||
@ -1754,13 +1632,13 @@ size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, co
|
||||
rSize = ZSTD_copyUncompressedBlock(dst, maxDstSize, src, srcSize);
|
||||
break;
|
||||
case bt_rle :
|
||||
return (size_t)-ZSTD_ERROR_GENERIC; /* not yet handled */
|
||||
return ERROR(GENERIC); /* not yet handled */
|
||||
break;
|
||||
case bt_end : /* should never happen (filtered at phase 1) */
|
||||
rSize = 0;
|
||||
break;
|
||||
default:
|
||||
return (size_t)-ZSTD_ERROR_GENERIC;
|
||||
return ERROR(GENERIC);
|
||||
}
|
||||
ctx->phase = 1;
|
||||
ctx->expected = ZSTD_blockHeaderSize;
|
||||
|
||||
10
lib/zstd.h
10
lib/zstd.h
@ -46,14 +46,14 @@ extern "C" {
|
||||
* Version
|
||||
**************************************/
|
||||
#define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */
|
||||
#define ZSTD_VERSION_MINOR 1 /* for new (non-breaking) interface capabilities */
|
||||
#define ZSTD_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */
|
||||
#define ZSTD_VERSION_MINOR 2 /* for new (non-breaking) interface capabilities */
|
||||
#define ZSTD_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */
|
||||
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
|
||||
unsigned ZSTD_versionNumber (void);
|
||||
|
||||
|
||||
/**************************************
|
||||
* Simple one-step functions
|
||||
* Simple functions
|
||||
**************************************/
|
||||
size_t ZSTD_compress( void* dst, size_t maxDstSize,
|
||||
const void* src, size_t srcSize);
|
||||
@ -73,7 +73,7 @@ ZSTD_decompress() :
|
||||
compressedSize : is the exact source size
|
||||
maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated.
|
||||
It must be equal or larger than originalSize, otherwise decompression will fail.
|
||||
return : the number of bytes decompressed into destination buffer (originalSize)
|
||||
return : the number of bytes decompressed into destination buffer (originalSize <= maxOriginalSize)
|
||||
or an errorCode if it fails (which can be tested using ZSTD_isError())
|
||||
*/
|
||||
|
||||
@ -85,7 +85,7 @@ size_t ZSTD_compressBound(size_t srcSize); /* maximum compressed size (wo
|
||||
|
||||
/* Error Management */
|
||||
unsigned ZSTD_isError(size_t code); /* tells if a return value is an error code */
|
||||
const char* ZSTD_getErrorName(size_t code); /* provides error code string (useful for debugging) */
|
||||
const char* ZSTD_getErrorName(size_t code); /* provides error code string */
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
|
||||
@ -69,23 +69,21 @@ size_t ZSTD_decompressContinue(ZSTD_Dctx* dctx, void* dst, size_t maxDstSize, co
|
||||
/*
|
||||
Use above functions alternatively.
|
||||
ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue().
|
||||
ZSTD_decompressContinue() will use previous data blocks to improve compresson if they are located prior to current block.
|
||||
ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block.
|
||||
Result is the number of bytes regenerated within 'dst'.
|
||||
It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
|
||||
*/
|
||||
|
||||
/**************************************
|
||||
* Prefix - version detection
|
||||
**************************************/
|
||||
#define ZSTD_magicNumber 0xFD2FB522 /* v0.2 (current)*/
|
||||
|
||||
|
||||
/**************************************
|
||||
* Error management
|
||||
**************************************/
|
||||
#define ZSTD_LIST_ERRORS(ITEM) \
|
||||
ITEM(ZSTD_OK_NoError) ITEM(ZSTD_ERROR_GENERIC) \
|
||||
ITEM(ZSTD_ERROR_MagicNumber) \
|
||||
ITEM(ZSTD_ERROR_SrcSize) ITEM(ZSTD_ERROR_maxDstSize_tooSmall) \
|
||||
ITEM(ZSTD_ERROR_corruption) \
|
||||
ITEM(ZSTD_ERROR_maxCode)
|
||||
|
||||
#define ZSTD_GENERATE_ENUM(ENUM) ENUM,
|
||||
typedef enum { ZSTD_LIST_ERRORS(ZSTD_GENERATE_ENUM) } ZSTD_errorCodes; /* exposed list of errors; static linking only */
|
||||
#include "error.h"
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
|
||||
@ -30,13 +30,13 @@
|
||||
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
|
||||
# ##########################################################################
|
||||
|
||||
VERSION?= v0.1.2
|
||||
VERSION?= v0.2.0
|
||||
|
||||
DESTDIR?=
|
||||
PREFIX ?= /usr/local
|
||||
CPPFLAGS= -I../lib -I../lib/legacy -DZSTD_VERSION=\"$(VERSION)\" -DZSTD_LEGACY_SUPPORT=1
|
||||
CFLAGS ?= -O3
|
||||
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -DZSTD_VERSION=\"$(VERSION)\"
|
||||
LDFLAGS = -I../lib
|
||||
CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes
|
||||
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
|
||||
|
||||
BINDIR = $(PREFIX)/bin
|
||||
@ -58,22 +58,22 @@ default: zstd
|
||||
|
||||
all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 datagen
|
||||
|
||||
zstd : $(ZSTDDIR)/zstd.c xxhash.c bench.c fileio.c zstdcli.c
|
||||
zstd : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
zstd32: $(ZSTDDIR)/zstd.c xxhash.c bench.c fileio.c zstdcli.c
|
||||
zstd32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fullbench : $(ZSTDDIR)/zstd.c datagen.c fullbench.c
|
||||
fullbench : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c datagen.c fullbench.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fullbench32: $(ZSTDDIR)/zstd.c datagen.c fullbench.c
|
||||
fullbench32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c datagen.c fullbench.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fuzzer : $(ZSTDDIR)/zstd.c datagen.c xxhash.c fuzzer.c
|
||||
fuzzer : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c datagen.c xxhash.c fuzzer.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fuzzer32: $(ZSTDDIR)/zstd.c datagen.c xxhash.c fuzzer.c
|
||||
fuzzer32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c datagen.c xxhash.c fuzzer.c
|
||||
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
datagen : datagen.c datagencli.c
|
||||
|
||||
@ -29,6 +29,16 @@
|
||||
The license of this file is GPLv2.
|
||||
*/
|
||||
|
||||
/**************************************
|
||||
* Tuning options
|
||||
**************************************/
|
||||
#ifndef ZSTD_LEGACY_SUPPORT
|
||||
/**LEGACY_SUPPORT :
|
||||
* decompressor can decode older formats (starting from Zstd 0.1+) */
|
||||
# define ZSTD_LEGACY_SUPPORT 1
|
||||
#endif // ZSTD_LEGACY_SUPPORT
|
||||
|
||||
|
||||
/**************************************
|
||||
* Compiler Options
|
||||
**************************************/
|
||||
@ -53,9 +63,14 @@
|
||||
#include <string.h> /* strcmp, strlen */
|
||||
#include <time.h> /* clock */
|
||||
#include <errno.h> /* errno */
|
||||
#include "mem.h"
|
||||
#include "fileio.h"
|
||||
#include "zstd_static.h"
|
||||
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
# include "zstd_v01.h" /* legacy */
|
||||
#endif // ZSTD_LEGACY_SUPPORT
|
||||
|
||||
|
||||
/**************************************
|
||||
* OS-specific Includes
|
||||
@ -75,25 +90,6 @@
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
* Basic Types
|
||||
**************************************/
|
||||
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
|
||||
# include <stdint.h>
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef uint32_t U32;
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
#else
|
||||
typedef unsigned char BYTE;
|
||||
typedef unsigned short U16;
|
||||
typedef unsigned int U32;
|
||||
typedef signed int S32;
|
||||
typedef unsigned long long U64;
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
* Constants
|
||||
**************************************/
|
||||
@ -217,8 +213,8 @@ static void FIO_getFileHandles(FILE** pfinput, FILE** pfoutput, const char* inpu
|
||||
*pfoutput = fopen( output_filename, "wb" );
|
||||
}
|
||||
|
||||
if ( *pfinput==0 ) EXM_THROW(12, "Pb opening %s", input_filename);
|
||||
if ( *pfoutput==0) EXM_THROW(13, "Pb opening %s", output_filename);
|
||||
if ( *pfinput==0 ) EXM_THROW(12, "Pb opening src : %s", input_filename);
|
||||
if ( *pfoutput==0) EXM_THROW(13, "Pb opening dst : %s", output_filename);
|
||||
}
|
||||
|
||||
|
||||
@ -308,7 +304,70 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
|
||||
}
|
||||
|
||||
|
||||
#define MAXHEADERSIZE FIO_FRAMEHEADERSIZE+3
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
|
||||
unsigned long long FIOv01_decompressFrame(FILE* foutput, FILE* finput)
|
||||
{
|
||||
size_t outBuffSize = 512 KB;
|
||||
BYTE* outBuff = (BYTE*)malloc(outBuffSize);
|
||||
size_t inBuffSize = 128 KB + 8;
|
||||
BYTE inBuff[128 KB + 8];
|
||||
BYTE* op = outBuff;
|
||||
BYTE* const oend = outBuff + outBuffSize;
|
||||
U64 filesize = 0;
|
||||
size_t toRead;
|
||||
size_t sizeCheck;
|
||||
ZSTDv01_Dctx* dctx = ZSTDv01_createDCtx();
|
||||
|
||||
|
||||
/* init */
|
||||
if (outBuff==NULL) EXM_THROW(41, "Error : not enough memory to decode legacy frame");
|
||||
|
||||
/* restore header, already read from input */
|
||||
MEM_writeLE32(inBuff, ZSTDv01_magicNumberLE);
|
||||
sizeCheck = ZSTDv01_decompressContinue(dctx, NULL, 0, inBuff, sizeof(ZSTDv01_magicNumberLE)); /* Decode frame header */
|
||||
if (ZSTDv01_isError(sizeCheck)) EXM_THROW(42, "Error decoding legacy header");
|
||||
|
||||
/* Main decompression Loop */
|
||||
toRead = ZSTDv01_nextSrcSizeToDecompress(dctx);
|
||||
while (toRead)
|
||||
{
|
||||
size_t readSize, decodedSize;
|
||||
|
||||
/* Fill input buffer */
|
||||
if (toRead > inBuffSize)
|
||||
EXM_THROW(43, "too large block");
|
||||
readSize = fread(inBuff, 1, toRead, finput);
|
||||
if (readSize != toRead)
|
||||
EXM_THROW(44, "Read error");
|
||||
|
||||
/* Decode block */
|
||||
decodedSize = ZSTDv01_decompressContinue(dctx, op, oend-op, inBuff, readSize);
|
||||
if (ZSTDv01_isError(decodedSize)) EXM_THROW(45, "Decoding error : input corrupted");
|
||||
|
||||
if (decodedSize) /* not a header */
|
||||
{
|
||||
/* Write block */
|
||||
sizeCheck = fwrite(op, 1, decodedSize, foutput);
|
||||
if (sizeCheck != decodedSize) EXM_THROW(46, "Write error : unable to write data block to destination file");
|
||||
filesize += decodedSize;
|
||||
op += decodedSize;
|
||||
if (op==oend) op = outBuff;
|
||||
DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)(filesize>>20) );
|
||||
}
|
||||
|
||||
/* prepare for next Block */
|
||||
toRead = ZSTDv01_nextSrcSizeToDecompress(dctx);
|
||||
}
|
||||
|
||||
/* release resources */
|
||||
free(outBuff);
|
||||
free(dctx);
|
||||
return filesize;
|
||||
}
|
||||
#endif /* ZSTD_LEGACY_SUPPORT */
|
||||
|
||||
|
||||
unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
|
||||
BYTE* inBuff, size_t inBuffSize,
|
||||
BYTE* outBuff, size_t outBuffSize,
|
||||
@ -357,6 +416,98 @@ unsigned long long FIO_decompressFrame(FILE* foutput, FILE* finput,
|
||||
}
|
||||
|
||||
|
||||
#define MAXHEADERSIZE (FIO_FRAMEHEADERSIZE+3)
|
||||
unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
|
||||
{
|
||||
FILE* finput, *foutput;
|
||||
BYTE* inBuff=NULL;
|
||||
size_t inBuffSize = 0;
|
||||
BYTE* outBuff=NULL;
|
||||
size_t outBuffSize = 0;
|
||||
U32 blockSize = 128 KB;
|
||||
U32 wNbBlocks = 4;
|
||||
U64 filesize = 0;
|
||||
BYTE* header[MAXHEADERSIZE];
|
||||
size_t toRead;
|
||||
size_t sizeCheck;
|
||||
|
||||
|
||||
/* Init */
|
||||
ZSTD_Dctx* dctx = ZSTD_createDCtx();
|
||||
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
|
||||
|
||||
/* for each frame */
|
||||
for ( ; ; )
|
||||
{
|
||||
/* check magic number -> version */
|
||||
U32 magicNumber;
|
||||
toRead = sizeof(ZSTD_magicNumber);;
|
||||
sizeCheck = fread(header, (size_t)1, toRead, finput);
|
||||
if (sizeCheck==0) break; /* no more input */
|
||||
if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
|
||||
|
||||
magicNumber = MEM_readLE32(header);
|
||||
switch(magicNumber)
|
||||
{
|
||||
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
|
||||
case ZSTDv01_magicNumberLE:
|
||||
filesize += FIOv01_decompressFrame(foutput, finput);
|
||||
continue;
|
||||
#endif /* ZSTD_LEGACY_SUPPORT */
|
||||
case ZSTD_magicNumber:
|
||||
break; /* normal case */
|
||||
default :
|
||||
EXM_THROW(32, "Error : unknown frame prefix");
|
||||
}
|
||||
|
||||
/* prepare frame decompression, by completing header */
|
||||
ZSTD_resetDCtx(dctx);
|
||||
toRead = ZSTD_nextSrcSizeToDecompress(dctx) - sizeof(ZSTD_magicNumber);
|
||||
if (toRead > MAXHEADERSIZE) EXM_THROW(30, "Not enough memory to read header");
|
||||
sizeCheck = fread(header+sizeof(ZSTD_magicNumber), (size_t)1, toRead, finput);
|
||||
if (sizeCheck != toRead) EXM_THROW(31, "Read error : cannot read header");
|
||||
sizeCheck = ZSTD_decompressContinue(dctx, NULL, 0, header, sizeof(ZSTD_magicNumber)+toRead); // Decode frame header
|
||||
if (ZSTD_isError(sizeCheck)) EXM_THROW(32, "Error decoding header");
|
||||
|
||||
/* Here later : blockSize determination */
|
||||
|
||||
/* Allocate Memory (if needed) */
|
||||
{
|
||||
size_t newInBuffSize = blockSize + FIO_blockHeaderSize;
|
||||
size_t newOutBuffSize = wNbBlocks * blockSize;
|
||||
if (newInBuffSize > inBuffSize)
|
||||
{
|
||||
free(inBuff);
|
||||
inBuffSize = newInBuffSize;
|
||||
inBuff = (BYTE*)malloc(inBuffSize);
|
||||
}
|
||||
if (newOutBuffSize > outBuffSize)
|
||||
{
|
||||
free(outBuff);
|
||||
outBuffSize = newOutBuffSize;
|
||||
outBuff = (BYTE*)malloc(outBuffSize);
|
||||
}
|
||||
}
|
||||
if (!inBuff || !outBuff) EXM_THROW(33, "Allocation error : not enough memory");
|
||||
|
||||
filesize += FIO_decompressFrame(foutput, finput, inBuff, inBuffSize, outBuff, outBuffSize, dctx);
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
DISPLAYLEVEL(2, "Decoded %llu bytes \n", (long long unsigned)filesize);
|
||||
|
||||
/* clean */
|
||||
free(inBuff);
|
||||
free(outBuff);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
fclose(finput);
|
||||
if (fclose(foutput)) EXM_THROW(38, "Write error : cannot properly close %s", output_filename);
|
||||
|
||||
return filesize;
|
||||
}
|
||||
|
||||
|
||||
#if 0
|
||||
unsigned long long FIO_decompressFilename(const char* output_filename, const char* input_filename)
|
||||
{
|
||||
FILE* finput, *foutput;
|
||||
@ -415,7 +566,7 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
DISPLAYLEVEL(2,"Decoded %llu bytes \n", (long long unsigned)filesize);
|
||||
DISPLAYLEVEL(2, "Decoded %llu bytes \n", (long long unsigned)filesize);
|
||||
|
||||
/* clean */
|
||||
free(inBuff);
|
||||
@ -426,4 +577,4 @@ unsigned long long FIO_decompressFilename(const char* output_filename, const cha
|
||||
|
||||
return filesize;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@ -60,6 +60,7 @@
|
||||
# include <sys/time.h> /* gettimeofday */
|
||||
#endif
|
||||
|
||||
#include "mem.h"
|
||||
#include "zstd.h"
|
||||
#include "fse_static.h"
|
||||
#include "datagen.h"
|
||||
@ -74,25 +75,6 @@
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
* Basic Types
|
||||
**************************************/
|
||||
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
|
||||
# include <stdint.h>
|
||||
typedef uint8_t BYTE;
|
||||
typedef uint16_t U16;
|
||||
typedef uint32_t U32;
|
||||
typedef int32_t S32;
|
||||
typedef uint64_t U64;
|
||||
#else
|
||||
typedef unsigned char BYTE;
|
||||
typedef unsigned short U16;
|
||||
typedef unsigned int U32;
|
||||
typedef signed int S32;
|
||||
typedef unsigned long long U64;
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
* Constants
|
||||
**************************************/
|
||||
@ -243,15 +225,12 @@ size_t local_ZSTD_decompress(void* dst, size_t dstSize, void* buff2, const void*
|
||||
return ZSTD_decompress(dst, dstSize, buff2, g_cSize);
|
||||
}
|
||||
|
||||
extern size_t ZSTD_decodeLiteralsBlock(void* ctx, void* dst, size_t maxDstSize, const BYTE** litStart, size_t* litSize, const void* src, size_t srcSize);
|
||||
extern size_t ZSTD_decodeLiteralsBlock(void* ctx, const void* src, size_t srcSize);
|
||||
size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
|
||||
{
|
||||
U32 ctx[1<<12];
|
||||
const BYTE* ll;
|
||||
size_t llSize;
|
||||
(void)src; (void)srcSize;
|
||||
ZSTD_decodeLiteralsBlock(ctx, dst, dstSize, &ll, &llSize, buff2, g_cSize);
|
||||
return (const BYTE*)dst + dstSize - ll;
|
||||
U32 ctx[40 * 1024];
|
||||
(void)src; (void)srcSize; (void)dst; (void)dstSize;
|
||||
return ZSTD_decodeLiteralsBlock(ctx, buff2, g_cSize);
|
||||
}
|
||||
|
||||
size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
|
||||
|
||||
@ -199,20 +199,20 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++);
|
||||
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize-1);
|
||||
if (!ZSTD_isError(result)) goto _output_error;
|
||||
if (result != (size_t)-ZSTD_ERROR_SrcSize) goto _output_error;
|
||||
if (result != ERROR(srcSize_wrong)) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : decompress with 1 too much byte : ", testNb++);
|
||||
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize+1);
|
||||
if (!ZSTD_isError(result)) goto _output_error;
|
||||
if (result != (size_t)-ZSTD_ERROR_SrcSize) goto _output_error;
|
||||
if (result != ERROR(srcSize_wrong)) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* Decompression defense tests */
|
||||
DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++);
|
||||
result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 3);
|
||||
if (!ZSTD_isError(result)) goto _output_error;
|
||||
if (result != (size_t)-ZSTD_ERROR_SrcSize) goto _output_error;
|
||||
if (result != ERROR(srcSize_wrong)) goto _output_error;
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user