AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

HC modes can be selected from command line

Browse Source
This commit is contained in:
Yann Collet 2015-10-29 16:49:43 +01:00
parent 76c5c2a4a0
commit 2acb5d3d48
12 changed files with 426 additions and 335 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
lib/zstd.c
View File

@ -68,6 +68,7 @@
#include <stdio.h> /* debug : printf */ #include <stdio.h> /* debug : printf */
#include "mem.h" /* low level memory routines */ #include "mem.h" /* low level memory routines */
#include "zstd_static.h" #include "zstd_static.h"
#include "zstd_Ccommon.h"
#include "fse_static.h" #include "fse_static.h"
#include "huff0.h" #include "huff0.h"
@ -527,20 +528,24 @@ size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize,
//static const U32 hashMask = (1<<HASH_LOG)-1; //static const U32 hashMask = (1<<HASH_LOG)-1;
//static const U64 prime5bytes = 889523592379ULL;
//static const U64 prime6bytes = 227718039650203ULL;
static const U64 prime7bytes = 58295818150454627ULL;
//static const U64 prime8bytes = 14923729446516375013ULL;
//static U32 ZSTD_hashPtr(const void* p) { return (U32) _bextr_u64(*(U64*)p * prime7bytes, (56-HASH_LOG), HASH_LOG); } //static U32 ZSTD_hashPtr(const void* p) { return (U32) _bextr_u64(*(U64*)p * prime7bytes, (56-HASH_LOG), HASH_LOG); }
//static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime7bytes) << 8 >> (64-HASH_LOG)); } //static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime7bytes) << 8 >> (64-HASH_LOG)); }
//static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime7bytes) >> (56-HASH_LOG)) & ((1<<HASH_LOG)-1); } //static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime7bytes) >> (56-HASH_LOG)) & ((1<<HASH_LOG)-1); }
//static U32 ZSTD_hashPtr(const void* p) { return ( ((*(U64*)p & 0xFFFFFFFFFFFFFF) * prime7bytes) >> (64-HASH_LOG)); } //static U32 ZSTD_hashPtr(const void* p) { return ( ((*(U64*)p & 0xFFFFFFFFFFFFFF) * prime7bytes) >> (64-HASH_LOG)); }
//static const U64 prime8bytes = 14923729446516375013ULL;
//static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime8bytes) >> (64-HASH_LOG)); } //static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime8bytes) >> (64-HASH_LOG)); }
static const U64 prime7bytes = 58295818150454627ULL;
static U32 ZSTD_hashPtr(const void* p) { return ( (MEM_read64(p) * prime7bytes) >> (56-HASH_LOG)) & HASH_MASK; } static U32 ZSTD_hashPtr(const void* p) { return ( (MEM_read64(p) * prime7bytes) >> (56-HASH_LOG)) & HASH_MASK; }
//static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime6bytes) >> (48-HASH_LOG)) & HASH_MASK; }
//static const U64 prime6bytes = 227718039650203ULL;
//static U32 ZSTD_hashPtr(const void* p) { return ( (MEM_read64(p) * prime6bytes) >> (48-HASH_LOG)) & HASH_MASK; }
//static const U64 prime5bytes = 889523592379ULL;
//static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime5bytes) >> (40-HASH_LOG)) & HASH_MASK; } //static U32 ZSTD_hashPtr(const void* p) { return ( (*(U64*)p * prime5bytes) >> (40-HASH_LOG)) & HASH_MASK; }
//static U32 ZSTD_hashPtr(const void* p) { return ( (*(U32*)p * KNUTH) >> (32-HASH_LOG)); } //static U32 ZSTD_hashPtr(const void* p) { return ( (*(U32*)p * KNUTH) >> (32-HASH_LOG)); }
static void ZSTD_addPtr(U32* table, const BYTE* p, const BYTE* start) { table[ZSTD_hashPtr(p)] = (U32)(p-start); } static void ZSTD_addPtr(U32* table, const BYTE* p, const BYTE* start) { table[ZSTD_hashPtr(p)] = (U32)(p-start); }
@ -572,7 +577,7 @@ static size_t ZSTD_compressBlock(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, c
const BYTE* const iend = istart + srcSize; const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8; const BYTE* const ilimit = iend - 8;
size_t prevOffset=4, offset=4; size_t offset_2=4, offset_1=4;
/* init */ /* init */
@ -591,11 +596,11 @@ static size_t ZSTD_compressBlock(ZSTD_CCtx* ctx, void* dst, size_t maxDstSize, c
size_t litLength = ip-anchor; size_t litLength = ip-anchor;
size_t matchLength = ZSTD_count(ip+MINMATCH, match+MINMATCH, iend); size_t matchLength = ZSTD_count(ip+MINMATCH, match+MINMATCH, iend);
size_t offsetCode; size_t offsetCode;
if (litLength) prevOffset = offset; if (litLength) offset_2 = offset_1;
offsetCode = ip-match; offsetCode = ip-match;
if (offsetCode == prevOffset) offsetCode = 0; if (offsetCode == offset_2) offsetCode = 0;
prevOffset = offset; offset_2 = offset_1;
offset = ip-match; offset_1 = ip-match;
ZSTD_storeSeq(seqStorePtr, litLength, anchor, offsetCode, matchLength); ZSTD_storeSeq(seqStorePtr, litLength, anchor, offsetCode, matchLength);
/* Fill Table */ /* Fill Table */
@ -1159,7 +1164,7 @@ static size_t ZSTD_execSequence(BYTE* op,
/* check */ /* check */
if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */ if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */
if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */ if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */
if (litEnd > litLimit_8) return ERROR(corruption_detected); /* overRead beyond lit buffer */ if (litEnd > litLimit_8) return ERROR(corruption_detected); /* risk read beyond lit buffer */
/* copy Literals */ /* copy Literals */
ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */ ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */

243
lib/zstd_Ccommon.h Normal file
View File

@ -0,0 +1,243 @@
/*
zstd_CCommon - common functions
Header File for include
Copyright (C) 2014-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- zstd source repository : https://github.com/Cyan4973/zstd
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
*/
#ifndef ZSTD_CCOMMON_H_MODULE
#define ZSTD_CCOMMON_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/* *************************************
* Includes
***************************************/
#include "mem.h"
#include "error.h"
/* *************************************
* Function body to include
***************************************/
static size_t ZSTD_read_ARCH(const void* p) { size_t r; memcpy(&r, p, sizeof(r)); return r; }
MEM_STATIC unsigned ZSTD_NbCommonBytes (register size_t val)
{
if (MEM_isLittleEndian())
{
if (MEM_64bits())
{
# if defined(_MSC_VER) && defined(_WIN64)
unsigned long r = 0;
_BitScanForward64( &r, (U64)val );
return (int)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_ctzll((U64)val) >> 3);
# else
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
# endif
}
else /* 32 bits */
{
# if defined(_MSC_VER)
unsigned long r;
_BitScanForward( &r, (U32)val );
return (int)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_ctz((U32)val) >> 3);
# else
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
# endif
}
}
else /* Big Endian CPU */
{
if (MEM_32bits())
{
# if defined(_MSC_VER) && defined(_WIN64)
unsigned long r = 0;
_BitScanReverse64( &r, val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_clzll(val) >> 3);
# else
unsigned r;
const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
r += (!val);
return r;
# endif
}
else /* 32 bits */
{
# if defined(_MSC_VER)
unsigned long r = 0;
_BitScanReverse( &r, (unsigned long)val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_clz((U32)val) >> 3);
# else
unsigned r;
if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
r += (!val);
return r;
# endif
}
}
}
MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
{
const BYTE* const pStart = pIn;
while ((pIn<pInLimit-(sizeof(size_t)-1)))
{
size_t diff = ZSTD_read_ARCH(pMatch) ^ ZSTD_read_ARCH(pIn);
if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
pIn += ZSTD_NbCommonBytes(diff);
return (size_t)(pIn - pStart);
}
if (MEM_32bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
return (size_t)(pIn - pStart);
}
static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
BYTE* const oend = op + length;
do COPY8(op, ip) while (op < oend);
}
typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
typedef struct
{
blockType_t blockType;
U32 origSize;
} blockProperties_t;
size_t ZSTD_noCompressBlock(void* op, size_t maxDstSize, const void* ip, size_t blockSize);
typedef struct {
void* buffer;
U32* offsetStart;
U32* offset;
BYTE* offCodeStart;
BYTE* offCode;
BYTE* litStart;
BYTE* lit;
BYTE* litLengthStart;
BYTE* litLength;
BYTE* matchLengthStart;
BYTE* matchLength;
BYTE* dumpsStart;
BYTE* dumps;
} seqStore_t;
void ZSTD_resetSeqStore(seqStore_t* ssPtr);
#define REPCODE_STARTVALUE 4
#define MLbits 7
#define LLbits 6
#define Offbits 5
#define MaxML ((1<<MLbits) - 1)
#define MaxLL ((1<<LLbits) - 1)
#define MaxOff 31
/** ZSTD_storeSeq
Store a sequence (literal length, literals, offset code and match length) into seqStore_t
@offsetCode : distance to match, or 0 == repCode
@matchCode : matchLength - MINMATCH
*/
MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, size_t offsetCode, size_t matchCode)
{
/* copy Literals */
ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
seqStorePtr->lit += litLength;
/* literal Length */
if (litLength >= MaxLL)
{
*(seqStorePtr->litLength++) = MaxLL;
if (litLength<255 + MaxLL)
*(seqStorePtr->dumps++) = (BYTE)(litLength - MaxLL);
else
{
*(seqStorePtr->dumps++) = 255;
MEM_writeLE32(seqStorePtr->dumps, (U32)litLength); seqStorePtr->dumps += 3;
}
}
else *(seqStorePtr->litLength++) = (BYTE)litLength;
/* match offset */
*(seqStorePtr->offset++) = (U32)offsetCode;
/* match Length */
if (matchCode >= MaxML)
{
*(seqStorePtr->matchLength++) = MaxML;
if (matchCode < 255+MaxML)
*(seqStorePtr->dumps++) = (BYTE)(matchCode - MaxML);
else
{
*(seqStorePtr->dumps++) = 255;
MEM_writeLE32(seqStorePtr->dumps, (U32)matchCode); seqStorePtr->dumps += 3;
}
}
else *(seqStorePtr->matchLength++) = (BYTE)matchCode;
}
/* prototype, body into zstd.c */
size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize, const seqStore_t* seqStorePtr, size_t srcSize);
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_CCOMMON_H_MODULE */

190
lib/zstd_static.h
View File

@ -82,196 +82,6 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, co
#include "error.h" #include "error.h"
/* *************************************
* Function body to include
***************************************/
#include "mem.h"
static size_t ZSTD_read_ARCH(const void* p) { size_t r; memcpy(&r, p, sizeof(r)); return r; }
MEM_STATIC unsigned ZSTD_NbCommonBytes (register size_t val)
{
if (MEM_isLittleEndian())
{
if (MEM_64bits())
{
# if defined(_MSC_VER) && defined(_WIN64)
unsigned long r = 0;
_BitScanForward64( &r, (U64)val );
return (int)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_ctzll((U64)val) >> 3);
# else
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
# endif
}
else /* 32 bits */
{
# if defined(_MSC_VER)
unsigned long r;
_BitScanForward( &r, (U32)val );
return (int)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_ctz((U32)val) >> 3);
# else
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
# endif
}
}
else /* Big Endian CPU */
{
if (MEM_32bits())
{
# if defined(_MSC_VER) && defined(_WIN64)
unsigned long r = 0;
_BitScanReverse64( &r, val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_clzll(val) >> 3);
# else
unsigned r;
const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
r += (!val);
return r;
# endif
}
else /* 32 bits */
{
# if defined(_MSC_VER)
unsigned long r = 0;
_BitScanReverse( &r, (unsigned long)val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_clz((U32)val) >> 3);
# else
unsigned r;
if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
r += (!val);
return r;
# endif
}
}
}
MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
{
const BYTE* const pStart = pIn;
while ((pIn<pInLimit-(sizeof(size_t)-1)))
{
size_t diff = ZSTD_read_ARCH(pMatch) ^ ZSTD_read_ARCH(pIn);
if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
pIn += ZSTD_NbCommonBytes(diff);
return (size_t)(pIn - pStart);
}
if (MEM_32bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
return (size_t)(pIn - pStart);
}
static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
BYTE* const oend = op + length;
do COPY8(op, ip) while (op < oend);
}
typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
typedef struct
{
blockType_t blockType;
U32 origSize;
} blockProperties_t;
size_t ZSTD_noCompressBlock(void* op, size_t maxDstSize, const void* ip, size_t blockSize);
typedef struct {
void* buffer;
U32* offsetStart;
U32* offset;
BYTE* offCodeStart;
BYTE* offCode;
BYTE* litStart;
BYTE* lit;
BYTE* litLengthStart;
BYTE* litLength;
BYTE* matchLengthStart;
BYTE* matchLength;
BYTE* dumpsStart;
BYTE* dumps;
} seqStore_t;
void ZSTD_resetSeqStore(seqStore_t* ssPtr);
#define REPCODE_STARTVALUE 4
#define MLbits 7
#define LLbits 6
#define Offbits 5
#define MaxML ((1<<MLbits) - 1)
#define MaxLL ((1<<LLbits) - 1)
#define MaxOff 31
/** ZSTD_storeSeq
Store a sequence (literal length, literals, offset code and match length) into seqStore_t
@offsetCode : distance to match, or 0 == repCode
@matchCode : matchLength - MINMATCH
*/
MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, size_t offsetCode, size_t matchCode)
{
/* copy Literals */
ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
seqStorePtr->lit += litLength;
/* literal Length */
if (litLength >= MaxLL)
{
*(seqStorePtr->litLength++) = MaxLL;
if (litLength<255 + MaxLL)
*(seqStorePtr->dumps++) = (BYTE)(litLength - MaxLL);
else
{
*(seqStorePtr->dumps++) = 255;
MEM_writeLE32(seqStorePtr->dumps, (U32)litLength); seqStorePtr->dumps += 3;
}
}
else *(seqStorePtr->litLength++) = (BYTE)litLength;
/* match offset */
*(seqStorePtr->offset++) = (U32)offsetCode;
/* match Length */
if (matchCode >= MaxML)
{
*(seqStorePtr->matchLength++) = MaxML;
if (matchCode < 255+MaxML)
*(seqStorePtr->dumps++) = (BYTE)(matchCode - MaxML);
else
{
*(seqStorePtr->dumps++) = 255;
MEM_writeLE32(seqStorePtr->dumps, (U32)matchCode); seqStorePtr->dumps += 3;
}
}
else *(seqStorePtr->matchLength++) = (BYTE)matchCode;
}
size_t ZSTD_compressSequences(BYTE* dst, size_t maxDstSize, const seqStore_t* seqStorePtr, size_t srcSize);
#if defined (__cplusplus) #if defined (__cplusplus)
} }
#endif #endif

179
lib/zstdhc.c
View File

@ -39,6 +39,7 @@
#include <string.h> /* memset */ #include <string.h> /* memset */
#include "zstdhc_static.h" #include "zstdhc_static.h"
#include "zstd_static.h" #include "zstd_static.h"
#include "zstd_Ccommon.h"
#include "mem.h" #include "mem.h"
@ -73,8 +74,7 @@ struct ZSTD_HC_CCtx_s
U32 lowLimit; /* below that point, no more data */ U32 lowLimit; /* below that point, no more data */
U32 nextToUpdate; /* index from which to continue dictionary update */ U32 nextToUpdate; /* index from which to continue dictionary update */
ZSTD_HC_parameters params; ZSTD_HC_parameters params;
U32 hashTableLog; size_t tableSpace;
U32 chainTableLog;
U32* hashTable; U32* hashTable;
U32* chainTable; U32* chainTable;
seqStore_t seqStore; /* sequences storage ptrs */ seqStore_t seqStore; /* sequences storage ptrs */
@ -86,47 +86,49 @@ ZSTD_HC_CCtx* ZSTD_HC_createCCtx(void)
{ {
ZSTD_HC_CCtx* ctx = (ZSTD_HC_CCtx*) malloc(sizeof(ZSTD_HC_CCtx)); ZSTD_HC_CCtx* ctx = (ZSTD_HC_CCtx*) malloc(sizeof(ZSTD_HC_CCtx));
ctx->hashTable = NULL; ctx->hashTable = NULL;
ctx->chainTable = NULL; ctx->tableSpace = 0;
ctx->hashTableLog = 0;
ctx->chainTableLog = 0;
return ctx; return ctx;
} }
size_t ZSTD_HC_freeCCtx(ZSTD_HC_CCtx* cctx) size_t ZSTD_HC_freeCCtx(ZSTD_HC_CCtx* cctx)
{ {
free(cctx->hashTable); free(cctx->hashTable);
free(cctx->chainTable);
free(cctx); free(cctx);
return 0; return 0;
} }
static void ZSTD_HC_resetCCtx_advanced (ZSTD_HC_CCtx* zc, static void ZSTD_HC_resetCCtx_advanced (ZSTD_HC_CCtx* zc,
const ZSTD_HC_parameters params, const void* start) ZSTD_HC_parameters params)
{ {
U32 outOfReach = ( 1 << params.searchLog) + 1; /* validate params */
if (params.windowLog > ZSTD_HC_WINDOWLOG_MAX) params.windowLog = ZSTD_HC_WINDOWLOG_MAX;
if (params.windowLog < ZSTD_HC_WINDOWLOG_MIN) params.windowLog = ZSTD_HC_WINDOWLOG_MIN;
if (params.chainLog > params.windowLog) params.chainLog = params.windowLog; /* <= ZSTD_HC_CHAINLOG_MAX */
if (params.chainLog < ZSTD_HC_CHAINLOG_MIN) params.chainLog = ZSTD_HC_CHAINLOG_MIN;
if (params.hashLog > ZSTD_HC_HASHLOG_MAX) params.hashLog = ZSTD_HC_HASHLOG_MAX;
if (params.hashLog < ZSTD_HC_HASHLOG_MIN) params.hashLog = ZSTD_HC_HASHLOG_MIN;
if (params.searchLog > ZSTD_HC_SEARCHLOG_MAX) params.searchLog = ZSTD_HC_SEARCHLOG_MAX;
if (params.searchLog < ZSTD_HC_SEARCHLOG_MIN) params.searchLog = ZSTD_HC_SEARCHLOG_MIN;
if (zc->hashTableLog < params.hashLog) /* reserve table memory */
{ {
free(zc->hashTable); const size_t neededSpace = ((1 << params.chainLog) + (1 << params.hashLog)) * sizeof(U32);
zc->hashTableLog = params.hashLog; if (neededSpace > zc->tableSpace)
zc->hashTable = (U32*) malloc ( (1 << zc->hashTableLog) * sizeof(U32) ); {
free(zc->hashTable);
zc->tableSpace = neededSpace;
zc->hashTable = (U32*) malloc ( neededSpace );
}
zc->chainTable = zc->hashTable + (1 << params.hashLog);
memset(zc->hashTable, 0, neededSpace );
} }
memset(zc->hashTable, 0, (1 << params.hashLog) * sizeof(U32) );
if (zc->chainTableLog < params.chainLog) zc->nextToUpdate = 0;
{ zc->end = NULL;
free(zc->chainTable); zc->base = NULL;
zc->chainTableLog = params.chainLog; zc->dictBase = NULL;
zc->chainTable = (U32*) malloc ( (1 << zc->chainTableLog) * sizeof(U32) ); zc->dictLimit = 0;
} zc->lowLimit = 0;
memset(zc->chainTable, 0, (1 << params.chainLog) * sizeof(U32) );
zc->nextToUpdate = outOfReach;
zc->end = (const BYTE*)start;
zc->base = zc->end - outOfReach;
zc->dictBase = zc->base;
zc->dictLimit = outOfReach;
zc->lowLimit = outOfReach;
zc->params = params; zc->params = params;
zc->seqStore.buffer = zc->buffer; zc->seqStore.buffer = zc->buffer;
zc->seqStore.offsetStart = (U32*) (zc->seqStore.buffer); zc->seqStore.offsetStart = (U32*) (zc->seqStore.buffer);
@ -135,17 +137,23 @@ static void ZSTD_HC_resetCCtx_advanced (ZSTD_HC_CCtx* zc,
zc->seqStore.litLengthStart = zc->seqStore.litStart + BLOCKSIZE; zc->seqStore.litLengthStart = zc->seqStore.litStart + BLOCKSIZE;
zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + (BLOCKSIZE>>2); zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + (BLOCKSIZE>>2);
zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + (BLOCKSIZE>>2); zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + (BLOCKSIZE>>2);
} }
/* ************************************* /* *************************************
* Local Macros * Local Macros
***************************************/ ***************************************/
#define KNUTH 2654435761U #define KNUTH 2654435761U
static U32 ZSTD_HC_hash(U32 u, U32 h) { return (u * KNUTH) >> (32-h) ; } static U32 ZSTD_HC_hash(U32 u, U32 h) { return (u * KNUTH) >> (32-h) ; }
static U32 ZSTD_HC_hashPtr(const void* ptr, U32 h) { return ZSTD_HC_hash(MEM_read32(ptr), h); }
//static const U64 prime5bytes = 889523592379ULL;
//static U32 ZSTD_HC_hashPtr(const void* p, U32 h) { return ((MEM_read64(p) * prime5bytes) << (64-40)) >> (64-h); }
#define NEXT_IN_CHAIN(d) chainTable[(d) & chainMask] /* flexible, CHAINSIZE dependent */ #define NEXT_IN_CHAIN(d) chainTable[(d) & chainMask] /* flexible, CHAINSIZE dependent */
static U32 ZSTD_HC_hashPtr(const void* ptr, U32 h) { return ZSTD_HC_hash(MEM_read32(ptr), h); }
/* ************************************* /* *************************************
@ -369,87 +377,60 @@ static size_t ZSTD_HC_compress_generic (ZSTD_HC_CCtx* ctxPtr,
} }
size_t ZSTD_HC_loadDict(ZSTD_HC_CCtx* ctx, const void* dictionary, size_t dictSize) size_t ZSTD_HC_compressContinue (ZSTD_HC_CCtx* ctxPtr,
void* dst, size_t dstSize,
const void* src, size_t srcSize)
{ {
/* TBD */ const BYTE* const ip = (const BYTE*) src;
(void)ctx; (void)dictionary; (void)dictSize;
return 0;
}
static void ZSTD_HC_setExternalDict(ZSTD_HC_CCtx* ctxPtr, const void* newBlock)
{
if (ctxPtr->end >= ctxPtr->base + 4)
ZSTD_HC_insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */
/* Only one memory segment for extDict, so any previous extDict is lost at this stage */
ctxPtr->lowLimit = ctxPtr->dictLimit;
ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);
ctxPtr->dictBase = ctxPtr->base;
ctxPtr->base = (const BYTE*)newBlock - ctxPtr->dictLimit;
ctxPtr->end = (const BYTE*)newBlock;
ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */
}
size_t ZSTD_HC_compress_continue (ZSTD_HC_CCtx* ctxPtr,
void* dst, size_t dstSize,
const void* src, size_t srcSize)
{
const U32 maxDistance = 1 << ctxPtr->params.windowLog;
/* Check overflow */
if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB)
{
size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit;
if (dictSize > maxDistance) dictSize = maxDistance;
ZSTD_HC_loadDict(ctxPtr, ctxPtr->end - dictSize, dictSize);
}
/* Check if blocks follow each other */ /* Check if blocks follow each other */
if ((const BYTE*)src != ctxPtr->end) if (ip != ctxPtr->end)
ZSTD_HC_setExternalDict(ctxPtr, (const BYTE*)src);
/* Check overlapping src/dictionary space (typical of cycling buffers) */
{ {
const BYTE* sourceEnd = (const BYTE*) src + srcSize; if (ctxPtr->end != NULL)
const BYTE* dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit; ZSTD_HC_resetCCtx_advanced(ctxPtr, ctxPtr->params); /* reset */
const BYTE* dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit; ctxPtr->base = ip;
if ((sourceEnd > dictBegin) && ((const BYTE*)src < dictEnd))
{
if (sourceEnd > dictEnd) sourceEnd = dictEnd;
ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase);
if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit;
}
} }
ctxPtr->end = ip + srcSize;
return ZSTD_HC_compress_generic (ctxPtr, dst, dstSize, src, srcSize); return ZSTD_HC_compress_generic (ctxPtr, dst, dstSize, src, srcSize);
} }
size_t ZSTD_HC_compressBegin_advanced(ZSTD_HC_CCtx* ctx, size_t ZSTD_HC_compressBegin_advanced(ZSTD_HC_CCtx* ctx,
void* dst, size_t maxDstSize, void* dst, size_t maxDstSize,
const ZSTD_HC_parameters params, const void* src) const ZSTD_HC_parameters params)
{ {
/* Sanity check */
if (maxDstSize < 4) return ERROR(dstSize_tooSmall); if (maxDstSize < 4) return ERROR(dstSize_tooSmall);
ZSTD_HC_resetCCtx_advanced(ctx, params);
/* Init */ MEM_writeLE32(dst, ZSTD_magicNumber); /* Write Header */
ZSTD_HC_resetCCtx_advanced(ctx, params, src);
/* Write Header */
MEM_writeLE32(dst, ZSTD_magicNumber);
return 4; return 4;
} }
size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, unsigned compressionLevel, const void* src) size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel)
{ {
if (compressionLevel==0) compressionLevel = ZSTD_HC_compressionLevel_default; if (compressionLevel<=0) compressionLevel = 1;
if (compressionLevel > ZSTD_HC_MAX_CLEVEL) compressionLevel = ZSTD_HC_MAX_CLEVEL; if (compressionLevel > ZSTD_HC_MAX_CLEVEL) compressionLevel = ZSTD_HC_MAX_CLEVEL;
return ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_HC_defaultParameters[compressionLevel], src); return ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_HC_defaultParameters[compressionLevel]);
} }
size_t ZSTD_HC_compressEnd(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize)
{
BYTE* op = (BYTE*)dst;
/* Sanity check */
(void)ctx;
if (maxDstSize < 3) return ERROR(dstSize_tooSmall);
/* End of frame */
op[0] = (BYTE)(bt_end << 6);
op[1] = 0;
op[2] = 0;
return 3;
}
size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx, size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx,
void* dst, size_t maxDstSize, void* dst, size_t maxDstSize,
const void* src, size_t srcSize, const void* src, size_t srcSize,
@ -459,51 +440,37 @@ size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx,
BYTE* op = ostart; BYTE* op = ostart;
/* Header */ /* Header */
size_t oSize = ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, params, src); size_t oSize = ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, params);
if(ZSTD_isError(oSize)) return oSize; if(ZSTD_isError(oSize)) return oSize;
op += oSize; op += oSize;
maxDstSize -= oSize; maxDstSize -= oSize;
/* body (compression) */ /* body (compression) */
ctx->base = src;
op += ZSTD_HC_compress_generic (ctx, op, maxDstSize, src, srcSize); op += ZSTD_HC_compress_generic (ctx, op, maxDstSize, src, srcSize);
if(ZSTD_isError(oSize)) return oSize; if(ZSTD_isError(oSize)) return oSize;
op += oSize; op += oSize;
maxDstSize -= oSize; maxDstSize -= oSize;
/* Close frame */ /* Close frame */
oSize = ZSTD_compressEnd((ZSTD_CCtx*)ctx, op, maxDstSize); oSize = ZSTD_HC_compressEnd(ctx, op, maxDstSize);
if(ZSTD_isError(oSize)) return oSize; if(ZSTD_isError(oSize)) return oSize;
op += oSize; op += oSize;
return (op - ostart); return (op - ostart);
} }
size_t ZSTD_HC_compressCCtx (ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel) size_t ZSTD_HC_compressCCtx (ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
{ {
if (compressionLevel==0) return ZSTD_compress(dst, maxDstSize, src, srcSize); /* fast mode */ if (compressionLevel<=1) return ZSTD_compress(dst, maxDstSize, src, srcSize); /* fast mode */
if (compressionLevel > ZSTD_HC_MAX_CLEVEL) compressionLevel = ZSTD_HC_MAX_CLEVEL; if (compressionLevel > ZSTD_HC_MAX_CLEVEL) compressionLevel = ZSTD_HC_MAX_CLEVEL;
return ZSTD_HC_compress_advanced(ctx, dst, maxDstSize, src, srcSize, ZSTD_HC_defaultParameters[compressionLevel]); return ZSTD_HC_compress_advanced(ctx, dst, maxDstSize, src, srcSize, ZSTD_HC_defaultParameters[compressionLevel]);
} }
size_t ZSTD_HC_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel) size_t ZSTD_HC_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
{ {
ZSTD_HC_CCtx* ctx = ZSTD_HC_createCCtx(); ZSTD_HC_CCtx* ctx = ZSTD_HC_createCCtx();
size_t result = ZSTD_HC_compressCCtx(ctx, dst, maxDstSize, src, srcSize, compressionLevel); size_t result = ZSTD_HC_compressCCtx(ctx, dst, maxDstSize, src, srcSize, compressionLevel);
ZSTD_HC_freeCCtx(ctx); ZSTD_HC_freeCCtx(ctx);
return result; return result;
} }
/**************************************
* Streaming Functions
**************************************/
/* dictionary saving */
size_t ZSTD_HC_saveDict (ZSTD_HC_CCtx* ctx, void* safeBuffer, size_t dictSize)
{
/* TBD */
(void)ctx; (void)safeBuffer; (void)dictSize;
return 0;
}

4
lib/zstdhc.h
View File

@ -54,7 +54,7 @@ ZSTD_HC_compress() :
*/ */
size_t ZSTD_HC_compress(void* dst, size_t maxDstSize, size_t ZSTD_HC_compress(void* dst, size_t maxDstSize,
const void* src, size_t srcSize, const void* src, size_t srcSize,
unsigned compressionLevel); int compressionLevel);
/* ************************************* /* *************************************
@ -68,7 +68,7 @@ size_t ZSTD_HC_freeCCtx(ZSTD_HC_CCtx* cctx);
ZSTD_HC_compressCCtx() : ZSTD_HC_compressCCtx() :
Same as ZSTD_compress(), but requires a ZSTD_HC_CCtx working space already allocated Same as ZSTD_compress(), but requires a ZSTD_HC_CCtx working space already allocated
*/ */
size_t ZSTD_HC_compressCCtx(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel); size_t ZSTD_HC_compressCCtx(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel);
#if defined (__cplusplus) #if defined (__cplusplus)

17
lib/zstdhc_static.h
View File

@ -65,7 +65,7 @@ typedef struct
/* ************************************* /* *************************************
* Functions * Advanced function
***************************************/ ***************************************/
/** ZSTD_HC_compress_advanced /** ZSTD_HC_compress_advanced
* Same as ZSTD_HC_compressCCtx(), but can fine-tune each compression parameter */ * Same as ZSTD_HC_compressCCtx(), but can fine-tune each compression parameter */
@ -75,16 +75,25 @@ size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx,
ZSTD_HC_parameters params); ZSTD_HC_parameters params);
/* *************************************
* Streaming functions
***************************************/
size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel);
size_t ZSTD_HC_compressContinue(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
size_t ZSTD_HC_compressEnd(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize);
/* ************************************* /* *************************************
* Pre-defined compression levels * Pre-defined compression levels
***************************************/ ***************************************/
#define ZSTD_HC_MAX_CLEVEL 25 #define ZSTD_HC_MAX_CLEVEL 26
static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] = { static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] = {
/* W, C, H, S */ /* W, C, H, S */
{ 18, 12, 14, 1 }, /* level 0 - never used */ { 18, 12, 14, 1 }, /* level 0 - never used */
{ 18, 12, 14, 1 }, /* real level 1 - all levels below are +1 */
{ 18, 12, 15, 2 }, /* level 1 */ { 18, 12, 15, 2 }, /* level 1 */
{ 19, 14, 16, 3 }, /* level 2 */ { 19, 14, 16, 3 }, /* level 2 */
{ 20, 19, 19, 2 }, /* level 3 */ { 20, 18, 18, 3 }, /* level 3 */
{ 20, 19, 19, 3 }, /* level 4 */ { 20, 19, 19, 3 }, /* level 4 */
{ 20, 19, 19, 4 }, /* level 5 */ { 20, 19, 19, 4 }, /* level 5 */
{ 20, 20, 19, 4 }, /* level 6 */ { 20, 20, 19, 4 }, /* level 6 */
@ -110,8 +119,6 @@ static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1]
}; };
#if defined (__cplusplus) #if defined (__cplusplus)
} }
#endif #endif

2
programs/Makefile
View File

@ -58,7 +58,7 @@ default: zstd
all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 paramgrill datagen all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 paramgrill datagen
zstd : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c zstd: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c
$(CC) $(FLAGS) $^ -o $@$(EXT) $(CC) $(FLAGS) $^ -o $@$(EXT)
zstd32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c zstd32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c

17
programs/bench.c
View File

@ -229,9 +229,9 @@ typedef struct
size_t resSize; size_t resSize;
} blockParam_t; } blockParam_t;
typedef size_t (*compressor_t) (void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel); typedef size_t (*compressor_t) (void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel);
static size_t local_compress_fast (void* dst, size_t maxDstSize, const void* src, size_t srcSize, unsigned compressionLevel) static size_t local_compress_fast (void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
{ {
(void)compressionLevel; (void)compressionLevel;
return ZSTD_compress(dst, maxDstSize, src, srcSize); return ZSTD_compress(dst, maxDstSize, src, srcSize);
@ -323,10 +323,10 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
for (blockNb=0; blockNb<nbBlocks; blockNb++) for (blockNb=0; blockNb<nbBlocks; blockNb++)
cSize += blockTable[blockNb].cSize; cSize += blockTable[blockNb].cSize;
if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime / nbLoops; if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime / nbLoops;
ratio = (double)cSize / (double)srcSize*100.; ratio = (double)srcSize / (double)cSize;
DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s\r", loopNb, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000.); DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.3f),%7.1f MB/s\r", loopNb, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000.);
#if 0 #if 1
/* Decompression */ /* Decompression */
memset(resultBuffer, 0xD6, srcSize); memset(resultBuffer, 0xD6, srcSize);
@ -343,7 +343,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
milliTime = BMK_GetMilliSpan(milliTime); milliTime = BMK_GetMilliSpan(milliTime);
if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime / nbLoops; if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime / nbLoops;
DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\r", loopNb, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.); DISPLAY("%1i-%-14.14s : %9i -> %9i (%5.3f),%7.1f MB/s ,%7.1f MB/s\r", loopNb, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
/* CRC Checking */ /* CRC Checking */
crcCheck = XXH64(resultBuffer, srcSize, 0); crcCheck = XXH64(resultBuffer, srcSize, 0);
@ -367,10 +367,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
if (crcOrig == crcCheck) if (crcOrig == crcCheck)
{ {
if (ratio<100.) DISPLAY("%-16.16s : %9i -> %9i (%5.3f),%7.1f MB/s ,%7.1f MB/s \n", fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\n", fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
else
DISPLAY("%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
} }
} }

76
programs/fileio.c
View File

@ -66,6 +66,7 @@
#include "mem.h" #include "mem.h"
#include "fileio.h" #include "fileio.h"
#include "zstd_static.h" #include "zstd_static.h"
#include "zstdhc_static.h"
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
# include "zstd_v01.h" /* legacy */ # include "zstd_v01.h" /* legacy */
@ -213,8 +214,47 @@ static void FIO_getFileHandles(FILE** pfinput, FILE** pfoutput, const char* inpu
if ( *pfoutput==0) EXM_THROW(13, "Pb opening dst : %s", output_filename); if ( *pfoutput==0) EXM_THROW(13, "Pb opening dst : %s", output_filename);
} }
typedef void* (*FIO_createC) (void);
static void* local_ZSTD_createCCtx(void) { return (void*) ZSTD_createCCtx(); }
static void* local_ZSTD_HC_createCCtx(void) { return (void*) ZSTD_HC_createCCtx(); }
unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename, unsigned cLevel) typedef size_t (*FIO_initC) (void* ctx, void* dst, size_t maxDstSize, int cLevel);
static size_t local_ZSTD_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel)
{
(void)cLevel;
return ZSTD_compressBegin((ZSTD_CCtx*)ctx, dst, maxDstSize);
}
static size_t local_ZSTD_HC_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel)
{
return ZSTD_HC_compressBegin((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, cLevel);
}
typedef size_t (*FIO_continueC) (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
static size_t local_ZSTD_compressContinue (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
return ZSTD_compressContinue((ZSTD_CCtx*)ctx, dst, maxDstSize, src, srcSize);
}
static size_t local_ZSTD_HC_compressContinue (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
return ZSTD_HC_compressContinue((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, src, srcSize);
}
typedef size_t (*FIO_endC) (void* ctx, void* dst, size_t maxDstSize);
static size_t local_ZSTD_compressEnd (void* ctx, void* dst, size_t maxDstSize)
{
return ZSTD_compressEnd((ZSTD_CCtx*)ctx, dst, maxDstSize);
}
static size_t local_ZSTD_HC_compressEnd (void* ctx, void* dst, size_t maxDstSize)
{
return ZSTD_HC_compressEnd((ZSTD_HC_CCtx*)ctx, dst, maxDstSize);
}
typedef void (*FIO_freeC) (void* ctx);
static void local_ZSTD_freeCCtx(void* ctx) { ZSTD_freeCCtx((ZSTD_CCtx*)ctx); }
static void local_ZSTD_HC_freeCCtx(void* ctx) { ZSTD_HC_freeCCtx((ZSTD_HC_CCtx*)ctx); }
unsigned long long FIO_compressFilename(const char* output_filename, const char* input_filename, int cLevel)
{ {
U64 filesize = 0; U64 filesize = 0;
U64 compressedfilesize = 0; U64 compressedfilesize = 0;
@ -228,14 +268,34 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
FILE* finput; FILE* finput;
FILE* foutput; FILE* foutput;
size_t sizeCheck, cSize; size_t sizeCheck, cSize;
ZSTD_CCtx* ctx = ZSTD_createCCtx(); void* ctx;
FIO_createC createC=NULL;
FIO_initC initC=NULL;
FIO_continueC continueC = NULL;
FIO_endC endC = NULL;
FIO_freeC freeC = NULL;
/* Init */ /* Init */
(void)cLevel; if (cLevel <= 1)
{
createC = local_ZSTD_createCCtx;
initC = local_ZSTD_compressBegin;
continueC = local_ZSTD_compressContinue;
endC = local_ZSTD_compressEnd;
freeC = local_ZSTD_freeCCtx;
}
else
{
createC = local_ZSTD_HC_createCCtx;
initC = local_ZSTD_HC_compressBegin;
continueC = local_ZSTD_HC_compressContinue;
endC = local_ZSTD_HC_compressEnd;
freeC = local_ZSTD_HC_freeCCtx;
}
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename); FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
/* Allocate Memory */ /* Allocate Memory */
ctx = createC();
inBuff = (BYTE*)malloc(inBuffSize); inBuff = (BYTE*)malloc(inBuffSize);
outBuff = (BYTE*)malloc(outBuffSize); outBuff = (BYTE*)malloc(outBuffSize);
if (!inBuff || !outBuff || !ctx) EXM_THROW(21, "Allocation error : not enough memory"); if (!inBuff || !outBuff || !ctx) EXM_THROW(21, "Allocation error : not enough memory");
@ -243,7 +303,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
inEnd = inBuff + inBuffSize; inEnd = inBuff + inBuffSize;
/* Write Frame Header */ /* Write Frame Header */
cSize = ZSTD_compressBegin(ctx, outBuff, outBuffSize); cSize = initC(ctx, outBuff, outBuffSize, cLevel);
if (ZSTD_isError(cSize)) EXM_THROW(22, "Compression error : cannot create frame header"); if (ZSTD_isError(cSize)) EXM_THROW(22, "Compression error : cannot create frame header");
sizeCheck = fwrite(outBuff, 1, cSize, foutput); sizeCheck = fwrite(outBuff, 1, cSize, foutput);
@ -263,7 +323,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20)); DISPLAYUPDATE(2, "\rRead : %u MB ", (U32)(filesize>>20));
/* Compress Block */ /* Compress Block */
cSize = ZSTD_compressContinue(ctx, outBuff, outBuffSize, inSlot, inSize); cSize = continueC(ctx, outBuff, outBuffSize, inSlot, inSize);
if (ZSTD_isError(cSize)) if (ZSTD_isError(cSize))
EXM_THROW(24, "Compression error : %s ", ZSTD_getErrorName(cSize)); EXM_THROW(24, "Compression error : %s ", ZSTD_getErrorName(cSize));
@ -277,7 +337,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
} }
/* End of Frame */ /* End of Frame */
cSize = ZSTD_compressEnd(ctx, outBuff, outBuffSize); cSize = endC(ctx, outBuff, outBuffSize);
if (ZSTD_isError(cSize)) EXM_THROW(26, "Compression error : cannot create frame end"); if (ZSTD_isError(cSize)) EXM_THROW(26, "Compression error : cannot create frame end");
sizeCheck = fwrite(outBuff, 1, cSize, foutput); sizeCheck = fwrite(outBuff, 1, cSize, foutput);
@ -292,7 +352,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
/* clean */ /* clean */
free(inBuff); free(inBuff);
free(outBuff); free(outBuff);
ZSTD_freeCCtx(ctx); freeC(ctx);
fclose(finput); fclose(finput);
if (fclose(foutput)) EXM_THROW(28, "Write error : cannot properly close %s", output_filename); if (fclose(foutput)) EXM_THROW(28, "Write error : cannot properly close %s", output_filename);

2
programs/fileio.h
View File

@ -52,7 +52,7 @@ void FIO_setNotificationLevel(unsigned level);
/* ************************************* /* *************************************
* Stream/File functions * Stream/File functions
***************************************/ ***************************************/
unsigned long long FIO_compressFilename (const char* outfilename, const char* infilename, unsigned cLevel); unsigned long long FIO_compressFilename (const char* outfilename, const char* infilename, int compressionLevel);
unsigned long long FIO_decompressFilename (const char* outfilename, const char* infilename); unsigned long long FIO_decompressFilename (const char* outfilename, const char* infilename);
/** /**
FIO_compressFilename : FIO_compressFilename :

2
programs/paramgrill.c
View File

@ -446,7 +446,7 @@ static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSi
int cLevel; int cLevel;
fprintf(f, "\n /* Selected configurations : */ \n"); fprintf(f, "\n /* Selected configurations : */ \n");
fprintf(f, "#define ZSTD_HC_MAX_CLEVEL 20 \n"); fprintf(f, "#define ZSTD_HC_MAX_CLEVEL %2u \n", ZSTD_HC_MAX_CLEVEL);
fprintf(f, "static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] = {\n"); fprintf(f, "static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] = {\n");
fprintf(f, " /* W, C, H, S */ \n"); fprintf(f, " /* W, C, H, S */ \n");

2
programs/zstdcli.c
View File

@ -118,6 +118,8 @@ static int usage(const char* programName)
DISPLAY( "input : a filename\n"); DISPLAY( "input : a filename\n");
DISPLAY( " with no FILE, or when FILE is - , read standard input\n"); DISPLAY( " with no FILE, or when FILE is - , read standard input\n");
DISPLAY( "Arguments :\n"); DISPLAY( "Arguments :\n");
DISPLAY( " -1 : Fast compression (default) \n");
DISPLAY( " -9 : High compression \n");
DISPLAY( " -d : decompression (default for %s extension)\n", ZSTD_EXTENSION); DISPLAY( " -d : decompression (default for %s extension)\n", ZSTD_EXTENSION);
//DISPLAY( " -z : force compression\n"); //DISPLAY( " -z : force compression\n");
DISPLAY( " -f : overwrite output without prompting \n"); DISPLAY( " -f : overwrite output without prompting \n");