AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

integrate fast mode

Browse Source
This commit is contained in:
Yann Collet 2015-11-05 18:16:59 +01:00
parent 1f44b3f778
commit f12c1304a8
8 changed files with 41 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -32,7 +32,7 @@
# ################################################################ # ################################################################
# Version number # Version number
export VERSION := 0.3.3 export VERSION := 0.3.4
PRGDIR = programs PRGDIR = programs
ZSTDDIR = lib ZSTDDIR = lib

3
NEWS
View File

@ -1,3 +1,6 @@
v0.3.4
Faster low cLevels
v0.3.3 v0.3.3
Small compression ratio improvement Small compression ratio improvement

2
lib/zstd.h
View File

@ -48,7 +48,7 @@ extern "C" {
***************************************/ ***************************************/
#define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */ #define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */
#define ZSTD_VERSION_MINOR 3 /* for new (non-breaking) interface capabilities */ #define ZSTD_VERSION_MINOR 3 /* for new (non-breaking) interface capabilities */
#define ZSTD_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */ #define ZSTD_VERSION_RELEASE 4 /* for tweaks, bug-fixes, or development */
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
unsigned ZSTD_versionNumber (void); unsigned ZSTD_versionNumber (void);

6
lib/zstdhc.c
View File

@ -225,10 +225,12 @@ size_t ZSTD_HC_compressBlock_fast_generic(ZSTD_HC_CCtx* ctx,
const U32 hBits = ctx->params.hashLog; const U32 hBits = ctx->params.hashLog;
seqStore_t* seqStorePtr = &(ctx->seqStore); seqStore_t* seqStorePtr = &(ctx->seqStore);
const BYTE* const base = ctx->base; const BYTE* const base = ctx->base;
const size_t maxDist = ((size_t)1 << ctx->params.windowLog);
const BYTE* const istart = (const BYTE*)src; const BYTE* const istart = (const BYTE*)src;
const BYTE* ip = istart + 1; const BYTE* ip = istart + 1;
const BYTE* anchor = istart; const BYTE* anchor = istart;
const BYTE* const lowest = (size_t)(istart-base) > maxDist ? istart-maxDist : base;
const BYTE* const iend = istart + srcSize; const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8; const BYTE* const ilimit = iend - 8;
@ -253,7 +255,9 @@ size_t ZSTD_HC_compressBlock_fast_generic(ZSTD_HC_CCtx* ctx,
hashTable[h] = (U32)(ip-base); hashTable[h] = (U32)(ip-base);
if (MEM_read32(ip-offset_2) == MEM_read32(ip)) match = ip-offset_2; if (MEM_read32(ip-offset_2) == MEM_read32(ip)) match = ip-offset_2;
if (MEM_read32(match) != MEM_read32(ip)) { ip += ((ip-anchor) >> g_searchStrength) + 1; offset_2 = offset_1; continue; } if ( (match < lowest) ||
(MEM_read32(match) != MEM_read32(ip)) )
{ ip += ((ip-anchor) >> g_searchStrength) + 1; offset_2 = offset_1; continue; }
while ((ip>anchor) && (match>base) && (ip[-1] == match[-1])) { ip--; match--; } /* catch up */ while ((ip>anchor) && (match>base) && (ip[-1] == match[-1])) { ip--; match--; } /* catch up */
{ {

11
lib/zstdhc_static.h
View File

@ -51,7 +51,7 @@ typedef enum { ZSTD_HC_fast, ZSTD_HC_greedy, ZSTD_HC_lazy, ZSTD_HC_lazy2, ZSTD_H
typedef struct typedef struct
{ {
U32 windowLog; /* largest match distance : impact decompression buffer size */ U32 windowLog; /* largest match distance : impact decompression buffer size */
U32 contentLog; /* full search segment : larger == more compression, slower, more memory*/ U32 contentLog; /* full search segment : larger == more compression, slower, more memory (useless for fast) */
U32 hashLog; /* dispatch table : larger == more memory, faster*/ U32 hashLog; /* dispatch table : larger == more memory, faster*/
U32 searchLog; /* nb of searches : larger == more compression, slower*/ U32 searchLog; /* nb of searches : larger == more compression, slower*/
U32 searchLength; /* size of matches : larger == faster decompression */ U32 searchLength; /* size of matches : larger == faster decompression */
@ -98,11 +98,11 @@ size_t ZSTD_HC_compressEnd(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize);
/* ************************************* /* *************************************
* Pre-defined compression levels * Pre-defined compression levels
***************************************/ ***************************************/
#define ZSTD_HC_MAX_CLEVEL 22 #define ZSTD_HC_MAX_CLEVEL 25
static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] = { static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] = {
/* W, C, H, S, L, strat */ /* W, C, H, S, L, strat */
{ 18, 12, 14, 1, 4, ZSTD_HC_greedy }, /* level 0 - never used */ { 18, 12, 14, 1, 4, ZSTD_HC_fast }, /* level 0 - never used */
{ 18, 12, 14, 1, 4, ZSTD_HC_greedy }, /* level 1 - in fact redirected towards zstd fast */ { 18, 12, 14, 1, 7, ZSTD_HC_fast }, /* level 1 - in fact redirected towards zstd fast */
{ 18, 12, 15, 2, 4, ZSTD_HC_greedy }, /* level 2 */ { 18, 12, 15, 2, 4, ZSTD_HC_greedy }, /* level 2 */
{ 19, 14, 18, 2, 5, ZSTD_HC_greedy }, /* level 3 */ { 19, 14, 18, 2, 5, ZSTD_HC_greedy }, /* level 3 */
{ 20, 17, 19, 3, 5, ZSTD_HC_greedy }, /* level 4 */ { 20, 17, 19, 3, 5, ZSTD_HC_greedy }, /* level 4 */
@ -124,6 +124,9 @@ static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1]
{ 25, 25, 23, 6, 5, ZSTD_HC_btlazy2 }, /* level 20 */ { 25, 25, 23, 6, 5, ZSTD_HC_btlazy2 }, /* level 20 */
{ 25, 26, 23, 8, 5, ZSTD_HC_btlazy2 }, /* level 21 */ { 25, 26, 23, 8, 5, ZSTD_HC_btlazy2 }, /* level 21 */
{ 25, 26, 23, 8, 5, ZSTD_HC_btlazy2 }, /* level 22 */ { 25, 26, 23, 8, 5, ZSTD_HC_btlazy2 }, /* level 22 */
{ 25, 26, 23, 8, 5, ZSTD_HC_btlazy2 }, /* level 23 */
{ 25, 26, 23, 8, 5, ZSTD_HC_btlazy2 }, /* level 24 */
{ 25, 26, 23, 8, 5, ZSTD_HC_btlazy2 }, /* level 25 */
}; };

2
programs/Makefile
View File

@ -30,7 +30,7 @@
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode # fullbench32: Same as fullbench, but forced to compile in 32-bits mode
# ########################################################################## # ##########################################################################
VERSION?= 0.3.3 VERSION?= 0.3.4
DESTDIR?= DESTDIR?=
PREFIX ?= /usr/local PREFIX ?= /usr/local

40
programs/paramgrill.c
View File

@ -487,7 +487,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_HC_parameters params,
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params); BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params);
for (cLevel = 2; cLevel <= ZSTD_HC_MAX_CLEVEL; cLevel++) for (cLevel = 1; cLevel <= ZSTD_HC_MAX_CLEVEL; cLevel++)
{ {
if (testResult.cSpeed < g_cSpeedTarget[cLevel]) if (testResult.cSpeed < g_cSpeedTarget[cLevel])
continue; /* not fast enough for this level */ continue; /* not fast enough for this level */
@ -513,8 +513,10 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_HC_parameters params,
double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed);
double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed);
size_t W_CMemUsed = (1 << params.windowLog) + 4 * (1 << params.hashLog) + 4 * (1 << params.contentLog); size_t W_CMemUsed = (1 << params.windowLog) + 4 * (1 << params.hashLog) +
size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + 4 * (1 << winners[cLevel].params.hashLog) + 4 * (1 << winners[cLevel].params.contentLog); ((params.strategy==ZSTD_HC_fast) ? 0 : 4 * (1 << params.contentLog));
size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + 4 * (1 << winners[cLevel].params.hashLog) +
((winners[cLevel].params.strategy==ZSTD_HC_fast) ? 0 : 4 * (1 << winners[cLevel].params.contentLog));
double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed);
double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed);
@ -669,8 +671,8 @@ static void BMK_selectRandomStart(
const void* srcBuffer, size_t srcSize, const void* srcBuffer, size_t srcSize,
ZSTD_HC_CCtx* ctx) ZSTD_HC_CCtx* ctx)
{ {
U32 id = (FUZ_rand(&g_rand) % ZSTD_HC_MAX_CLEVEL) + 1; U32 id = (FUZ_rand(&g_rand) % (ZSTD_HC_MAX_CLEVEL+1));
if ((id<2) || (winners[id].params.windowLog==0)) if ((id==0) || (winners[id].params.windowLog==0))
{ {
/* totally random entry */ /* totally random entry */
ZSTD_HC_parameters p; ZSTD_HC_parameters p;
@ -712,27 +714,29 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); } if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); }
if (g_target) if (g_target)
g_cSpeedTarget[2] = g_target * 1000; g_cSpeedTarget[1] = g_target * 1000;
else else
{ {
/* baseline config for level 2 */ /* baseline config for level 1 */
BMK_result_t testResult; BMK_result_t testResult;
params = g_seedParams[2]; params.windowLog = MIN(srcLog, 18);
params.windowLog = MIN(srcLog, params.windowLog); params.hashLog = 14;
params.contentLog = MIN(params.windowLog, params.contentLog); params.contentLog = 1;
params.searchLog = MIN(params.contentLog, params.searchLog); params.searchLog = 1;
params.searchLength = 7;
params.strategy = ZSTD_HC_fast;
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params); BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params);
g_cSpeedTarget[2] = (testResult.cSpeed * 15) >> 4; g_cSpeedTarget[1] = (testResult.cSpeed * 15) >> 4;
} }
/* establish speed objectives (relative to level 2) */ /* establish speed objectives (relative to level 1) */
for (i=3; i<=ZSTD_HC_MAX_CLEVEL; i++) for (i=2; i<=ZSTD_HC_MAX_CLEVEL; i++)
g_cSpeedTarget[i] = (g_cSpeedTarget[i-1] * 13) >> 4; g_cSpeedTarget[i] = (g_cSpeedTarget[i-1] * 25) >> 5;
/* populate initial solution */ /* populate initial solution */
{ {
const int maxSeeds = g_noSeed ? 2 : ZSTD_HC_MAX_CLEVEL; const int maxSeeds = g_noSeed ? 1 : ZSTD_HC_MAX_CLEVEL;
for (i=2; i<=maxSeeds; i++) for (i=1; i<=maxSeeds; i++)
{ {
params = g_seedParams[i]; params = g_seedParams[i];
params.windowLog = MIN(srcLog, params.windowLog); params.windowLog = MIN(srcLog, params.windowLog);
@ -1004,7 +1008,7 @@ int main(int argc, char** argv)
} }
break; break;
/* target level2 speed objective, in MB/s */ /* target level1 speed objective, in MB/s */
case 'T': case 'T':
argument++; argument++;
g_target = 0; g_target = 0;

2
programs/zstdcli.c
View File

@ -70,7 +70,7 @@
**************************************/ **************************************/
#define COMPRESSOR_NAME "zstd command line interface" #define COMPRESSOR_NAME "zstd command line interface"
#ifndef ZSTD_VERSION #ifndef ZSTD_VERSION
# define ZSTD_VERSION "v0.3.3" # define ZSTD_VERSION "v0.3.4"
#endif #endif
#define AUTHOR "Yann Collet" #define AUTHOR "Yann Collet"
#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), ZSTD_VERSION, AUTHOR, __DATE__ #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), ZSTD_VERSION, AUTHOR, __DATE__