AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #72 from Cyan4973/dev

Browse Source 
Dev
This commit is contained in:
Yann Collet 2015-11-10 11:17:57 +01:00
commit dc1af5f7b8
13 changed files with 107 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -32,7 +32,7 @@
# ################################################################
# Version number
export VERSION := 0.3.5
export VERSION := 0.3.6
PRGDIR = programs
ZSTDDIR = lib

3
NEWS
View File

@ -1,3 +1,6 @@
v0.3.6
small blocks params
v0.3.5
minor generic compression improvements

BIN
images/CSpeed.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 36 KiB

After

Width:  |  Height:  |  Size: 36 KiB

2
lib/zstd.h
View File

@ -48,7 +48,7 @@ extern "C" {
***************************************/
#define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */
#define ZSTD_VERSION_MINOR 3 /* for new (non-breaking) interface capabilities */
#define ZSTD_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */
#define ZSTD_VERSION_RELEASE 6 /* for tweaks, bug-fixes, or development */
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
unsigned ZSTD_versionNumber (void);

8
lib/zstd_internal.h
View File

@ -226,6 +226,14 @@ static const U32 g_searchStrength = 8;
*/
MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, size_t offsetCode, size_t matchCode)
{
#if 0
static const BYTE* g_start = NULL;
if (g_start==NULL) g_start = literals;
if (literals - g_start == 8695)
printf("pos %6u : %3u literals & match %3u bytes at distance %6u \n",
(U32)(literals - g_start), (U32)litLength, (U32)matchCode+4, (U32)offsetCode);
#endif
/* copy Literals */
ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
seqStorePtr->lit += litLength;

28
lib/zstdhc.c
View File

@ -111,7 +111,7 @@ size_t ZSTD_HC_freeCCtx(ZSTD_HC_CCtx* cctx)
/** ZSTD_HC_validateParams
correct params value to remain within authorized range
optimize for srcSize if srcSize > 0 */
void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize)
void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, U64 srcSizeHint)
{
const U32 btPlus = (params->strategy == ZSTD_HC_btlazy2);
@ -120,9 +120,9 @@ void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize)
if (params->windowLog < ZSTD_HC_WINDOWLOG_MIN) params->windowLog = ZSTD_HC_WINDOWLOG_MIN;
/* correct params, to use less memory */
if (srcSize > 0)
if ((srcSizeHint > 0) && (srcSizeHint < (1<<ZSTD_HC_WINDOWLOG_MAX)))
{
U32 srcLog = ZSTD_highbit((U32)srcSize-1) + 1;
U32 srcLog = ZSTD_highbit((U32)srcSizeHint-1) + 1;
if (params->windowLog > srcLog) params->windowLog = srcLog;
}
@ -139,9 +139,10 @@ void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize)
static size_t ZSTD_HC_resetCCtx_advanced (ZSTD_HC_CCtx* zc,
ZSTD_HC_parameters params)
ZSTD_HC_parameters params,
U64 srcSizeHint)
{
ZSTD_HC_validateParams(&params, 0);
ZSTD_HC_validateParams(&params, srcSizeHint);
/* reserve table memory */
{
@ -930,7 +931,7 @@ size_t ZSTD_HC_compressContinue (ZSTD_HC_CCtx* ctxPtr,
if (ip != ctxPtr->end)
{
if (ctxPtr->end != NULL)
ZSTD_HC_resetCCtx_advanced(ctxPtr, ctxPtr->params);
ZSTD_HC_resetCCtx_advanced(ctxPtr, ctxPtr->params, srcSize);
ctxPtr->base = ip;
}
@ -941,22 +942,24 @@ size_t ZSTD_HC_compressContinue (ZSTD_HC_CCtx* ctxPtr,
size_t ZSTD_HC_compressBegin_advanced(ZSTD_HC_CCtx* ctx,
void* dst, size_t maxDstSize,
const ZSTD_HC_parameters params)
const ZSTD_HC_parameters params,
U64 srcSizeHint)
{
size_t errorCode;
if (maxDstSize < 4) return ERROR(dstSize_tooSmall);
errorCode = ZSTD_HC_resetCCtx_advanced(ctx, params);
errorCode = ZSTD_HC_resetCCtx_advanced(ctx, params, srcSizeHint);
if (ZSTD_isError(errorCode)) return errorCode;
MEM_writeLE32(dst, ZSTD_magicNumber); /* Write Header */
return 4;
}
size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel)
size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint)
{
int tableID = ((srcSizeHint-1) > 128 KB); /* intentional underflow for 0 */
if (compressionLevel<=0) compressionLevel = 1;
if (compressionLevel > ZSTD_HC_MAX_CLEVEL) compressionLevel = ZSTD_HC_MAX_CLEVEL;
return ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_HC_defaultParameters[compressionLevel]);
return ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_HC_defaultParameters[tableID][compressionLevel], srcSizeHint);
}
@ -994,7 +997,7 @@ size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx,
}
/* Header */
oSize = ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, params);
oSize = ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, params, srcSize);
if(ZSTD_isError(oSize)) return oSize;
op += oSize;
maxDstSize -= oSize;
@ -1016,9 +1019,10 @@ size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx,
size_t ZSTD_HC_compressCCtx (ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)
{
const int tableID = (srcSize > 128 KB);
if (compressionLevel<=1) return ZSTD_compress(dst, maxDstSize, src, srcSize); /* fast mode */
if (compressionLevel > ZSTD_HC_MAX_CLEVEL) compressionLevel = ZSTD_HC_MAX_CLEVEL;
return ZSTD_HC_compress_advanced(ctx, dst, maxDstSize, src, srcSize, ZSTD_HC_defaultParameters[compressionLevel]);
return ZSTD_HC_compress_advanced(ctx, dst, maxDstSize, src, srcSize, ZSTD_HC_defaultParameters[tableID][compressionLevel]);
}
size_t ZSTD_HC_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSize, int compressionLevel)

34
lib/zstdhc_static.h
View File

@ -83,14 +83,14 @@ size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx,
/** ZSTD_HC_validateParams
correct params value to remain within authorized range
optimize for srcSize if srcSize > 0 */
void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize);
srcSizeHint value is optional, select 0 if not known */
void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, U64 srcSizeHint);
/* *************************************
* Streaming functions
***************************************/
size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel);
size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint);
size_t ZSTD_HC_compressContinue(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
size_t ZSTD_HC_compressEnd(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize);
@ -99,7 +99,32 @@ size_t ZSTD_HC_compressEnd(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize);
* Pre-defined compression levels
***************************************/
#define ZSTD_HC_MAX_CLEVEL 20
static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1] = {
static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[2][ZSTD_HC_MAX_CLEVEL+1] = {
{ /* for <= 128 KB */
/* W, C, H, S, L, strat */
{ 17, 12, 12, 1, 4, ZSTD_HC_fast }, /* level 0 - never used */
{ 17, 12, 13, 1, 6, ZSTD_HC_fast }, /* level 1 */
{ 17, 15, 16, 1, 5, ZSTD_HC_fast }, /* level 2 */
{ 17, 16, 17, 1, 5, ZSTD_HC_fast }, /* level 3 */
{ 17, 13, 15, 2, 4, ZSTD_HC_greedy }, /* level 4 */
{ 17, 15, 17, 3, 4, ZSTD_HC_greedy }, /* level 5 */
{ 17, 14, 17, 3, 4, ZSTD_HC_lazy }, /* level 6 */
{ 17, 16, 17, 4, 4, ZSTD_HC_lazy }, /* level 7 */
{ 17, 16, 17, 4, 4, ZSTD_HC_lazy2 }, /* level 8 */
{ 17, 17, 16, 5, 4, ZSTD_HC_lazy2 }, /* level 9 */
{ 17, 17, 16, 6, 4, ZSTD_HC_lazy2 }, /* level 10 */
{ 17, 17, 16, 7, 4, ZSTD_HC_lazy2 }, /* level 11 */
{ 17, 17, 16, 8, 4, ZSTD_HC_lazy2 }, /* level 12 */
{ 17, 18, 16, 4, 4, ZSTD_HC_btlazy2 }, /* level 13 */
{ 17, 18, 16, 5, 4, ZSTD_HC_btlazy2 }, /* level 14 */
{ 17, 18, 16, 6, 4, ZSTD_HC_btlazy2 }, /* level 15 */
{ 17, 18, 16, 7, 4, ZSTD_HC_btlazy2 }, /* level 16 */
{ 17, 18, 16, 8, 4, ZSTD_HC_btlazy2 }, /* level 17 */
{ 17, 18, 16, 9, 4, ZSTD_HC_btlazy2 }, /* level 18 */
{ 17, 18, 16, 10, 4, ZSTD_HC_btlazy2 }, /* level 19 */
{ 17, 18, 18, 12, 4, ZSTD_HC_btlazy2 }, /* level 20 */
},
{ /* for > 128 KB */
/* W, C, H, S, L, strat */
{ 18, 12, 12, 1, 4, ZSTD_HC_fast }, /* level 0 - never used */
{ 18, 14, 14, 1, 7, ZSTD_HC_fast }, /* level 1 - in fact redirected towards zstd fast */
@ -122,6 +147,7 @@ static const ZSTD_HC_parameters ZSTD_HC_defaultParameters[ZSTD_HC_MAX_CLEVEL+1]
{ 25, 24, 23, 5, 5, ZSTD_HC_btlazy2 }, /* level 18 */
{ 25, 26, 23, 5, 5, ZSTD_HC_btlazy2 }, /* level 19 */
{ 26, 27, 24, 6, 5, ZSTD_HC_btlazy2 }, /* level 20 */
}
};

2
programs/Makefile
View File

@ -30,7 +30,7 @@
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
# ##########################################################################
VERSION?= 0.3.5
VERSION?= 0.3.6
DESTDIR?=
PREFIX ?= /usr/local

2
programs/bench.c
View File

@ -380,7 +380,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
}
static U64 BMK_GetFileSize(char* infilename)
static U64 BMK_GetFileSize(const char* infilename)
{
int r;
#if defined(_MSC_VER)

48
programs/fileio.c
View File

@ -36,7 +36,7 @@
/**LEGACY_SUPPORT :
* decompressor can decode older formats (starting from Zstd 0.1+) */
# define ZSTD_LEGACY_SUPPORT 1
#endif // ZSTD_LEGACY_SUPPORT
#endif
/* *************************************
@ -58,11 +58,13 @@
/* *************************************
* Includes
***************************************/
#include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* strcmp, strlen */
#include <time.h> /* clock */
#include <errno.h> /* errno */
#include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* strcmp, strlen */
#include <time.h> /* clock */
#include <errno.h> /* errno */
#include <sys/types.h> /* stat64 */
#include <sys/stat.h> /* stat64 */
#include "mem.h"
#include "fileio.h"
#include "zstd_static.h"
@ -91,6 +93,10 @@
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
#endif
#if !defined(S_ISREG)
# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
#endif
/* *************************************
* Constants
@ -215,19 +221,35 @@ static void FIO_getFileHandles(FILE** pfinput, FILE** pfoutput, const char* inpu
if ( *pfoutput==0) EXM_THROW(13, "Pb opening dst : %s", output_filename);
}
static U64 FIO_getFileSize(const char* infilename)
{
int r;
#if defined(_MSC_VER)
struct _stat64 statbuf;
r = _stat64(infilename, &statbuf);
#else
struct stat statbuf;
r = stat(infilename, &statbuf);
#endif
if (r || !S_ISREG(statbuf.st_mode)) return 0;
return (U64)statbuf.st_size;
}
typedef void* (*FIO_createC) (void);
static void* local_ZSTD_createCCtx(void) { return (void*) ZSTD_createCCtx(); }
static void* local_ZSTD_HC_createCCtx(void) { return (void*) ZSTD_HC_createCCtx(); }
typedef size_t (*FIO_initC) (void* ctx, void* dst, size_t maxDstSize, int cLevel);
static size_t local_ZSTD_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel)
typedef size_t (*FIO_initC) (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint);
static size_t local_ZSTD_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint)
{
(void)cLevel;
(void)cLevel; (void)srcSizeHint;
return ZSTD_compressBegin((ZSTD_CCtx*)ctx, dst, maxDstSize);
}
static size_t local_ZSTD_HC_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel)
static size_t local_ZSTD_HC_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint)
{
return ZSTD_HC_compressBegin((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, cLevel);
return ZSTD_HC_compressBegin((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, cLevel, srcSizeHint);
}
typedef size_t (*FIO_continueC) (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
@ -294,6 +316,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
freeC = local_ZSTD_HC_freeCCtx;
}
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
filesize = FIO_getFileSize(input_filename);
/* Allocate Memory */
ctx = createC();
@ -304,12 +327,13 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
inEnd = inBuff + inBuffSize;
/* Write Frame Header */
cSize = initC(ctx, outBuff, outBuffSize, cLevel);
cSize = initC(ctx, outBuff, outBuffSize, cLevel, filesize);
if (ZSTD_isError(cSize)) EXM_THROW(22, "Compression error : cannot create frame header");
sizeCheck = fwrite(outBuff, 1, cSize, foutput);
if (sizeCheck!=cSize) EXM_THROW(23, "Write error : cannot write header into %s", output_filename);
compressedfilesize += cSize;
filesize = 0;
/* Main compression loop */
while (1)

2
programs/fuzzer.c
View File

@ -355,7 +355,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
dSize = ZSTD_decompress(dstBuffer, sampleSize + dSupSize, cBuffer, cSize);
CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize);
crcDest = XXH64(dstBuffer, sampleSize, 0);
CHECK(crcOrig != crcDest, "dstBuffer corrupted (pos %u / %u)", (U32)findDiff(srcBuffer+sampleStart, dstBuffer, sampleSize), (U32)sampleSize);
CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(srcBuffer+sampleStart, dstBuffer, sampleSize), (U32)sampleSize);
/* truncated src decompression test */
{

12
programs/paramgrill.c
View File

@ -122,7 +122,7 @@ static U32 g_rand = 1;
static U32 g_singleRun = 0;
static U32 g_target = 0;
static U32 g_noSeed = 0;
static const ZSTD_HC_parameters* g_seedParams = ZSTD_HC_defaultParameters;
static const ZSTD_HC_parameters* g_seedParams = ZSTD_HC_defaultParameters[0];
static ZSTD_HC_parameters g_params = { 0, 0, 0, 0, 0, ZSTD_HC_greedy };
void BMK_SetNbIterations(int nbLoops)
@ -703,12 +703,13 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
int i;
const char* rfName = "grillResults.txt";
FILE* f;
const U32 srcLog = BMK_highbit((U32)( (g_blockSize ? g_blockSize : srcSize) -1))+1;
const size_t blockSize = g_blockSize ? g_blockSize : srcSize;
const U32 srcLog = BMK_highbit((U32)(blockSize-1))+1;
if (g_singleRun)
{
BMK_result_t testResult;
ZSTD_HC_validateParams(&g_params, g_blockSize ? g_blockSize : srcSize);
ZSTD_HC_validateParams(&g_params, blockSize);
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params);
DISPLAY("\n");
return;
@ -725,12 +726,13 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
{
/* baseline config for level 1 */
BMK_result_t testResult;
params.windowLog = MIN(srcLog, 18);
params.windowLog = 18;
params.hashLog = 14;
params.contentLog = 1;
params.searchLog = 1;
params.searchLength = 7;
params.strategy = ZSTD_HC_fast;
ZSTD_HC_validateParams(&params, blockSize);
BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params);
g_cSpeedTarget[1] = (testResult.cSpeed * 15) >> 4;
}
@ -741,7 +743,9 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize)
/* populate initial solution */
{
const int tableID = (blockSize > 128 KB);
const int maxSeeds = g_noSeed ? 1 : ZSTD_HC_MAX_CLEVEL;
g_seedParams = ZSTD_HC_defaultParameters[tableID];
for (i=1; i<=maxSeeds; i++)
{
const U32 btPlus = (params.strategy == ZSTD_HC_btlazy2);

2
programs/zstdcli.c
View File

@ -70,7 +70,7 @@
**************************************/
#define COMPRESSOR_NAME "zstd command line interface"
#ifndef ZSTD_VERSION
# define ZSTD_VERSION "v0.3.5"
# define ZSTD_VERSION "v0.3.6"
#endif
#define AUTHOR "Yann Collet"
#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", COMPRESSOR_NAME, (int)(sizeof(void*)*8), ZSTD_VERSION, AUTHOR, __DATE__