AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fileio can use small blocks params

Browse Source
This commit is contained in:
Yann Collet 2015-11-09 17:42:17 +01:00
parent 43e0cd512a
commit 9f432926b4
4 changed files with 50 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
lib/zstdhc.c
View File

@ -111,7 +111,7 @@ size_t ZSTD_HC_freeCCtx(ZSTD_HC_CCtx* cctx)
/** ZSTD_HC_validateParams /** ZSTD_HC_validateParams
correct params value to remain within authorized range correct params value to remain within authorized range
optimize for srcSize if srcSize > 0 */ optimize for srcSize if srcSize > 0 */
void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize) void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, U64 srcSizeHint)
{ {
const U32 btPlus = (params->strategy == ZSTD_HC_btlazy2); const U32 btPlus = (params->strategy == ZSTD_HC_btlazy2);
@ -120,9 +120,9 @@ void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize)
if (params->windowLog < ZSTD_HC_WINDOWLOG_MIN) params->windowLog = ZSTD_HC_WINDOWLOG_MIN; if (params->windowLog < ZSTD_HC_WINDOWLOG_MIN) params->windowLog = ZSTD_HC_WINDOWLOG_MIN;
/* correct params, to use less memory */ /* correct params, to use less memory */
if (srcSize > 0) if ((srcSizeHint > 0) && (srcSizeHint < (1<<ZSTD_HC_WINDOWLOG_MAX)))
{ {
U32 srcLog = ZSTD_highbit((U32)srcSize-1) + 1; U32 srcLog = ZSTD_highbit((U32)srcSizeHint-1) + 1;
if (params->windowLog > srcLog) params->windowLog = srcLog; if (params->windowLog > srcLog) params->windowLog = srcLog;
} }
@ -139,9 +139,10 @@ void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize)
static size_t ZSTD_HC_resetCCtx_advanced (ZSTD_HC_CCtx* zc, static size_t ZSTD_HC_resetCCtx_advanced (ZSTD_HC_CCtx* zc,
ZSTD_HC_parameters params) ZSTD_HC_parameters params,
U64 srcSizeHint)
{ {
ZSTD_HC_validateParams(&params, 0); ZSTD_HC_validateParams(&params, srcSizeHint);
/* reserve table memory */ /* reserve table memory */
{ {
@ -930,7 +931,7 @@ size_t ZSTD_HC_compressContinue (ZSTD_HC_CCtx* ctxPtr,
if (ip != ctxPtr->end) if (ip != ctxPtr->end)
{ {
if (ctxPtr->end != NULL) if (ctxPtr->end != NULL)
ZSTD_HC_resetCCtx_advanced(ctxPtr, ctxPtr->params); ZSTD_HC_resetCCtx_advanced(ctxPtr, ctxPtr->params, srcSize);
ctxPtr->base = ip; ctxPtr->base = ip;
} }
@ -941,23 +942,24 @@ size_t ZSTD_HC_compressContinue (ZSTD_HC_CCtx* ctxPtr,
size_t ZSTD_HC_compressBegin_advanced(ZSTD_HC_CCtx* ctx, size_t ZSTD_HC_compressBegin_advanced(ZSTD_HC_CCtx* ctx,
void* dst, size_t maxDstSize, void* dst, size_t maxDstSize,
const ZSTD_HC_parameters params) const ZSTD_HC_parameters params,
U64 srcSizeHint)
{ {
size_t errorCode; size_t errorCode;
if (maxDstSize < 4) return ERROR(dstSize_tooSmall); if (maxDstSize < 4) return ERROR(dstSize_tooSmall);
errorCode = ZSTD_HC_resetCCtx_advanced(ctx, params); errorCode = ZSTD_HC_resetCCtx_advanced(ctx, params, srcSizeHint);
if (ZSTD_isError(errorCode)) return errorCode; if (ZSTD_isError(errorCode)) return errorCode;
MEM_writeLE32(dst, ZSTD_magicNumber); /* Write Header */ MEM_writeLE32(dst, ZSTD_magicNumber); /* Write Header */
return 4; return 4;
} }
size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel) size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint)
{ {
int tableID = 1; int tableID = ((srcSizeHint-1) > 128 KB); /* intentional underflow for 0 */
if (compressionLevel<=0) compressionLevel = 1; if (compressionLevel<=0) compressionLevel = 1;
if (compressionLevel > ZSTD_HC_MAX_CLEVEL) compressionLevel = ZSTD_HC_MAX_CLEVEL; if (compressionLevel > ZSTD_HC_MAX_CLEVEL) compressionLevel = ZSTD_HC_MAX_CLEVEL;
return ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_HC_defaultParameters[tableID][compressionLevel]); return ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_HC_defaultParameters[tableID][compressionLevel], srcSizeHint);
} }
@ -995,7 +997,7 @@ size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx,
} }
/* Header */ /* Header */
oSize = ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, params); oSize = ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, params, srcSize);
if(ZSTD_isError(oSize)) return oSize; if(ZSTD_isError(oSize)) return oSize;
op += oSize; op += oSize;
maxDstSize -= oSize; maxDstSize -= oSize;

6
lib/zstdhc_static.h
View File

@ -83,14 +83,14 @@ size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx,
/** ZSTD_HC_validateParams /** ZSTD_HC_validateParams
correct params value to remain within authorized range correct params value to remain within authorized range
optimize for srcSize if srcSize > 0 */ srcSizeHint value is optional, select 0 if not known */
void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize); void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, U64 srcSizeHint);
/* ************************************* /* *************************************
* Streaming functions * Streaming functions
***************************************/ ***************************************/
size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel); size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint);
size_t ZSTD_HC_compressContinue(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); size_t ZSTD_HC_compressContinue(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
size_t ZSTD_HC_compressEnd(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize); size_t ZSTD_HC_compressEnd(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize);

2
programs/bench.c
View File

@ -380,7 +380,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
} }
static U64 BMK_GetFileSize(char* infilename) static U64 BMK_GetFileSize(const char* infilename)
{ {
int r; int r;
#if defined(_MSC_VER) #if defined(_MSC_VER)

44
programs/fileio.c
View File

@ -36,7 +36,7 @@
/**LEGACY_SUPPORT : /**LEGACY_SUPPORT :
* decompressor can decode older formats (starting from Zstd 0.1+) */ * decompressor can decode older formats (starting from Zstd 0.1+) */
# define ZSTD_LEGACY_SUPPORT 1 # define ZSTD_LEGACY_SUPPORT 1
#endif // ZSTD_LEGACY_SUPPORT #endif
/* ************************************* /* *************************************
@ -58,11 +58,13 @@
/* ************************************* /* *************************************
* Includes * Includes
***************************************/ ***************************************/
#include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */ #include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */
#include <stdlib.h> /* malloc, free */ #include <stdlib.h> /* malloc, free */
#include <string.h> /* strcmp, strlen */ #include <string.h> /* strcmp, strlen */
#include <time.h> /* clock */ #include <time.h> /* clock */
#include <errno.h> /* errno */ #include <errno.h> /* errno */
#include <sys/types.h> /* stat64 */
#include <sys/stat.h> /* stat64 */
#include "mem.h" #include "mem.h"
#include "fileio.h" #include "fileio.h"
#include "zstd_static.h" #include "zstd_static.h"
@ -215,19 +217,35 @@ static void FIO_getFileHandles(FILE** pfinput, FILE** pfoutput, const char* inpu
if ( *pfoutput==0) EXM_THROW(13, "Pb opening dst : %s", output_filename); if ( *pfoutput==0) EXM_THROW(13, "Pb opening dst : %s", output_filename);
} }
static U64 FIO_getFileSize(const char* infilename)
{
int r;
#if defined(_MSC_VER)
struct _stat64 statbuf;
r = _stat64(infilename, &statbuf);
#else
struct stat statbuf;
r = stat(infilename, &statbuf);
#endif
if (r || !S_ISREG(statbuf.st_mode)) return 0;
return (U64)statbuf.st_size;
}
typedef void* (*FIO_createC) (void); typedef void* (*FIO_createC) (void);
static void* local_ZSTD_createCCtx(void) { return (void*) ZSTD_createCCtx(); } static void* local_ZSTD_createCCtx(void) { return (void*) ZSTD_createCCtx(); }
static void* local_ZSTD_HC_createCCtx(void) { return (void*) ZSTD_HC_createCCtx(); } static void* local_ZSTD_HC_createCCtx(void) { return (void*) ZSTD_HC_createCCtx(); }
typedef size_t (*FIO_initC) (void* ctx, void* dst, size_t maxDstSize, int cLevel); typedef size_t (*FIO_initC) (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint);
static size_t local_ZSTD_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel) static size_t local_ZSTD_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint)
{ {
(void)cLevel; (void)cLevel; (void)srcSizeHint;
return ZSTD_compressBegin((ZSTD_CCtx*)ctx, dst, maxDstSize); return ZSTD_compressBegin((ZSTD_CCtx*)ctx, dst, maxDstSize);
} }
static size_t local_ZSTD_HC_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel) static size_t local_ZSTD_HC_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint)
{ {
return ZSTD_HC_compressBegin((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, cLevel); return ZSTD_HC_compressBegin((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, cLevel, srcSizeHint);
} }
typedef size_t (*FIO_continueC) (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); typedef size_t (*FIO_continueC) (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
@ -294,6 +312,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
freeC = local_ZSTD_HC_freeCCtx; freeC = local_ZSTD_HC_freeCCtx;
} }
FIO_getFileHandles(&finput, &foutput, input_filename, output_filename); FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
filesize = FIO_getFileSize(input_filename);
/* Allocate Memory */ /* Allocate Memory */
ctx = createC(); ctx = createC();
@ -304,12 +323,13 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
inEnd = inBuff + inBuffSize; inEnd = inBuff + inBuffSize;
/* Write Frame Header */ /* Write Frame Header */
cSize = initC(ctx, outBuff, outBuffSize, cLevel); cSize = initC(ctx, outBuff, outBuffSize, cLevel, filesize);
if (ZSTD_isError(cSize)) EXM_THROW(22, "Compression error : cannot create frame header"); if (ZSTD_isError(cSize)) EXM_THROW(22, "Compression error : cannot create frame header");
sizeCheck = fwrite(outBuff, 1, cSize, foutput); sizeCheck = fwrite(outBuff, 1, cSize, foutput);
if (sizeCheck!=cSize) EXM_THROW(23, "Write error : cannot write header into %s", output_filename); if (sizeCheck!=cSize) EXM_THROW(23, "Write error : cannot write header into %s", output_filename);
compressedfilesize += cSize; compressedfilesize += cSize;
filesize = 0;
/* Main compression loop */ /* Main compression loop */
while (1) while (1)