Merge remote-tracking branch 'refs/remotes/Cyan4973/dev' into dev08
# Conflicts: # zstd_compression_format.md
This commit is contained in:
inikep
commit
f6aed96f61
4
NEWS
4
NEWS
@ -1,3 +1,7 @@
|
||||
v0.8.1
|
||||
Changed : -i# now selects benchmark time in second
|
||||
Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell
|
||||
|
||||
v0.8.0
|
||||
Improved : better speed on clang and gcc -O2, thanks to Eric Biggers
|
||||
New : Build on FreeBSD and DragonFly, thanks to JrMarino
|
||||
|
||||
@ -56,4 +56,8 @@ test: all
|
||||
./simple_compression tmp
|
||||
@echo starting simple_decompression
|
||||
./simple_decompression tmp.zst
|
||||
@echo dictionary compression
|
||||
./dictionary_compression tmp README.md
|
||||
@echo dictionary decompression
|
||||
./dictionary_decompression tmp.zst README.md
|
||||
@echo tests completed
|
||||
|
||||
@ -31,7 +31,7 @@
|
||||
#include <zstd.h> // presumes zstd library is installed
|
||||
|
||||
|
||||
static off_t fsize_X(const char *filename)
|
||||
static off_t fsize_orDie(const char *filename)
|
||||
{
|
||||
struct stat st;
|
||||
if (stat(filename, &st) == 0) return st.st_size;
|
||||
@ -40,7 +40,7 @@ static off_t fsize_X(const char *filename)
|
||||
exit(1);
|
||||
}
|
||||
|
||||
static FILE* fopen_X(const char *filename, const char *instruction)
|
||||
static FILE* fopen_orDie(const char *filename, const char *instruction)
|
||||
{
|
||||
FILE* const inFile = fopen(filename, instruction);
|
||||
if (inFile) return inFile;
|
||||
@ -49,20 +49,20 @@ static FILE* fopen_X(const char *filename, const char *instruction)
|
||||
exit(2);
|
||||
}
|
||||
|
||||
static void* malloc_X(size_t size)
|
||||
static void* malloc_orDie(size_t size)
|
||||
{
|
||||
void* const buff = malloc(size);
|
||||
if (buff) return buff;
|
||||
/* error */
|
||||
perror(NULL);
|
||||
perror("malloc");
|
||||
exit(3);
|
||||
}
|
||||
|
||||
static void* loadFile_X(const char* fileName, size_t* size)
|
||||
static void* loadFile_orDie(const char* fileName, size_t* size)
|
||||
{
|
||||
off_t const buffSize = fsize_X(fileName);
|
||||
FILE* const inFile = fopen_X(fileName, "rb");
|
||||
void* const buffer = malloc_X(buffSize);
|
||||
off_t const buffSize = fsize_orDie(fileName);
|
||||
FILE* const inFile = fopen_orDie(fileName, "rb");
|
||||
void* const buffer = malloc_orDie(buffSize);
|
||||
size_t const readSize = fread(buffer, 1, buffSize, inFile);
|
||||
if (readSize != (size_t)buffSize) {
|
||||
fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
|
||||
@ -73,9 +73,9 @@ static void* loadFile_X(const char* fileName, size_t* size)
|
||||
return buffer;
|
||||
}
|
||||
|
||||
static void saveFile_X(const char* fileName, const void* buff, size_t buffSize)
|
||||
static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize)
|
||||
{
|
||||
FILE* const oFile = fopen_X(fileName, "wb");
|
||||
FILE* const oFile = fopen_orDie(fileName, "wb");
|
||||
size_t const wSize = fwrite(buff, 1, buffSize, oFile);
|
||||
if (wSize != (size_t)buffSize) {
|
||||
fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno));
|
||||
@ -89,23 +89,27 @@ static void saveFile_X(const char* fileName, const void* buff, size_t buffSize)
|
||||
|
||||
/* createDict() :
|
||||
`dictFileName` is supposed to have been created using `zstd --train` */
|
||||
static const ZSTD_CDict* createDict(const char* dictFileName)
|
||||
static ZSTD_CDict* createCDict_orDie(const char* dictFileName)
|
||||
{
|
||||
size_t dictSize;
|
||||
printf("loading dictionary %s \n", dictFileName);
|
||||
void* const dictBuffer = loadFile_X(dictFileName, &dictSize);
|
||||
const ZSTD_CDict* const ddict = ZSTD_createCDict(dictBuffer, dictSize, 3);
|
||||
void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize);
|
||||
ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 3);
|
||||
if (!cdict) {
|
||||
fprintf(stderr, "ZSTD_createCDict error \n");
|
||||
exit(7);
|
||||
}
|
||||
free(dictBuffer);
|
||||
return ddict;
|
||||
return cdict;
|
||||
}
|
||||
|
||||
|
||||
static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdict)
|
||||
{
|
||||
size_t fSize;
|
||||
void* const fBuff = loadFile_X(fname, &fSize);
|
||||
void* const fBuff = loadFile_orDie(fname, &fSize);
|
||||
size_t const cBuffSize = ZSTD_compressBound(fSize);
|
||||
void* const cBuff = malloc_X(cBuffSize);
|
||||
void* const cBuff = malloc_orDie(cBuffSize);
|
||||
|
||||
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
|
||||
size_t const cSize = ZSTD_compress_usingCDict(cctx, cBuff, cBuffSize, fBuff, fSize, cdict);
|
||||
@ -114,7 +118,7 @@ static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdi
|
||||
exit(7);
|
||||
}
|
||||
|
||||
saveFile_X(oname, cBuff, cSize);
|
||||
saveFile_orDie(oname, cBuff, cSize);
|
||||
|
||||
/* success */
|
||||
printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
|
||||
@ -125,11 +129,11 @@ static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdi
|
||||
}
|
||||
|
||||
|
||||
static char* createOutFilename(const char* filename)
|
||||
static char* createOutFilename_orDie(const char* filename)
|
||||
{
|
||||
size_t const inL = strlen(filename);
|
||||
size_t const outL = inL + 5;
|
||||
void* outSpace = malloc_X(outL);
|
||||
void* outSpace = malloc_orDie(outL);
|
||||
memset(outSpace, 0, outL);
|
||||
strcat(outSpace, filename);
|
||||
strcat(outSpace, ".zst");
|
||||
@ -149,15 +153,17 @@ int main(int argc, const char** argv)
|
||||
|
||||
/* load dictionary only once */
|
||||
const char* const dictName = argv[argc-1];
|
||||
const ZSTD_CDict* const dictPtr = createDict(dictName);
|
||||
ZSTD_CDict* const dictPtr = createCDict_orDie(dictName);
|
||||
|
||||
int u;
|
||||
for (u=1; u<argc-1; u++) {
|
||||
const char* inFilename = argv[u];
|
||||
char* const outFilename = createOutFilename(inFilename);
|
||||
char* const outFilename = createOutFilename_orDie(inFilename);
|
||||
compress(inFilename, outFilename, dictPtr);
|
||||
free(outFilename);
|
||||
}
|
||||
|
||||
ZSTD_freeCDict(dictPtr);
|
||||
printf("All %u files compressed. \n", argc-2);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -31,41 +31,41 @@
|
||||
#include <zstd.h> // presumes zstd library is installed
|
||||
|
||||
|
||||
static off_t fsize_X(const char *filename)
|
||||
static off_t fsize_orDie(const char *filename)
|
||||
{
|
||||
struct stat st;
|
||||
if (stat(filename, &st) == 0) return st.st_size;
|
||||
/* error */
|
||||
printf("stat: %s : %s \n", filename, strerror(errno));
|
||||
perror(filename);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
static FILE* fopen_X(const char *filename, const char *instruction)
|
||||
static FILE* fopen_orDie(const char *filename, const char *instruction)
|
||||
{
|
||||
FILE* const inFile = fopen(filename, instruction);
|
||||
if (inFile) return inFile;
|
||||
/* error */
|
||||
printf("fopen: %s : %s \n", filename, strerror(errno));
|
||||
perror(filename);
|
||||
exit(2);
|
||||
}
|
||||
|
||||
static void* malloc_X(size_t size)
|
||||
static void* malloc_orDie(size_t size)
|
||||
{
|
||||
void* const buff = malloc(size);
|
||||
if (buff) return buff;
|
||||
/* error */
|
||||
printf("malloc: %s \n", strerror(errno));
|
||||
perror("malloc");
|
||||
exit(3);
|
||||
}
|
||||
|
||||
static void* loadFile_X(const char* fileName, size_t* size)
|
||||
static void* loadFile_orDie(const char* fileName, size_t* size)
|
||||
{
|
||||
off_t const buffSize = fsize_X(fileName);
|
||||
FILE* const inFile = fopen_X(fileName, "rb");
|
||||
void* const buffer = malloc_X(buffSize);
|
||||
off_t const buffSize = fsize_orDie(fileName);
|
||||
FILE* const inFile = fopen_orDie(fileName, "rb");
|
||||
void* const buffer = malloc_orDie(buffSize);
|
||||
size_t const readSize = fread(buffer, 1, buffSize, inFile);
|
||||
if (readSize != (size_t)buffSize) {
|
||||
printf("fread: %s : %s \n", fileName, strerror(errno));
|
||||
fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
|
||||
exit(4);
|
||||
}
|
||||
fclose(inFile);
|
||||
@ -75,12 +75,13 @@ static void* loadFile_X(const char* fileName, size_t* size)
|
||||
|
||||
/* createDict() :
|
||||
`dictFileName` is supposed to have been created using `zstd --train` */
|
||||
static const ZSTD_DDict* createDict(const char* dictFileName)
|
||||
static ZSTD_DDict* createDict_orDie(const char* dictFileName)
|
||||
{
|
||||
size_t dictSize;
|
||||
printf("loading dictionary %s \n", dictFileName);
|
||||
void* const dictBuffer = loadFile_X(dictFileName, &dictSize);
|
||||
const ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize);
|
||||
void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize);
|
||||
ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize);
|
||||
if (ddict==NULL) { fprintf(stderr, "ZSTD_createDDict error \n"); exit(5); }
|
||||
free(dictBuffer);
|
||||
return ddict;
|
||||
}
|
||||
@ -89,19 +90,19 @@ static const ZSTD_DDict* createDict(const char* dictFileName)
|
||||
static void decompress(const char* fname, const ZSTD_DDict* ddict)
|
||||
{
|
||||
size_t cSize;
|
||||
void* const cBuff = loadFile_X(fname, &cSize);
|
||||
void* const cBuff = loadFile_orDie(fname, &cSize);
|
||||
unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize);
|
||||
if (rSize==0) {
|
||||
printf("%s : original size unknown \n", fname);
|
||||
exit(5);
|
||||
fprintf(stderr, "%s : original size unknown \n", fname);
|
||||
exit(6);
|
||||
}
|
||||
void* const rBuff = malloc_X(rSize);
|
||||
void* const rBuff = malloc_orDie(rSize);
|
||||
|
||||
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
|
||||
size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff, rSize, cBuff, cSize, ddict);
|
||||
|
||||
if (dSize != rSize) {
|
||||
printf("error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
|
||||
fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
|
||||
exit(7);
|
||||
}
|
||||
|
||||
@ -127,10 +128,12 @@ int main(int argc, const char** argv)
|
||||
|
||||
/* load dictionary only once */
|
||||
const char* const dictName = argv[argc-1];
|
||||
const ZSTD_DDict* const dictPtr = createDict(dictName);
|
||||
ZSTD_DDict* const dictPtr = createDict_orDie(dictName);
|
||||
|
||||
int u;
|
||||
for (u=1; u<argc-1; u++) decompress(argv[u], dictPtr);
|
||||
|
||||
ZSTD_freeDDict(dictPtr);
|
||||
printf("All %u files correctly decoded (in memory) \n", argc-2);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -155,6 +155,16 @@ MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, size_t length)
|
||||
while (op < oend);
|
||||
}
|
||||
|
||||
MEM_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, void* dstEnd) /* should be faster for decoding, but strangely, not verified on all platform */
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
BYTE* op = (BYTE*)dst;
|
||||
BYTE* const oend = (BYTE*)dstEnd;
|
||||
do
|
||||
COPY8(op, ip)
|
||||
while (op < oend);
|
||||
}
|
||||
|
||||
|
||||
/*-*******************************************
|
||||
* Private interfaces
|
||||
|
||||
@ -973,7 +973,7 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
|
||||
const BYTE* match = base + matchIndex;
|
||||
hashTable[h] = current; /* update hash table */
|
||||
|
||||
if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { /* note : by construction, offset_1 <= current */
|
||||
if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) {
|
||||
mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
|
||||
ip++;
|
||||
ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH);
|
||||
@ -2249,6 +2249,21 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
|
||||
if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */
|
||||
if (remaining < blockSize) blockSize = remaining;
|
||||
|
||||
/* preemptive overflow correction */
|
||||
if (cctx->lowLimit > (1<<30)) {
|
||||
U32 const btplus = (cctx->params.cParams.strategy == ZSTD_btlazy2) | (cctx->params.cParams.strategy == ZSTD_btopt);
|
||||
U32 const chainMask = (1 << (cctx->params.cParams.chainLog - btplus)) - 1;
|
||||
U32 const newLowLimit = cctx->lowLimit & chainMask; /* preserve position % chainSize */
|
||||
U32 const correction = cctx->lowLimit - newLowLimit;
|
||||
ZSTD_reduceIndex(cctx, correction);
|
||||
cctx->base += correction;
|
||||
cctx->dictBase += correction;
|
||||
cctx->lowLimit = newLowLimit;
|
||||
cctx->dictLimit -= correction;
|
||||
if (cctx->nextToUpdate < correction) cctx->nextToUpdate = 0;
|
||||
else cctx->nextToUpdate -= correction;
|
||||
}
|
||||
|
||||
if ((U32)(ip+blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) {
|
||||
/* enforce maxDist */
|
||||
U32 const newLowLimit = (U32)(ip+blockSize - cctx->base) - maxDist;
|
||||
@ -2322,7 +2337,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
|
||||
}
|
||||
|
||||
|
||||
static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
|
||||
static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
U32 frame, U32 lastFrameChunk)
|
||||
@ -2330,53 +2345,40 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* zc,
|
||||
const BYTE* const ip = (const BYTE*) src;
|
||||
size_t fhSize = 0;
|
||||
|
||||
if (zc->stage==ZSTDcs_created) return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */
|
||||
if (cctx->stage==ZSTDcs_created) return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */
|
||||
|
||||
if (frame && (zc->stage==ZSTDcs_init)) {
|
||||
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, zc->frameContentSize, zc->dictID);
|
||||
if (frame && (cctx->stage==ZSTDcs_init)) {
|
||||
fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, cctx->frameContentSize, cctx->dictID);
|
||||
if (ZSTD_isError(fhSize)) return fhSize;
|
||||
dstCapacity -= fhSize;
|
||||
dst = (char*)dst + fhSize;
|
||||
zc->stage = ZSTDcs_ongoing;
|
||||
cctx->stage = ZSTDcs_ongoing;
|
||||
}
|
||||
|
||||
/* Check if blocks follow each other */
|
||||
if (src != zc->nextSrc) {
|
||||
if (src != cctx->nextSrc) {
|
||||
/* not contiguous */
|
||||
ptrdiff_t const delta = zc->nextSrc - ip;
|
||||
zc->lowLimit = zc->dictLimit;
|
||||
zc->dictLimit = (U32)(zc->nextSrc - zc->base);
|
||||
zc->dictBase = zc->base;
|
||||
zc->base -= delta;
|
||||
zc->nextToUpdate = zc->dictLimit;
|
||||
if (zc->dictLimit - zc->lowLimit < HASH_READ_SIZE) zc->lowLimit = zc->dictLimit; /* too small extDict */
|
||||
ptrdiff_t const delta = cctx->nextSrc - ip;
|
||||
cctx->lowLimit = cctx->dictLimit;
|
||||
cctx->dictLimit = (U32)(cctx->nextSrc - cctx->base);
|
||||
cctx->dictBase = cctx->base;
|
||||
cctx->base -= delta;
|
||||
cctx->nextToUpdate = cctx->dictLimit;
|
||||
if (cctx->dictLimit - cctx->lowLimit < HASH_READ_SIZE) cctx->lowLimit = cctx->dictLimit; /* too small extDict */
|
||||
}
|
||||
|
||||
/* preemptive overflow correction */
|
||||
if (zc->lowLimit > (1<<30)) {
|
||||
U32 const btplus = (zc->params.cParams.strategy == ZSTD_btlazy2) | (zc->params.cParams.strategy == ZSTD_btopt);
|
||||
U32 const chainMask = (1 << (zc->params.cParams.chainLog - btplus)) - 1;
|
||||
U32 const newLowLimit = zc->lowLimit & chainMask; /* preserve position % chainSize */
|
||||
U32 const correction = zc->lowLimit - newLowLimit;
|
||||
ZSTD_reduceIndex(zc, correction);
|
||||
zc->base += correction;
|
||||
zc->dictBase += correction;
|
||||
zc->lowLimit = newLowLimit;
|
||||
zc->dictLimit -= correction;
|
||||
if (zc->nextToUpdate < correction) zc->nextToUpdate = 0;
|
||||
else zc->nextToUpdate -= correction;
|
||||
/* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */
|
||||
if ((ip+srcSize > cctx->dictBase + cctx->lowLimit) & (ip < cctx->dictBase + cctx->dictLimit)) {
|
||||
ptrdiff_t const highInputIdx = (ip + srcSize) - cctx->dictBase;
|
||||
U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)cctx->dictLimit) ? cctx->dictLimit : (U32)highInputIdx;
|
||||
cctx->lowLimit = lowLimitMax;
|
||||
}
|
||||
|
||||
/* if input and dictionary overlap : reduce dictionary (presumed modified by input) */
|
||||
if ((ip+srcSize > zc->dictBase + zc->lowLimit) && (ip < zc->dictBase + zc->dictLimit)) {
|
||||
zc->lowLimit = (U32)(ip + srcSize - zc->dictBase);
|
||||
if (zc->lowLimit > zc->dictLimit) zc->lowLimit = zc->dictLimit;
|
||||
}
|
||||
cctx->nextSrc = ip + srcSize;
|
||||
|
||||
zc->nextSrc = ip + srcSize;
|
||||
{ size_t const cSize = frame ?
|
||||
ZSTD_compress_generic (zc, dst, dstCapacity, src, srcSize, lastFrameChunk) :
|
||||
ZSTD_compressBlock_internal (zc, dst, dstCapacity, src, srcSize);
|
||||
ZSTD_compress_generic (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) :
|
||||
ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize);
|
||||
if (ZSTD_isError(cSize)) return cSize;
|
||||
return cSize + fhSize;
|
||||
}
|
||||
@ -2624,22 +2626,6 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
|
||||
}
|
||||
|
||||
|
||||
/*! ZSTD_compress_usingPreparedCCtx() :
|
||||
* Same as ZSTD_compress_usingDict, but using a reference context `preparedCCtx`, where dictionary has been loaded.
|
||||
* It avoids reloading the dictionary each time.
|
||||
* `preparedCCtx` must have been properly initialized using ZSTD_compressBegin_usingDict() or ZSTD_compressBegin_advanced().
|
||||
* Requires 2 contexts : 1 for reference (preparedCCtx) which will not be modified, and 1 to run the compression operation (cctx) */
|
||||
static size_t ZSTD_compress_usingPreparedCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
size_t const errorCode = ZSTD_copyCCtx(cctx, preparedCCtx);
|
||||
if (ZSTD_isError(errorCode)) return errorCode;
|
||||
|
||||
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
|
||||
}
|
||||
|
||||
|
||||
static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
@ -2735,9 +2721,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_pa
|
||||
ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel)
|
||||
{
|
||||
ZSTD_customMem const allocator = { NULL, NULL, NULL };
|
||||
ZSTD_parameters params;
|
||||
memset(¶ms, 0, sizeof(params));
|
||||
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
|
||||
ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize);
|
||||
params.fParams.contentSizeFlag = 1;
|
||||
return ZSTD_createCDict_advanced(dict, dictSize, params, allocator);
|
||||
}
|
||||
@ -2752,14 +2736,24 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*! ZSTD_compress_usingCDict() :
|
||||
* Compression using a digested Dictionary.
|
||||
* Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
|
||||
* Note that compression level is decided during dictionary creation */
|
||||
ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const ZSTD_CDict* cdict)
|
||||
{
|
||||
return ZSTD_compress_usingPreparedCCtx(cctx, cdict->refContext,
|
||||
dst, dstCapacity,
|
||||
src, srcSize);
|
||||
size_t const errorCode = ZSTD_copyCCtx(cctx, cdict->refContext);
|
||||
if (ZSTD_isError(errorCode)) return errorCode;
|
||||
|
||||
if (cdict->refContext->params.fParams.contentSizeFlag==1) {
|
||||
cctx->params.fParams.contentSizeFlag = 1;
|
||||
cctx->frameContentSize = srcSize;
|
||||
}
|
||||
|
||||
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -332,12 +332,13 @@ static dictItem ZDICT_analyzePos(
|
||||
} while (length >=MINMATCHLENGTH);
|
||||
|
||||
/* look backward */
|
||||
do {
|
||||
length = ZDICT_count(b + pos, b + suffix[start-1]);
|
||||
if (length >= LLIMIT) length = LLIMIT-1;
|
||||
length = MINMATCHLENGTH;
|
||||
while ((length >= MINMATCHLENGTH) & (start > 0)) {
|
||||
length = ZDICT_count(b + pos, b + suffix[start - 1]);
|
||||
if (length >= LLIMIT) length = LLIMIT - 1;
|
||||
lengthList[length]++;
|
||||
if (length >=MINMATCHLENGTH) start--;
|
||||
} while(length >= MINMATCHLENGTH);
|
||||
if (length >= MINMATCHLENGTH) start--;
|
||||
}
|
||||
|
||||
/* largest useful length */
|
||||
memset(cumulLength, 0, sizeof(cumulLength));
|
||||
@ -683,17 +684,10 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
size_t pos = 0, errorCode;
|
||||
size_t eSize = 0;
|
||||
size_t const totalSrcSize = ZDICT_totalSampleSize(fileSizes, nbFiles);
|
||||
size_t const averageSampleSize = totalSrcSize / nbFiles;
|
||||
size_t const averageSampleSize = totalSrcSize / (nbFiles + !nbFiles);
|
||||
BYTE* dstPtr = (BYTE*)dstBuffer;
|
||||
|
||||
/* init */
|
||||
if (offcodeMax>OFFCODE_MAX) { eSize = ERROR(dictionary_wrong); goto _cleanup; } /* too large dictionary */
|
||||
for (u=0; u<256; u++) countLit[u]=1; /* any character must be described */
|
||||
for (u=0; u<=offcodeMax; u++) offcodeCount[u]=1;
|
||||
for (u=0; u<=MaxML; u++) matchLengthCount[u]=1;
|
||||
for (u=0; u<=MaxLL; u++) litLengthCount[u]=1;
|
||||
repOffset[1] = repOffset[4] = repOffset[8] = 1;
|
||||
memset(bestRepOffset, 0, sizeof(bestRepOffset));
|
||||
esr.ref = ZSTD_createCCtx();
|
||||
esr.zc = ZSTD_createCCtx();
|
||||
esr.workPlace = malloc(ZSTD_BLOCKSIZE_ABSOLUTEMAX);
|
||||
@ -702,6 +696,13 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize,
|
||||
DISPLAYLEVEL(1, "Not enough memory");
|
||||
goto _cleanup;
|
||||
}
|
||||
if (offcodeMax>OFFCODE_MAX) { eSize = ERROR(dictionary_wrong); goto _cleanup; } /* too large dictionary */
|
||||
for (u=0; u<256; u++) countLit[u]=1; /* any character must be described */
|
||||
for (u=0; u<=offcodeMax; u++) offcodeCount[u]=1;
|
||||
for (u=0; u<=MaxML; u++) matchLengthCount[u]=1;
|
||||
for (u=0; u<=MaxLL; u++) litLengthCount[u]=1;
|
||||
repOffset[1] = repOffset[4] = repOffset[8] = 1;
|
||||
memset(bestRepOffset, 0, sizeof(bestRepOffset));
|
||||
if (compressionLevel==0) compressionLevel=g_compressionLevel_default;
|
||||
params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize);
|
||||
{ size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0);
|
||||
@ -920,7 +921,7 @@ size_t ZDICT_trainFromBuffer_unsafe(
|
||||
/* create dictionary */
|
||||
{ U32 dictContentSize = ZDICT_dictSize(dictList);
|
||||
if (dictContentSize < targetDictSize/2) {
|
||||
DISPLAYLEVEL(2, "! warning : created dictionary significantly smaller than requested (%u < %u) \n", dictContentSize, (U32)maxDictSize);
|
||||
DISPLAYLEVEL(2, "! warning : selected content significantly smaller than requested (%u < %u) \n", dictContentSize, (U32)maxDictSize);
|
||||
if (minRep > MINRATIO) {
|
||||
DISPLAYLEVEL(2, "! consider increasing selectivity to produce larger dictionary (-s%u) \n", selectivity+1);
|
||||
DISPLAYLEVEL(2, "! note : larger dictionaries are not necessarily better, test its efficiency on samples \n");
|
||||
|
||||
@ -55,7 +55,7 @@ extern "C" {
|
||||
/*====== Version ======*/
|
||||
#define ZSTD_VERSION_MAJOR 0
|
||||
#define ZSTD_VERSION_MINOR 8
|
||||
#define ZSTD_VERSION_RELEASE 0
|
||||
#define ZSTD_VERSION_RELEASE 1
|
||||
|
||||
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
|
||||
#define ZSTD_QUOTE(str) #str
|
||||
|
||||
@ -30,6 +30,7 @@
|
||||
#include <stdlib.h> /* malloc, free */
|
||||
#include <string.h> /* memset */
|
||||
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
||||
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
|
||||
|
||||
#include "mem.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
@ -69,6 +70,13 @@ static U32 g_compressibilityDefault = 50;
|
||||
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
|
||||
|
||||
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
|
||||
if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \
|
||||
{ g_time = clock(); DISPLAY(__VA_ARGS__); \
|
||||
if (g_displayLevel>=4) fflush(stdout); } }
|
||||
static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
|
||||
static clock_t g_time = 0;
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Exceptions
|
||||
@ -101,7 +109,7 @@ void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalP
|
||||
void BMK_SetNbIterations(unsigned nbLoops)
|
||||
{
|
||||
g_nbIterations = nbLoops;
|
||||
DISPLAYLEVEL(2, "- %i iterations -\n", g_nbIterations);
|
||||
DISPLAYLEVEL(3, "- test >= %i seconds per compression / decompression -\n", g_nbIterations);
|
||||
}
|
||||
|
||||
void BMK_SetBlockSize(size_t blockSize)
|
||||
@ -135,6 +143,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
const void* dictBuffer, size_t dictBufferSize)
|
||||
{
|
||||
size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
|
||||
size_t const avgSize = MIN(g_blockSize, (srcSize / nbFiles));
|
||||
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
|
||||
blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
|
||||
size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
|
||||
@ -182,13 +191,18 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
{ U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL);
|
||||
U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
|
||||
UTIL_time_t coolTime;
|
||||
U32 testNb;
|
||||
U64 const maxTime = (g_nbIterations * TIMELOOP_MICROSEC) + 100;
|
||||
U64 totalCTime=0, totalDTime=0;
|
||||
U32 cCompleted=0, dCompleted=0;
|
||||
# define NB_MARKS 4
|
||||
const char* const marks[NB_MARKS] = { " |", " /", " =", "\\" };
|
||||
U32 markNb = 0;
|
||||
size_t cSize = 0;
|
||||
double ratio = 0.;
|
||||
|
||||
UTIL_getTime(&coolTime);
|
||||
DISPLAYLEVEL(2, "\r%79s\r", "");
|
||||
for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) {
|
||||
while (!cCompleted | !dCompleted) {
|
||||
UTIL_time_t clockStart;
|
||||
U64 clockLoop = g_nbIterations ? TIMELOOP_MICROSEC : 1;
|
||||
|
||||
@ -200,16 +214,15 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
}
|
||||
|
||||
/* Compression */
|
||||
DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->\r", testNb, displayName, (U32)srcSize);
|
||||
memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */
|
||||
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
|
||||
if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */
|
||||
|
||||
UTIL_sleepMilli(1); /* give processor time to other processes */
|
||||
UTIL_waitForNextTick(ticksPerSecond);
|
||||
UTIL_getTime(&clockStart);
|
||||
|
||||
{ //size_t const refSrcSize = (nbBlocks == 1) ? srcSize : 0;
|
||||
//ZSTD_parameters const zparams = ZSTD_getParams(cLevel, refSrcSize, dictBufferSize);
|
||||
ZSTD_parameters const zparams = ZSTD_getParams(cLevel, blockSize, dictBufferSize);
|
||||
if (!cCompleted) { /* still some time to do compression tests */
|
||||
ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
|
||||
ZSTD_customMem const cmem = { NULL, NULL, NULL };
|
||||
U32 nbLoops = 0;
|
||||
ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem);
|
||||
@ -229,13 +242,16 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_freeCDict(cdict);
|
||||
{ U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
|
||||
if (clockSpan < fastestC*nbLoops) fastestC = clockSpan / nbLoops;
|
||||
totalCTime += clockSpan;
|
||||
cCompleted = totalCTime>maxTime;
|
||||
} }
|
||||
|
||||
cSize = 0;
|
||||
{ U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
|
||||
ratio = (double)srcSize / (double)cSize;
|
||||
DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
|
||||
testNb, displayName, (U32)srcSize, (U32)cSize, ratio,
|
||||
markNb = (markNb+1) % NB_MARKS;
|
||||
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
|
||||
marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
|
||||
(double)srcSize / fastestC );
|
||||
|
||||
(void)fastestD; (void)crcOrig; /* unused when decompression disabled */
|
||||
@ -247,7 +263,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
UTIL_waitForNextTick(ticksPerSecond);
|
||||
UTIL_getTime(&clockStart);
|
||||
|
||||
{ U32 nbLoops = 0;
|
||||
if (!dCompleted) {
|
||||
U32 nbLoops = 0;
|
||||
ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictBufferSize);
|
||||
if (!ddict) EXM_THROW(2, "ZSTD_createDDict() allocation failure");
|
||||
do {
|
||||
@ -270,10 +287,13 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
ZSTD_freeDDict(ddict);
|
||||
{ U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
|
||||
if (clockSpan < fastestD*nbLoops) fastestD = clockSpan / nbLoops;
|
||||
totalDTime += clockSpan;
|
||||
dCompleted = totalDTime>maxTime;
|
||||
} }
|
||||
|
||||
DISPLAYLEVEL(2, "%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
|
||||
testNb, displayName, (U32)srcSize, (U32)cSize, ratio,
|
||||
markNb = (markNb+1) % NB_MARKS;
|
||||
DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
|
||||
marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
|
||||
(double)srcSize / fastestC,
|
||||
(double)srcSize / fastestD );
|
||||
|
||||
@ -389,7 +409,7 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
|
||||
}
|
||||
f = fopen(fileNamesTable[n], "rb");
|
||||
if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
|
||||
DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]);
|
||||
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
|
||||
if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */
|
||||
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
|
||||
if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
|
||||
|
||||
@ -132,6 +132,16 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
|
||||
|
||||
/* Basic tests */
|
||||
DISPLAYLEVEL(4, "test%3i : ZSTD_getErrorName : ", testNb++);
|
||||
{ const char* errorString = ZSTD_getErrorName(0);
|
||||
DISPLAYLEVEL(4, "OK : %s \n", errorString);
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : ZSTD_getErrorName with wrong value : ", testNb++);
|
||||
{ const char* errorString = ZSTD_getErrorName(499);
|
||||
DISPLAYLEVEL(4, "OK : %s \n", errorString);
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, (U32)CNBuffSize);
|
||||
CHECKPLUS(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(CNBuffSize),
|
||||
CNBuffer, CNBuffSize, 1),
|
||||
|
||||
@ -365,13 +365,10 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i
|
||||
pos += len + 1;
|
||||
nbFiles++;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
} else {
|
||||
nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend);
|
||||
if (buf == NULL) return NULL;
|
||||
}
|
||||
}
|
||||
} }
|
||||
|
||||
if (nbFiles == 0) { free(buf); return NULL; }
|
||||
|
||||
|
||||
@ -128,7 +128,7 @@ Typical gains range from ~10% (at 64KB) to x5 better (at <1KB).
|
||||
benchmark file(s) using compression level #
|
||||
.TP
|
||||
.B \-i#
|
||||
iteration loops [1-9](default : 3), benchmark mode only
|
||||
minimum evaluation time, in seconds (default : 3s), benchmark mode only
|
||||
.TP
|
||||
.B \-B#
|
||||
cut file into independent blocks of size # (default: no block)
|
||||
|
||||
@ -165,7 +165,7 @@ static int usage_advanced(const char* programName)
|
||||
DISPLAY( "Benchmark arguments :\n");
|
||||
DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n");
|
||||
DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n");
|
||||
DISPLAY( " -i# : iteration loops [1-9](default : 3)\n");
|
||||
DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n");
|
||||
DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n");
|
||||
#endif
|
||||
return 0;
|
||||
@ -441,8 +441,8 @@ int main(int argCount, const char** argv)
|
||||
if (recursive) {
|
||||
fileNamesTable = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb);
|
||||
if (fileNamesTable) {
|
||||
unsigned i;
|
||||
for (i=0; i<fileNamesNb; i++) DISPLAYLEVEL(4, "%d %s\n", i, fileNamesTable[i]);
|
||||
unsigned u;
|
||||
for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%d %s\n", u, fileNamesTable[u]);
|
||||
free((void*)filenameTable);
|
||||
filenameTable = fileNamesTable;
|
||||
filenameIdx = fileNamesNb;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user