minor variable isolation
This commit is contained in:
parent
9bb87e5995
commit
27caf2afca
77
lib/zbuff.c
77
lib/zbuff.c
@ -128,18 +128,14 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, U64 pledgedSrcSize)
|
||||
{
|
||||
size_t neededInBuffSize;
|
||||
|
||||
ZSTD_adjustCParams(¶ms.cParams, pledgedSrcSize, dictSize);
|
||||
neededInBuffSize = (size_t)1 << params.cParams.windowLog;
|
||||
|
||||
/* allocate buffers */
|
||||
if (zbc->inBuffSize < neededInBuffSize) {
|
||||
zbc->inBuffSize = neededInBuffSize;
|
||||
free(zbc->inBuff); /* should not be necessary */
|
||||
zbc->inBuff = (char*)malloc(neededInBuffSize);
|
||||
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
|
||||
}
|
||||
{ size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
|
||||
if (zbc->inBuffSize < neededInBuffSize) {
|
||||
zbc->inBuffSize = neededInBuffSize;
|
||||
free(zbc->inBuff); /* should not be necessary */
|
||||
zbc->inBuff = (char*)malloc(neededInBuffSize);
|
||||
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
|
||||
} }
|
||||
zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, zbc->inBuffSize);
|
||||
if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) {
|
||||
zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1;
|
||||
@ -148,23 +144,24 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
|
||||
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
|
||||
}
|
||||
|
||||
zbc->outBuffContentSize = ZSTD_compressBegin_advanced(zbc->zc, dict, dictSize, params, pledgedSrcSize);
|
||||
if (ZSTD_isError(zbc->outBuffContentSize)) return zbc->outBuffContentSize;
|
||||
{ size_t const errorCode = ZSTD_compressBegin_advanced(zbc->zc, dict, dictSize, params, pledgedSrcSize);
|
||||
if (ZSTD_isError(errorCode)) return errorCode; }
|
||||
|
||||
zbc->inToCompress = 0;
|
||||
zbc->inBuffPos = 0;
|
||||
zbc->inBuffTarget = zbc->blockSize;
|
||||
zbc->outBuffFlushedSize = 0;
|
||||
zbc->stage = ZBUFFcs_flush; /* starts by flushing the header */
|
||||
zbc->stage = ZBUFFcs_load;
|
||||
return 0; /* ready to go */
|
||||
}
|
||||
|
||||
|
||||
ZSTDLIB_API size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel)
|
||||
size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel)
|
||||
{
|
||||
ZSTD_parameters params;
|
||||
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
|
||||
params.fParams.contentSizeFlag = 0;
|
||||
ZSTD_adjustCParams(¶ms.cParams, 0, dictSize);
|
||||
return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0);
|
||||
}
|
||||
|
||||
@ -190,11 +187,11 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
|
||||
{
|
||||
U32 notDone = 1;
|
||||
const char* const istart = (const char*)src;
|
||||
const char* ip = istart;
|
||||
const char* const iend = istart + *srcSizePtr;
|
||||
const char* ip = istart;
|
||||
char* const ostart = (char*)dst;
|
||||
char* op = ostart;
|
||||
char* const oend = ostart + *dstCapacityPtr;
|
||||
char* op = ostart;
|
||||
|
||||
while (notDone) {
|
||||
switch(zbc->stage)
|
||||
@ -203,19 +200,17 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
|
||||
|
||||
case ZBUFFcs_load:
|
||||
/* complete inBuffer */
|
||||
{
|
||||
size_t toLoad = zbc->inBuffTarget - zbc->inBuffPos;
|
||||
size_t loaded = ZBUFF_limitCopy(zbc->inBuff + zbc->inBuffPos, toLoad, ip, iend-ip);
|
||||
{ size_t const toLoad = zbc->inBuffTarget - zbc->inBuffPos;
|
||||
size_t const loaded = ZBUFF_limitCopy(zbc->inBuff + zbc->inBuffPos, toLoad, ip, iend-ip);
|
||||
zbc->inBuffPos += loaded;
|
||||
ip += loaded;
|
||||
if ( (zbc->inBuffPos==zbc->inToCompress) || (!flush && (toLoad != loaded)) ) {
|
||||
notDone = 0; break; /* not enough input to get a full block : stop there, wait for more */
|
||||
} }
|
||||
/* compress current block (note : this stage cannot be stopped in the middle) */
|
||||
{
|
||||
void* cDst;
|
||||
{ void* cDst;
|
||||
size_t cSize;
|
||||
size_t iSize = zbc->inBuffPos - zbc->inToCompress;
|
||||
size_t const iSize = zbc->inBuffPos - zbc->inToCompress;
|
||||
size_t oSize = oend-op;
|
||||
if (oSize >= ZSTD_compressBound(iSize))
|
||||
cDst = op; /* compress directly into output buffer (avoid flush stage) */
|
||||
@ -231,15 +226,13 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
|
||||
if (cDst == op) { op += cSize; break; } /* no need to flush */
|
||||
zbc->outBuffContentSize = cSize;
|
||||
zbc->outBuffFlushedSize = 0;
|
||||
zbc->stage = ZBUFFcs_flush;
|
||||
// break; /* flush stage follows */
|
||||
zbc->stage = ZBUFFcs_flush; /* continue to flush stage */
|
||||
}
|
||||
|
||||
case ZBUFFcs_flush:
|
||||
/* flush into dst */
|
||||
{
|
||||
size_t toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
|
||||
size_t flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
|
||||
{ size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
|
||||
size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
|
||||
op += flushed;
|
||||
zbc->outBuffFlushedSize += flushed;
|
||||
if (toFlush!=flushed) { notDone = 0; break; } /* not enough space within dst to store compressed block : stop there */
|
||||
@ -255,8 +248,7 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
|
||||
|
||||
*srcSizePtr = ip - istart;
|
||||
*dstCapacityPtr = op - ostart;
|
||||
{
|
||||
size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos;
|
||||
{ size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos;
|
||||
if (hintInSize==0) hintInSize = zbc->blockSize;
|
||||
return hintInSize;
|
||||
}
|
||||
@ -276,7 +268,7 @@ size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc,
|
||||
size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
|
||||
{
|
||||
size_t srcSize = 0;
|
||||
ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, &srcSize, &srcSize, 1); /* use a valid src address instead of NULL, as some sanitizer don't like it */
|
||||
ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, &srcSize, &srcSize, 1); /* use a valid src address instead of NULL */
|
||||
return zbc->outBuffContentSize - zbc->outBuffFlushedSize;
|
||||
}
|
||||
|
||||
@ -284,8 +276,8 @@ size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
|
||||
size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
|
||||
{
|
||||
BYTE* const ostart = (BYTE*)dst;
|
||||
BYTE* op = ostart;
|
||||
BYTE* const oend = ostart + *dstCapacityPtr;
|
||||
BYTE* op = ostart;
|
||||
size_t outSize = *dstCapacityPtr;
|
||||
size_t epilogueSize, remaining;
|
||||
ZBUFF_compressFlush(zbc, dst, &outSize); /* flush any remaining inBuff */
|
||||
@ -411,7 +403,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
|
||||
return headerSize;
|
||||
} }
|
||||
|
||||
/* Frame header provides buffer sizes */
|
||||
/* Frame header instruct buffer sizes */
|
||||
{ size_t const neededInSize = ZSTD_BLOCKSIZE_MAX; /* a block is never > ZSTD_BLOCKSIZE_MAX */
|
||||
if (zbc->inBuffSize < neededInSize) {
|
||||
free(zbc->inBuff);
|
||||
@ -419,8 +411,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
|
||||
zbc->inBuff = (char*)malloc(neededInSize);
|
||||
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
|
||||
} }
|
||||
{
|
||||
size_t const neededOutSize = (size_t)1 << zbc->fParams.windowLog;
|
||||
{ size_t const neededOutSize = (size_t)1 << zbc->fParams.windowLog;
|
||||
if (zbc->outBuffSize < neededOutSize) {
|
||||
free(zbc->outBuff);
|
||||
zbc->outBuffSize = neededOutSize;
|
||||
@ -430,8 +421,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
|
||||
zbc->stage = ZBUFFds_read;
|
||||
|
||||
case ZBUFFds_read:
|
||||
{
|
||||
size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
|
||||
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
|
||||
if (neededInSize==0) { /* end of frame */
|
||||
zbc->stage = ZBUFFds_init;
|
||||
notDone = 0;
|
||||
@ -454,8 +444,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
|
||||
}
|
||||
|
||||
case ZBUFFds_load:
|
||||
{
|
||||
size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
|
||||
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbc->zc);
|
||||
size_t const toLoad = neededInSize - zbc->inPos; /* should always be <= remaining space within inBuff */
|
||||
size_t loadedSize;
|
||||
if (toLoad > zbc->inBuffSize - zbc->inPos) return ERROR(corruption_detected); /* should never happen */
|
||||
@ -463,8 +452,8 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
|
||||
ip += loadedSize;
|
||||
zbc->inPos += loadedSize;
|
||||
if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */
|
||||
{
|
||||
size_t const decodedSize = ZSTD_decompressContinue(zbc->zc,
|
||||
/* decode loaded input */
|
||||
{ size_t const decodedSize = ZSTD_decompressContinue(zbc->zc,
|
||||
zbc->outBuff + zbc->outStart, zbc->outBuffSize - zbc->outStart,
|
||||
zbc->inBuff, neededInSize);
|
||||
if (ZSTD_isError(decodedSize)) return decodedSize;
|
||||
@ -475,8 +464,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
|
||||
// break; /* ZBUFFds_flush follows */
|
||||
} }
|
||||
case ZBUFFds_flush:
|
||||
{
|
||||
size_t const toFlushSize = zbc->outEnd - zbc->outStart;
|
||||
{ size_t const toFlushSize = zbc->outEnd - zbc->outStart;
|
||||
size_t const flushedSize = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outStart, toFlushSize);
|
||||
op += flushedSize;
|
||||
zbc->outStart += flushedSize;
|
||||
@ -496,8 +484,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc,
|
||||
/* result */
|
||||
*srcSizePtr = ip-istart;
|
||||
*dstCapacityPtr = op-ostart;
|
||||
{
|
||||
size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbc->zc);
|
||||
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbc->zc);
|
||||
if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */
|
||||
nextSrcSizeHint -= zbc->inPos; /* already loaded*/
|
||||
return nextSrcSizeHint;
|
||||
|
13
lib/zbuff.h
13
lib/zbuff.h
@ -31,11 +31,6 @@
|
||||
#ifndef ZSTD_BUFFERED_H
|
||||
#define ZSTD_BUFFERED_H
|
||||
|
||||
/* The objects defined into this file should be considered experimental.
|
||||
* They are not considered stable, as their prototype may change in the future.
|
||||
* You can use them for tests, provide feedback, or if you can endure risk of future changes.
|
||||
*/
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
@ -89,12 +84,12 @@ ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCap
|
||||
* *srcSizePtr and *dstCapacityPtr can be any size.
|
||||
* The function will report how many bytes were read or written within *srcSizePtr and *dstCapacityPtr.
|
||||
* Note that it may not consume the entire input, in which case it's up to the caller to present again remaining data.
|
||||
* The content of @dst will be overwritten (up to *dstCapacityPtr) at each call, so save its content if it matters or change @dst .
|
||||
* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each call, so save its content if it matters or change @dst .
|
||||
* @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency)
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* At any moment, it's possible to flush whatever data remains within buffer, using ZBUFF_compressFlush().
|
||||
* The nb of bytes written into @dst will be reported into *dstCapacityPtr.
|
||||
* The nb of bytes written into `dst` will be reported into *dstCapacityPtr.
|
||||
* Note that the function cannot output more than *dstCapacityPtr,
|
||||
* therefore, some content might still be left into internal buffer if *dstCapacityPtr is too small.
|
||||
* @return : nb of bytes still present into internal buffer (0 if it's empty)
|
||||
@ -139,13 +134,13 @@ ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx,
|
||||
* *srcSizePtr and *dstCapacityPtr can be any size.
|
||||
* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
|
||||
* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
|
||||
* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst.
|
||||
* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`.
|
||||
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency),
|
||||
* or 0 when a frame is completely decoded,
|
||||
* or an error code, which can be tested using ZBUFF_isError().
|
||||
*
|
||||
* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize()
|
||||
* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
|
||||
* output : ZBUFF_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
|
||||
* input : ZBUFF_recommendedDInSize == 128KB + 3;
|
||||
* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
|
||||
* *******************************************************************************/
|
||||
|
@ -208,7 +208,8 @@ size_t ZSTD_sizeofCCtx(ZSTD_compressionParameters cParams) /* hidden interface
|
||||
return ccsize; }
|
||||
}
|
||||
|
||||
|
||||
/*! ZSTD_resetCCtx_advanced() :
|
||||
note : 'params' is expected to be validated */
|
||||
static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
|
||||
ZSTD_parameters params)
|
||||
{ /* note : params considered validated here */
|
||||
@ -1200,8 +1201,8 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
|
||||
hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base);
|
||||
/* check immediate repcode */
|
||||
while (ip <= ilimit) {
|
||||
U32 current2 = (U32)(ip-base);
|
||||
const U32 repIndex2 = current2 - offset_2;
|
||||
U32 const current2 = (U32)(ip-base);
|
||||
U32 const repIndex2 = current2 - offset_2;
|
||||
const BYTE* repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2;
|
||||
if ( ((repIndex2 <= dictLimit-4) || (repIndex2 >= dictLimit))
|
||||
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
|
||||
@ -2233,18 +2234,12 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, si
|
||||
}
|
||||
|
||||
|
||||
/*! ZSTD_compressBegin_advanced() :
|
||||
/*! ZSTD_compressBegin_internal() :
|
||||
* @return : 0, or an error code */
|
||||
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc,
|
||||
static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, U64 pledgedSrcSize)
|
||||
{
|
||||
/* compression parameters verification and optimization */
|
||||
{ size_t const errorCode = ZSTD_checkCParams_advanced(params.cParams, pledgedSrcSize);
|
||||
if (ZSTD_isError(errorCode)) return errorCode; }
|
||||
|
||||
ZSTD_adjustCParams(¶ms.cParams, pledgedSrcSize, dictSize);
|
||||
|
||||
{ size_t const errorCode = ZSTD_resetCCtx_advanced(zc, params);
|
||||
if (ZSTD_isError(errorCode)) return errorCode; }
|
||||
|
||||
@ -2272,13 +2267,28 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc,
|
||||
}
|
||||
|
||||
|
||||
/*! ZSTD_compressBegin_advanced() :
|
||||
* @return : 0, or an error code */
|
||||
size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* zc,
|
||||
const void* dict, size_t dictSize,
|
||||
ZSTD_parameters params, U64 pledgedSrcSize)
|
||||
{
|
||||
/* compression parameters verification and optimization */
|
||||
{ size_t const errorCode = ZSTD_checkCParams_advanced(params.cParams, pledgedSrcSize);
|
||||
if (ZSTD_isError(errorCode)) return errorCode; }
|
||||
|
||||
return ZSTD_compressBegin_internal(zc, dict, dictSize, params, pledgedSrcSize);
|
||||
}
|
||||
|
||||
|
||||
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel)
|
||||
{
|
||||
ZSTD_parameters params;
|
||||
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
|
||||
params.fParams.contentSizeFlag = 0;
|
||||
ZSTD_adjustCParams(¶ms.cParams, 0, dictSize);
|
||||
ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", zc->base, compressionLevel);
|
||||
return ZSTD_compressBegin_advanced(zc, dict, dictSize, params, 0);
|
||||
return ZSTD_compressBegin_internal(zc, dict, dictSize, params, 0);
|
||||
}
|
||||
|
||||
size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel)
|
||||
@ -2342,7 +2352,7 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* ctx,
|
||||
BYTE* op = ostart;
|
||||
|
||||
/* Init */
|
||||
{ size_t const errorCode = ZSTD_compressBegin_advanced(ctx, dict, dictSize, params, srcSize);
|
||||
{ size_t const errorCode = ZSTD_compressBegin_internal(ctx, dict, dictSize, params, srcSize);
|
||||
if(ZSTD_isError(errorCode)) return errorCode; }
|
||||
|
||||
/* body (compression) */
|
||||
|
519
programs/result
Normal file
519
programs/result
Normal file
@ -0,0 +1,519 @@
|
||||
/*
|
||||
bench.c - Demo module to benchmark open-source compression algorithms
|
||||
Copyright (C) Yann Collet 2012-2015
|
||||
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- zstd source repository : https://github.com/Cyan4973/zstd
|
||||
- ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
|
||||
/* **************************************
|
||||
* Compiler Options
|
||||
****************************************/
|
||||
/* Disable some Visual warning messages */
|
||||
#ifdef _MSC_VER
|
||||
# define _CRT_SECURE_NO_WARNINGS /* fopen */
|
||||
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
|
||||
#endif
|
||||
|
||||
/* Unix Large Files support (>4GB) */
|
||||
#define _FILE_OFFSET_BITS 64
|
||||
#if (defined(__sun__) && (!defined(__LP64__))) /* Sun Solaris 32-bits requires specific definitions */
|
||||
# define _LARGEFILE_SOURCE
|
||||
#elif ! defined(__LP64__) /* No point defining Large file for 64 bit */
|
||||
# define _LARGEFILE64_SOURCE
|
||||
#endif
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Includes
|
||||
***************************************/
|
||||
#include <stdlib.h> /* malloc, free */
|
||||
#include <string.h> /* memset */
|
||||
#include <stdio.h> /* fprintf, fopen, ftello64 */
|
||||
#include <sys/types.h> /* stat64 */
|
||||
#include <sys/stat.h> /* stat64 */
|
||||
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
|
||||
|
||||
/* sleep : posix - windows - others */
|
||||
#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)))
|
||||
# include <unistd.h> /* sleep */
|
||||
# include <sys/resource.h> /* setpriority */
|
||||
# define BMK_sleep(s) sleep(s)
|
||||
# define HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
|
||||
#elif defined(_WIN32)
|
||||
# include <windows.h>
|
||||
# define BMK_sleep(s) Sleep(1000*s)
|
||||
# define HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS);
|
||||
#else
|
||||
# define BMK_sleep(s) /* disabled */
|
||||
# define HIGH_PRIORITY
|
||||
#endif
|
||||
|
||||
#include "mem.h"
|
||||
#include "zstd_static.h"
|
||||
#include "xxhash.h"
|
||||
#include "datagen.h" /* RDG_genBuffer */
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Compiler specifics
|
||||
***************************************/
|
||||
#if !defined(S_ISREG)
|
||||
# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
|
||||
#endif
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#define snprintf sprintf_s
|
||||
#endif
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Constants
|
||||
***************************************/
|
||||
#define NBLOOPS 3
|
||||
#define TIMELOOP_S 1
|
||||
#define ACTIVEPERIOD_S 70
|
||||
#define COOLPERIOD_S 10
|
||||
|
||||
#define KB *(1 <<10)
|
||||
#define MB *(1 <<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
|
||||
#define DEFAULT_CHUNKSIZE (4 MB)
|
||||
|
||||
static U32 g_compressibilityDefault = 50;
|
||||
|
||||
|
||||
/* *************************************
|
||||
* console display
|
||||
***************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Exceptions
|
||||
***************************************/
|
||||
#ifndef DEBUG
|
||||
# define DEBUG 0
|
||||
#endif
|
||||
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
|
||||
#define EXM_THROW(error, ...) \
|
||||
{ \
|
||||
DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
|
||||
DISPLAYLEVEL(1, "Error %i : ", error); \
|
||||
DISPLAYLEVEL(1, __VA_ARGS__); \
|
||||
DISPLAYLEVEL(1, "\n"); \
|
||||
exit(error); \
|
||||
}
|
||||
|
||||
|
||||
/* *************************************
|
||||
* Benchmark Parameters
|
||||
***************************************/
|
||||
static U32 g_nbIterations = NBLOOPS;
|
||||
static size_t g_blockSize = 0;
|
||||
|
||||
void BMK_SetNbIterations(unsigned nbLoops)
|
||||
{
|
||||
g_nbIterations = nbLoops;
|
||||
DISPLAY("- %i iterations -\n", g_nbIterations);
|
||||
}
|
||||
|
||||
void BMK_SetBlockSize(size_t blockSize)
|
||||
{
|
||||
g_blockSize = blockSize;
|
||||
DISPLAY("using blocks of size %u KB \n", (U32)(blockSize>>10));
|
||||
}
|
||||
|
||||
|
||||
/* ********************************************************
|
||||
* Private functions
|
||||
**********************************************************/
|
||||
static clock_t BMK_clockSpan( clock_t clockStart )
|
||||
{
|
||||
return clock() - clockStart; /* works even if overflow, span limited to <= ~30mn */
|
||||
}
|
||||
|
||||
|
||||
static U64 BMK_getFileSize(const char* infilename)
|
||||
{
|
||||
int r;
|
||||
#if defined(_MSC_VER)
|
||||
struct _stat64 statbuf;
|
||||
r = _stat64(infilename, &statbuf);
|
||||
#else
|
||||
struct stat statbuf;
|
||||
r = stat(infilename, &statbuf);
|
||||
#endif
|
||||
if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
|
||||
return (U64)statbuf.st_size;
|
||||
}
|
||||
|
||||
|
||||
/* ********************************************************
|
||||
* Bench functions
|
||||
**********************************************************/
|
||||
typedef struct
|
||||
{
|
||||
const char* srcPtr;
|
||||
size_t srcSize;
|
||||
char* cPtr;
|
||||
size_t cRoom;
|
||||
size_t cSize;
|
||||
char* resPtr;
|
||||
size_t resSize;
|
||||
} blockParam_t;
|
||||
|
||||
#define MIN(a,b) ((a)<(b) ? (a) : (b))
|
||||
#define MAX(a,b) ((a)>(b) ? (a) : (b))
|
||||
|
||||
static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
|
||||
const char* displayName, int cLevel,
|
||||
const size_t* fileSizes, U32 nbFiles,
|
||||
const void* dictBuffer, size_t dictBufferSize)
|
||||
{
|
||||
size_t const blockSize = (g_blockSize ? g_blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
|
||||
U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
|
||||
blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
|
||||
size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
|
||||
void* const compressedBuffer = malloc(maxCompressedSize);
|
||||
void* const resultBuffer = malloc(srcSize);
|
||||
ZSTD_CCtx* refCtx = ZSTD_createCCtx();
|
||||
ZSTD_CCtx* ctx = ZSTD_createCCtx();
|
||||
ZSTD_DCtx* refDCtx = ZSTD_createDCtx();
|
||||
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||
U32 nbBlocks;
|
||||
|
||||
/* checks */
|
||||
if (!compressedBuffer || !resultBuffer || !blockTable || !refCtx || !ctx || !refDCtx || !dctx)
|
||||
EXM_THROW(31, "not enough memory");
|
||||
|
||||
/* init */
|
||||
if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */
|
||||
HIGH_PRIORITY;
|
||||
|
||||
/* Init blockTable data */
|
||||
{ const char* srcPtr = (const char*)srcBuffer;
|
||||
char* cPtr = (char*)compressedBuffer;
|
||||
char* resPtr = (char*)resultBuffer;
|
||||
U32 fileNb;
|
||||
for (nbBlocks=0, fileNb=0; fileNb<nbFiles; fileNb++) {
|
||||
size_t remaining = fileSizes[fileNb];
|
||||
U32 const nbBlocksforThisFile = (U32)((remaining + (blockSize-1)) / blockSize);
|
||||
U32 const blockEnd = nbBlocks + nbBlocksforThisFile;
|
||||
for ( ; nbBlocks<blockEnd; nbBlocks++) {
|
||||
size_t const thisBlockSize = MIN(remaining, blockSize);
|
||||
blockTable[nbBlocks].srcPtr = srcPtr;
|
||||
blockTable[nbBlocks].cPtr = cPtr;
|
||||
blockTable[nbBlocks].resPtr = resPtr;
|
||||
blockTable[nbBlocks].srcSize = thisBlockSize;
|
||||
blockTable[nbBlocks].cRoom = ZSTD_compressBound(thisBlockSize);
|
||||
srcPtr += thisBlockSize;
|
||||
cPtr += blockTable[nbBlocks].cRoom;
|
||||
resPtr += thisBlockSize;
|
||||
remaining -= thisBlockSize;
|
||||
} } }
|
||||
|
||||
/* warmimg up memory */
|
||||
RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
|
||||
|
||||
/* Bench */
|
||||
{ double fastestC = 100000000., fastestD = 100000000.;
|
||||
U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
|
||||
clock_t coolTime = clock();
|
||||
U32 testNb;
|
||||
|
||||
DISPLAY("\r%79s\r", "");
|
||||
for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) {
|
||||
size_t cSize;
|
||||
double ratio = 0.;
|
||||
clock_t clockStart;
|
||||
clock_t const clockLoop = g_nbIterations ? TIMELOOP_S * CLOCKS_PER_SEC : 10;
|
||||
|
||||
/* overheat protection */
|
||||
if (BMK_clockSpan(coolTime) > ACTIVEPERIOD_S * CLOCKS_PER_SEC) {
|
||||
DISPLAY("\rcooling down ... \r");
|
||||
BMK_sleep(COOLPERIOD_S);
|
||||
coolTime = clock();
|
||||
}
|
||||
|
||||
/* Compression */
|
||||
DISPLAY("%2i-%-17.17s :%10u ->\r", testNb, displayName, (U32)srcSize);
|
||||
memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */
|
||||
|
||||
clockStart = clock();
|
||||
while (clock() == clockStart);
|
||||
clockStart = clock();
|
||||
{ U32 nbLoops;
|
||||
for (nbLoops = 0 ; BMK_clockSpan(clockStart) < clockLoop ; nbLoops++) {
|
||||
U32 blockNb;
|
||||
ZSTD_compressBegin_usingDict(refCtx, dictBuffer, dictBufferSize, cLevel);
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
|
||||
size_t const rSize = ZSTD_compress_usingPreparedCCtx(ctx, refCtx,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
|
||||
blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize);
|
||||
if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingPreparedCCtx() failed : %s", ZSTD_getErrorName(rSize));
|
||||
blockTable[blockNb].cSize = rSize;
|
||||
} }
|
||||
{ clock_t const clockSpan = BMK_clockSpan(clockStart);
|
||||
if ((double)clockSpan < fastestC*nbLoops) fastestC = (double)clockSpan / nbLoops;
|
||||
} }
|
||||
|
||||
cSize = 0;
|
||||
{ U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
|
||||
ratio = (double)srcSize / (double)cSize;
|
||||
DISPLAY("%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
|
||||
testNb, displayName, (U32)srcSize, (U32)cSize, ratio,
|
||||
(double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC) );
|
||||
|
||||
(void)fastestD; (void)crcOrig; /* unused when decompression disabled */
|
||||
#if 1
|
||||
/* Decompression */
|
||||
memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */
|
||||
|
||||
clockStart = clock();
|
||||
while (clock() == clockStart);
|
||||
clockStart = clock();
|
||||
|
||||
{ U32 nbLoops;
|
||||
for (nbLoops = 0 ; BMK_clockSpan(clockStart) < clockLoop ; nbLoops++) {
|
||||
U32 blockNb;
|
||||
ZSTD_decompressBegin_usingDict(refDCtx, dictBuffer, dictBufferSize);
|
||||
for (blockNb=0; blockNb<nbBlocks; blockNb++) {
|
||||
size_t const regenSize = ZSTD_decompress_usingPreparedDCtx(dctx, refDCtx,
|
||||
blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
|
||||
blockTable[blockNb].cPtr, blockTable[blockNb].cSize);
|
||||
if (ZSTD_isError(regenSize)) {
|
||||
DISPLAY("ZSTD_decompress_usingPreparedDCtx() failed on block %u : %s \n",
|
||||
blockNb, ZSTD_getErrorName(regenSize));
|
||||
clockStart -= clockLoop+1; /* force immediate test end */
|
||||
break;
|
||||
}
|
||||
blockTable[blockNb].resSize = regenSize;
|
||||
} }
|
||||
{ clock_t const clockSpan = BMK_clockSpan(clockStart);
|
||||
if ((double)clockSpan < fastestD*nbLoops) fastestD = (double)clockSpan / nbLoops;
|
||||
} }
|
||||
|
||||
DISPLAY("%2i-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
|
||||
testNb, displayName, (U32)srcSize, (U32)cSize, ratio,
|
||||
(double)srcSize / 1000000. / (fastestC / CLOCKS_PER_SEC),
|
||||
(double)srcSize / 1000000. / (fastestD / CLOCKS_PER_SEC) );
|
||||
|
||||
/* CRC Checking */
|
||||
{ U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
|
||||
if (crcOrig!=crcCheck) {
|
||||
size_t u;
|
||||
DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
|
||||
for (u=0; u<srcSize; u++) {
|
||||
if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u]) {
|
||||
U32 segNb, bNb, pos;
|
||||
size_t bacc = 0;
|
||||
DISPLAY("Decoding error at pos %u ", (U32)u);
|
||||
for (segNb = 0; segNb < nbBlocks; segNb++) {
|
||||
if (bacc + blockTable[segNb].srcSize > u) break;
|
||||
bacc += blockTable[segNb].srcSize;
|
||||
}
|
||||
pos = (U32)(u - bacc);
|
||||
bNb = pos / (128 KB);
|
||||
DISPLAY("(block %u, sub %u, pos %u) \n", segNb, bNb, pos);
|
||||
break;
|
||||
}
|
||||
if (u==srcSize-1) { /* should never happen */
|
||||
DISPLAY("no difference detected\n");
|
||||
} }
|
||||
break;
|
||||
} } /* CRC Checking */
|
||||
#endif
|
||||
} /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */
|
||||
DISPLAY("%2i#\n", cLevel);
|
||||
} /* Bench */
|
||||
|
||||
/* clean up */
|
||||
free(compressedBuffer);
|
||||
free(resultBuffer);
|
||||
ZSTD_freeCCtx(refCtx);
|
||||
ZSTD_freeCCtx(ctx);
|
||||
ZSTD_freeDCtx(refDCtx);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
static size_t BMK_findMaxMem(U64 requiredMem)
|
||||
{
|
||||
size_t const step = 64 MB;
|
||||
BYTE* testmem = NULL;
|
||||
|
||||
requiredMem = (((requiredMem >> 26) + 1) << 26);
|
||||
requiredMem += step;
|
||||
if (requiredMem > maxMemory) requiredMem = maxMemory;
|
||||
|
||||
do {
|
||||
testmem = (BYTE*)malloc((size_t)requiredMem);
|
||||
requiredMem -= step;
|
||||
} while (!testmem);
|
||||
|
||||
free(testmem);
|
||||
return (size_t)(requiredMem);
|
||||
}
|
||||
|
||||
static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
|
||||
const char* displayName, int cLevel,
|
||||
const size_t* fileSizes, unsigned nbFiles,
|
||||
const void* dictBuffer, size_t dictBufferSize)
|
||||
{
|
||||
if (cLevel < 0) { /* range mode : test all levels from 1 to l */
|
||||
int l;
|
||||
for (l=1; l <= -cLevel; l++) {
|
||||
BMK_benchMem(srcBuffer, benchedSize,
|
||||
displayName, l,
|
||||
fileSizes, nbFiles,
|
||||
dictBuffer, dictBufferSize);
|
||||
}
|
||||
return;
|
||||
}
|
||||
BMK_benchMem(srcBuffer, benchedSize,
|
||||
displayName, cLevel,
|
||||
fileSizes, nbFiles,
|
||||
dictBuffer, dictBufferSize);
|
||||
}
|
||||
|
||||
static U64 BMK_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles)
|
||||
{
|
||||
U64 total = 0;
|
||||
unsigned n;
|
||||
for (n=0; n<nbFiles; n++)
|
||||
total += BMK_getFileSize(fileNamesTable[n]);
|
||||
return total;
|
||||
}
|
||||
|
||||
/*! BMK_loadFiles() :
|
||||
Loads `buffer` with content of files listed within `fileNamesTable`.
|
||||
At most, fills `buffer` entirely */
|
||||
static void BMK_loadFiles(void* buffer, size_t bufferSize,
|
||||
size_t* fileSizes,
|
||||
const char** fileNamesTable, unsigned nbFiles)
|
||||
{
|
||||
size_t pos = 0;
|
||||
|
||||
unsigned n;
|
||||
for (n=0; n<nbFiles; n++) {
|
||||
U64 fileSize = BMK_getFileSize(fileNamesTable[n]);
|
||||
FILE* const f = fopen(fileNamesTable[n], "rb");
|
||||
if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
|
||||
DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]);
|
||||
if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */
|
||||
{ size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
|
||||
if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
|
||||
pos += readSize; }
|
||||
fileSizes[n] = (size_t)fileSize;
|
||||
fclose(f);
|
||||
}
|
||||
}
|
||||
|
||||
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
|
||||
const char* dictFileName, int cLevel)
|
||||
{
|
||||
void* srcBuffer;
|
||||
size_t benchedSize;
|
||||
void* dictBuffer = NULL;
|
||||
size_t dictBufferSize = 0;
|
||||
size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
|
||||
U64 totalSizeToLoad = BMK_getTotalFileSize(fileNamesTable, nbFiles);
|
||||
char mfName[20] = {0};
|
||||
const char* displayName = NULL;
|
||||
|
||||
if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");
|
||||
|
||||
/* Load dictionary */
|
||||
if (dictFileName != NULL) {
|
||||
U64 dictFileSize = BMK_getFileSize(dictFileName);
|
||||
if (dictFileSize > 64 MB) EXM_THROW(10, "dictionary file %s too large", dictFileName);
|
||||
dictBufferSize = (size_t)dictFileSize;
|
||||
dictBuffer = malloc(dictBufferSize);
|
||||
if (dictBuffer==NULL) EXM_THROW(11, "not enough memory for dictionary (%u bytes)", (U32)dictBufferSize);
|
||||
BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1);
|
||||
}
|
||||
|
||||
/* Memory allocation & restrictions */
|
||||
benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
|
||||
if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
|
||||
if (benchedSize < totalSizeToLoad)
|
||||
DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
|
||||
srcBuffer = malloc(benchedSize);
|
||||
if (!srcBuffer) EXM_THROW(12, "not enough memory");
|
||||
|
||||
/* Load input buffer */
|
||||
BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
|
||||
|
||||
/* Bench */
|
||||
snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
|
||||
if (nbFiles > 1) displayName = mfName;
|
||||
else displayName = fileNamesTable[0];
|
||||
|
||||
BMK_benchCLevel(srcBuffer, benchedSize,
|
||||
displayName, cLevel,
|
||||
fileSizes, nbFiles,
|
||||
dictBuffer, dictBufferSize);
|
||||
|
||||
/* clean up */
|
||||
free(srcBuffer);
|
||||
free(dictBuffer);
|
||||
free(fileSizes);
|
||||
}
|
||||
|
||||
|
||||
static void BMK_syntheticTest(int cLevel, double compressibility)
|
||||
{
|
||||
char name[20] = {0};
|
||||
size_t benchedSize = 10000000;
|
||||
void* srcBuffer = malloc(benchedSize);
|
||||
|
||||
/* Memory allocation */
|
||||
if (!srcBuffer) EXM_THROW(21, "not enough memory");
|
||||
|
||||
/* Fill input buffer */
|
||||
RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
|
||||
|
||||
/* Bench */
|
||||
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
|
||||
BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, &benchedSize, 1, NULL, 0);
|
||||
|
||||
/* clean up */
|
||||
free(srcBuffer);
|
||||
}
|
||||
|
||||
|
||||
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
|
||||
const char* dictFileName, int cLevel)
|
||||
{
|
||||
double const compressibility = (double)g_compressibilityDefault / 100;
|
||||
|
||||
if (nbFiles == 0)
|
||||
BMK_syntheticTest(cLevel, compressibility);
|
||||
else
|
||||
BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel);
|
||||
return 0;
|
||||
}
|
||||
|
BIN
programs/tmp.mgz
Normal file
BIN
programs/tmp.mgz
Normal file
Binary file not shown.
BIN
programs/tmp.zstd
Normal file
BIN
programs/tmp.zstd
Normal file
Binary file not shown.
@ -1,6 +1,6 @@
|
||||
/*
|
||||
Fuzzer test tool for zstd_buffered
|
||||
Copyright (C) Yann Collet 2105
|
||||
Copyright (C) Yann Collet 2015-2016
|
||||
|
||||
GPL v2 License
|
||||
|
||||
@ -19,11 +19,10 @@
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- ZSTD source repository : https://github.com/Cyan4973/zstd
|
||||
- ZSTD public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
- ZSTD homepage : https://www.zstd.net/
|
||||
*/
|
||||
|
||||
/**************************************
|
||||
/*-************************************
|
||||
* Compiler specific
|
||||
**************************************/
|
||||
#ifdef _MSC_VER /* Visual Studio */
|
||||
@ -33,7 +32,7 @@
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
/*-************************************
|
||||
* Includes
|
||||
**************************************/
|
||||
#include <stdlib.h> /* free */
|
||||
@ -47,8 +46,8 @@
|
||||
#include "xxhash.h" /* XXH64 */
|
||||
|
||||
|
||||
/**************************************
|
||||
Constants
|
||||
/*-************************************
|
||||
* Constants
|
||||
**************************************/
|
||||
#ifndef ZSTD_VERSION
|
||||
# define ZSTD_VERSION ""
|
||||
@ -66,7 +65,7 @@ static const U32 prime2 = 2246822519U;
|
||||
|
||||
|
||||
|
||||
/**************************************
|
||||
/*-************************************
|
||||
* Display Macros
|
||||
**************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
@ -83,7 +82,7 @@ static U32 g_displayTime = 0;
|
||||
static U32 g_testTime = 0;
|
||||
|
||||
|
||||
/*********************************************************
|
||||
/*-*******************************************************
|
||||
* Fuzzer functions
|
||||
*********************************************************/
|
||||
#define MAX(a,b) ((a)>(b)?(a):(b))
|
||||
@ -149,8 +148,7 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
CNBuffer = malloc(CNBufferSize);
|
||||
compressedBuffer = malloc(compressedBufferSize);
|
||||
decodedBuffer = malloc(decodedBufferSize);
|
||||
if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd)
|
||||
{
|
||||
if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) {
|
||||
DISPLAY("Not enough memory, aborting\n");
|
||||
goto _output_error;
|
||||
}
|
||||
@ -183,11 +181,9 @@ static int basicUnitTests(U32 seed, double compressibility)
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
|
||||
/* check regenerated data is byte exact */
|
||||
{
|
||||
size_t i;
|
||||
{ size_t i;
|
||||
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
|
||||
for (i=0; i<CNBufferSize; i++)
|
||||
{
|
||||
for (i=0; i<CNBufferSize; i++) {
|
||||
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
|
||||
}
|
||||
DISPLAYLEVEL(4, "OK \n");
|
||||
@ -213,8 +209,7 @@ static size_t findDiff(const void* buf1, const void* buf2, size_t max)
|
||||
const BYTE* b1 = (const BYTE*)buf1;
|
||||
const BYTE* b2 = (const BYTE*)buf2;
|
||||
size_t i;
|
||||
for (i=0; i<max; i++)
|
||||
{
|
||||
for (i=0; i<max; i++) {
|
||||
if (b1[i] != b2[i]) break;
|
||||
}
|
||||
return i;
|
||||
@ -228,7 +223,7 @@ static size_t findDiff(const void* buf1, const void* buf2, size_t max)
|
||||
static const U32 maxSrcLog = 24;
|
||||
static const U32 maxSampleLog = 19;
|
||||
|
||||
int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
|
||||
static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
|
||||
{
|
||||
BYTE* cNoiseBuffer[5];
|
||||
BYTE* srcBuffer;
|
||||
@ -276,8 +271,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
|
||||
FUZ_rand(&coreSeed);
|
||||
|
||||
/* test loop */
|
||||
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ )
|
||||
{
|
||||
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ ) {
|
||||
size_t sampleSize, sampleStart;
|
||||
const BYTE* dict;
|
||||
size_t cSize, dictSize;
|
||||
@ -294,20 +288,15 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
|
||||
lseed = coreSeed ^ prime1;
|
||||
buffNb = FUZ_rand(&lseed) & 127;
|
||||
if (buffNb & 7) buffNb=2; /* select buffer */
|
||||
else
|
||||
{
|
||||
else {
|
||||
buffNb >>= 3;
|
||||
if (buffNb & 7)
|
||||
{
|
||||
if (buffNb & 7) {
|
||||
const U32 tnb[2] = { 1, 3 };
|
||||
buffNb = tnb[buffNb >> 3];
|
||||
}
|
||||
else
|
||||
{
|
||||
} else {
|
||||
const U32 tnb[2] = { 0, 4 };
|
||||
buffNb = tnb[buffNb >> 3];
|
||||
}
|
||||
}
|
||||
} }
|
||||
srcBuffer = cNoiseBuffer[buffNb];
|
||||
|
||||
/* Multi - segments compression test */
|
||||
@ -327,8 +316,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
|
||||
|
||||
totalTestSize = 0;
|
||||
cSize = 0;
|
||||
for (n=0; n<nbChunks; n++)
|
||||
{
|
||||
for (n=0; n<nbChunks; n++) {
|
||||
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
|
||||
sampleSize = (size_t)1 << sampleSizeLog;
|
||||
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
|
||||
@ -349,8 +337,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
|
||||
cSize += genSize;
|
||||
totalTestSize += readSize;
|
||||
|
||||
if ((FUZ_rand(&lseed) & 15) == 0)
|
||||
{
|
||||
if ((FUZ_rand(&lseed) & 15) == 0) {
|
||||
/* add a few random flushes operations, to mess around */
|
||||
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
|
||||
sampleSize = (size_t)1 << sampleSizeLog;
|
||||
@ -374,8 +361,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
|
||||
ZBUFF_decompressInitDictionary(zd, dict, dictSize);
|
||||
totalCSize = 0;
|
||||
totalGenSize = 0;
|
||||
while (totalCSize < cSize)
|
||||
{
|
||||
while (totalCSize < cSize) {
|
||||
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
|
||||
sampleSize = (size_t)1 << sampleSizeLog;
|
||||
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
|
||||
@ -399,8 +385,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
|
||||
/* noisy/erroneous src decompression test */
|
||||
/* add some noise */
|
||||
nbChunks = (FUZ_rand(&lseed) & 7) + 2;
|
||||
for (n=0; n<nbChunks; n++)
|
||||
{
|
||||
for (n=0; n<nbChunks; n++) {
|
||||
size_t cStart;
|
||||
|
||||
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
|
||||
@ -417,8 +402,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
|
||||
ZBUFF_decompressInit(zd);
|
||||
totalCSize = 0;
|
||||
totalGenSize = 0;
|
||||
while ( (totalCSize < cSize) && (totalGenSize < dstBufferSize) )
|
||||
{
|
||||
while ( (totalCSize < cSize) && (totalGenSize < dstBufferSize) ) {
|
||||
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
|
||||
sampleSize = (size_t)1 << sampleSizeLog;
|
||||
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
|
||||
@ -431,8 +415,7 @@ int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibilit
|
||||
if (ZBUFF_isError(errorCode)) break; /* error correctly detected */
|
||||
totalGenSize += genSize;
|
||||
totalCSize += readSize;
|
||||
}
|
||||
}
|
||||
} }
|
||||
DISPLAY("\r%u fuzzer tests completed \n", testNb);
|
||||
|
||||
_cleanup:
|
||||
@ -454,7 +437,7 @@ _output_error:
|
||||
}
|
||||
|
||||
|
||||
/*********************************************************
|
||||
/*-*******************************************************
|
||||
* Command line
|
||||
*********************************************************/
|
||||
int FUZ_usage(char* programName)
|
||||
@ -488,19 +471,16 @@ int main(int argc, char** argv)
|
||||
|
||||
/* Check command line */
|
||||
programName = argv[0];
|
||||
for(argNb=1; argNb<argc; argNb++)
|
||||
{
|
||||
for(argNb=1; argNb<argc; argNb++) {
|
||||
char* argument = argv[argNb];
|
||||
|
||||
if(!argument) continue; /* Protection if argument empty */
|
||||
|
||||
/* Handle commands. Aggregated commands are allowed */
|
||||
if (argument[0]=='-')
|
||||
{
|
||||
if (argument[0]=='-') {
|
||||
argument++;
|
||||
|
||||
while (*argument!=0)
|
||||
{
|
||||
while (*argument!=0) {
|
||||
switch(*argument)
|
||||
{
|
||||
case 'h':
|
||||
@ -521,8 +501,7 @@ int main(int argc, char** argv)
|
||||
case 'i':
|
||||
argument++;
|
||||
nbTests=0; g_testTime=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
nbTests *= 10;
|
||||
nbTests += *argument - '0';
|
||||
argument++;
|
||||
@ -532,8 +511,7 @@ int main(int argc, char** argv)
|
||||
case 'T':
|
||||
argument++;
|
||||
nbTests=0; g_testTime=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
g_testTime *= 10;
|
||||
g_testTime += *argument - '0';
|
||||
argument++;
|
||||
@ -547,8 +525,7 @@ int main(int argc, char** argv)
|
||||
argument++;
|
||||
seed=0;
|
||||
seedset=1;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
seed *= 10;
|
||||
seed += *argument - '0';
|
||||
argument++;
|
||||
@ -558,8 +535,7 @@ int main(int argc, char** argv)
|
||||
case 't':
|
||||
argument++;
|
||||
testNb=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
testNb *= 10;
|
||||
testNb += *argument - '0';
|
||||
argument++;
|
||||
@ -569,8 +545,7 @@ int main(int argc, char** argv)
|
||||
case 'P': /* compressibility % */
|
||||
argument++;
|
||||
proba=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
proba *= 10;
|
||||
proba += *argument - '0';
|
||||
argument++;
|
||||
@ -582,9 +557,7 @@ int main(int argc, char** argv)
|
||||
default:
|
||||
return FUZ_usage(programName);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} } }
|
||||
|
||||
/* Get Seed */
|
||||
DISPLAY("Starting zstd_buffered tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION);
|
||||
@ -598,8 +571,8 @@ int main(int argc, char** argv)
|
||||
if (testNb==0) result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */
|
||||
if (!result)
|
||||
result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
|
||||
if (mainPause)
|
||||
{
|
||||
|
||||
if (mainPause) {
|
||||
int unused;
|
||||
DISPLAY("Press Enter \n");
|
||||
unused = getchar();
|
||||
|
Loading…
Reference in New Issue
Block a user