AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

code refactorization, for better clarity and reliability

Browse Source
This commit is contained in:
Yann Collet 2016-04-04 02:47:20 +02:00
parent 541cf1993f
commit 38461085d8
1 changed files with 73 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
programs/zbufftest.c
View File

@ -41,7 +41,7 @@
#include <string.h> /* strcmp */ #include <string.h> /* strcmp */
#include "mem.h" #include "mem.h"
#include "zbuff.h" #include "zbuff.h"
#include "zstd.h" /* ZSTD_compressBound() */ #include "zstd_static.h" /* ZSTD_compressBound(), ZSTD_maxCLevel() */
#include "datagen.h" /* RDG_genBuffer */ #include "datagen.h" /* RDG_genBuffer */
#include "xxhash.h" /* XXH64 */ #include "xxhash.h" /* XXH64 */
@ -106,15 +106,17 @@ static U32 FUZ_GetMilliSpan(U32 nTimeStart)
return nSpan; return nSpan;
} }
/*! FUZ_rand() :
@return : a 27 bits random value, from a 32-bits `seed`.
`seed` is also modified */
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) # define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
unsigned int FUZ_rand(unsigned int* src) unsigned int FUZ_rand(unsigned int* seedPtr)
{ {
U32 rand32 = *src; U32 rand32 = *seedPtr;
rand32 *= prime1; rand32 *= prime1;
rand32 += prime2; rand32 += prime2;
rand32 = FUZ_rotl32(rand32, 13); rand32 = FUZ_rotl32(rand32, 13);
*src = rand32; *seedPtr = rand32;
return rand32 >> 5; return rand32 >> 5;
} }
@ -220,6 +222,19 @@ static size_t findDiff(const void* buf1, const void* buf2, size_t max)
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ #define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
static size_t FUZ_rLogLength(U32* seed, U32 logLength)
{
size_t const lengthMask = ((size_t)1 << logLength) - 1;
return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
}
static size_t FUZ_randomLength(U32* seed, U32 maxLog)
{
U32 const logLength = FUZ_rand(seed) % maxLog;
return FUZ_rLogLength(seed, logLength);
}
static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility) static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
{ {
static const U32 maxSrcLog = 24; static const U32 maxSrcLog = 24;
@ -270,12 +285,12 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ ) { for ( ; (testNb <= nbTests) || (FUZ_GetMilliSpan(startTime) < g_testTime); testNb++ ) {
U32 lseed; U32 lseed;
const BYTE* srcBuffer; const BYTE* srcBuffer;
size_t sampleSize, sampleStart;
const BYTE* dict; const BYTE* dict;
size_t cSize, dictSize; U32 testLog;
size_t maxTestSize, totalTestSize, readSize, totalCSize, genSize, totalGenSize; size_t maxTestSize, dictSize;
size_t cSize, totalTestSize, totalCSize, totalGenSize;
size_t errorCode; size_t errorCode;
U32 sampleSizeLog, n, nbChunks; U32 n, nbChunks;
XXH64_CREATESTATE_STATIC(xxh64); XXH64_CREATESTATE_STATIC(xxh64);
U64 crcOrig; U64 crcOrig;
@ -285,102 +300,81 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
FUZ_rand(&coreSeed); FUZ_rand(&coreSeed);
lseed = coreSeed ^ prime1; lseed = coreSeed ^ prime1;
/* random state complete reset */ /* state total reset */
/* some problems only happen when states are re-used in a specific order */ /* some problems only happen when states are re-used in a specific order */
if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZBUFF_freeCCtx(zc); zc = ZBUFF_createCCtx(); } if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZBUFF_freeCCtx(zc); zc = ZBUFF_createCCtx(); }
if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZBUFF_freeDCtx(zd); zd = ZBUFF_createDCtx(); } if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZBUFF_freeDCtx(zd); zd = ZBUFF_createDCtx(); }
/* srcBuffer selection */ /* srcBuffer selection */
{ U32 buffNb = FUZ_rand(&lseed) & 0x7F; { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
if (buffNb & 7) buffNb=2; if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
else { else {
buffNb >>= 3; buffNb >>= 3;
if (buffNb & 7) { if (buffNb & 7) {
const U32 tnb[2] = { 1, 3 }; const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
buffNb = tnb[buffNb >> 3]; buffNb = tnb[buffNb >> 3];
} else { } else {
const U32 tnb[2] = { 0, 4 }; const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
buffNb = tnb[buffNb >> 3]; buffNb = tnb[buffNb >> 3];
} } } }
srcBuffer = cNoiseBuffer[buffNb]; srcBuffer = cNoiseBuffer[buffNb];
} }
/* Multi - segments compression test */ testLog = FUZ_rand(&lseed) % maxSrcLog;
maxTestSize = FUZ_rLogLength(&lseed, testLog);
/* random dictionary selection */
{ size_t dictStart;
U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
dict = srcBuffer + dictStart;
ZBUFF_compressInitDictionary(zc, dict, dictSize, cLevel);
}
/* multi-segments compression test */
XXH64_reset(xxh64, 0); XXH64_reset(xxh64, 0);
nbChunks = (FUZ_rand(&lseed) & 127) + 2; nbChunks = (FUZ_rand(&lseed) & 127) + 2;
sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog; for (n=0, cSize=0, totalTestSize=0 ; (n<nbChunks) && (totalTestSize < maxTestSize) ; n++) {
maxTestSize = (size_t)1 << sampleSizeLog; /* compress random chunk into random size dst buffer */
maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1); { size_t readChunkSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t dstBuffSize = MIN(cBufferSize - cSize, FUZ_randomLength(&lseed, maxSampleLog));
size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - readChunkSize);
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; errorCode = ZBUFF_compressContinue(zc, cBuffer+cSize, &dstBuffSize, srcBuffer+srcStart, &readChunkSize);
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
dict = srcBuffer + sampleStart;
dictSize = sampleSize;
ZBUFF_compressInitDictionary(zc, dict, dictSize, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1);
totalTestSize = 0;
cSize = 0;
for (n=0; (n<nbChunks) && (totalTestSize < maxTestSize); n++) {
{ size_t readChunkSize;
size_t dstBuffSize;
{ U32 const srcChSizeLog = FUZ_rand(&lseed) % maxSampleLog;
size_t const srcChSizeMask = ((size_t)1 << srcChSizeLog) - 1;
size_t const srcChSize = srcChSizeMask+1 + (FUZ_rand(&lseed) & srcChSizeMask);
readChunkSize = srcChSize;
}
{ U32 const dstChSizeLog = FUZ_rand(&lseed) % maxSampleLog;
size_t const dstChSizeMask = ((size_t)1 << dstChSizeLog) - 1;
size_t const dstChSize = dstChSizeMask+1 + (FUZ_rand(&lseed) & dstChSizeMask);
dstBuffSize = MIN (cBufferSize - cSize, dstChSize);
}
errorCode = ZBUFF_compressContinue(zc, cBuffer+cSize, &dstBuffSize, srcBuffer+sampleStart, &readChunkSize);
CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode)); CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode));
XXH64_update(xxh64, srcBuffer+sampleStart, readChunkSize); XXH64_update(xxh64, srcBuffer+srcStart, readChunkSize);
memcpy(copyBuffer+totalTestSize, srcBuffer+sampleStart, readChunkSize); memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, readChunkSize);
cSize += dstBuffSize; cSize += dstBuffSize;
totalTestSize += readChunkSize; totalTestSize += readChunkSize;
} }
/* random flush operation, to mess around */ /* random flush operation, to mess around */
if ((FUZ_rand(&lseed) & 15) == 0) { if ((FUZ_rand(&lseed) & 15) == 0) {
size_t dstBuffSize; size_t dstBuffSize = MIN(cBufferSize - cSize, FUZ_randomLength(&lseed, maxSampleLog));
{ U32 const dstChSizeLog = FUZ_rand(&lseed) % maxSampleLog; size_t const flushError = ZBUFF_compressFlush(zc, cBuffer+cSize, &dstBuffSize);
size_t const dstChSizeMask = ((size_t)1 << dstChSizeLog) - 1; CHECK (ZBUFF_isError(flushError), "flush error : %s", ZBUFF_getErrorName(flushError));
size_t const dstChSize = dstChSizeMask+1 + (FUZ_rand(&lseed) & dstChSizeMask);
dstBuffSize = MIN (cBufferSize - cSize, dstChSize);
}
{ size_t const flushError = ZBUFF_compressFlush(zc, cBuffer+cSize, &dstBuffSize);
CHECK (ZBUFF_isError(flushError), "flush error : %s", ZBUFF_getErrorName(flushError)); }
cSize += dstBuffSize; cSize += dstBuffSize;
} } } }
genSize = cBufferSize - cSize;
errorCode = ZBUFF_compressEnd(zc, cBuffer+cSize, &genSize); /* final frame epilogue */
CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode)); { size_t dstBuffSize = cBufferSize - cSize;
CHECK (errorCode != 0, "frame epilogue not fully consumed"); size_t const flushError = ZBUFF_compressEnd(zc, cBuffer+cSize, &dstBuffSize);
cSize += genSize; CHECK (ZBUFF_isError(flushError), "flush error : %s", ZBUFF_getErrorName(flushError));
cSize += dstBuffSize;
}
crcOrig = XXH64_digest(xxh64); crcOrig = XXH64_digest(xxh64);
/* multi - fragments decompression test */ /* multi - fragments decompression test */
ZBUFF_decompressInitDictionary(zd, dict, dictSize); ZBUFF_decompressInitDictionary(zd, dict, dictSize);
totalCSize = 0; for (totalCSize = 0, totalGenSize = 0 ; totalCSize < cSize ; ) {
totalGenSize = 0; size_t readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
while (totalCSize < cSize) { size_t dstBuffSize = MIN(dstBufferSize - totalGenSize, FUZ_randomLength(&lseed, maxSampleLog));
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; size_t const decompressError = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &dstBuffSize, cBuffer+totalCSize, &readCSrcSize);
sampleSize = (size_t)1 << sampleSizeLog; CHECK (ZBUFF_isError(decompressError), "decompression error : %s", ZBUFF_getErrorName(decompressError));
sampleSize += FUZ_rand(&lseed) & (sampleSize-1); totalGenSize += dstBuffSize;
readSize = sampleSize; totalCSize += readCSrcSize;
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; errorCode = decompressError; /* needed for != 0 last test */
sampleSize = (size_t)1 << sampleSizeLog;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
genSize = MIN(sampleSize, dstBufferSize - totalGenSize);
errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize);
CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode));
totalGenSize += genSize;
totalCSize += readSize;
} }
CHECK (errorCode != 0, "frame not fully decoded"); CHECK (errorCode != 0, "frame not fully decoded");
CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size") CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
@ -407,18 +401,12 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
totalCSize = 0; totalCSize = 0;
totalGenSize = 0; totalGenSize = 0;
while ( (totalCSize < cSize) && (totalGenSize < dstBufferSize) ) { while ( (totalCSize < cSize) && (totalGenSize < dstBufferSize) ) {
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; size_t readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
sampleSize = (size_t)1 << sampleSizeLog; size_t dstBuffSize = MIN(dstBufferSize - totalGenSize, FUZ_randomLength(&lseed, maxSampleLog));
sampleSize += FUZ_rand(&lseed) & (sampleSize-1); size_t const decompressError = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &dstBuffSize, cBuffer+totalCSize, &readCSrcSize);
readSize = sampleSize; if (ZBUFF_isError(decompressError)) break; /* error correctly detected */
sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog; totalGenSize += dstBuffSize;
sampleSize = (size_t)1 << sampleSizeLog; totalCSize += readCSrcSize;
sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
genSize = MIN(sampleSize, dstBufferSize - totalGenSize);
errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &genSize, cBuffer+totalCSize, &readSize);
if (ZBUFF_isError(errorCode)) break; /* error correctly detected */
totalGenSize += genSize;
totalCSize += readSize;
} } } }
DISPLAY("\r%u fuzzer tests completed \n", testNb); DISPLAY("\r%u fuzzer tests completed \n", testNb);