/* Fuzzer test tool for zstd streaming API Copyright (C) Yann Collet 2016 GPL v2 License This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : - ZSTD homepage : https://www.zstd.net/ */ /*-************************************ * Compiler specific **************************************/ #ifdef _MSC_VER /* Visual Studio */ # define _CRT_SECURE_NO_WARNINGS /* fgets */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ #endif /*-************************************ * Includes **************************************/ #include /* free */ #include /* fgets, sscanf */ #include /* timeb */ #include /* strcmp */ #include "mem.h" #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */ #include "zstd.h" /* ZSTD_compressBound */ #include "datagen.h" /* RDG_genBuffer */ #define XXH_STATIC_LINKING_ONLY #include "xxhash.h" /* XXH64_* */ /*-************************************ * Constants **************************************/ #define KB *(1U<<10) #define MB *(1U<<20) #define GB *(1U<<30) static const U32 nbTestsDefault = 10000; #define COMPRESSIBLE_NOISE_LENGTH (10 MB) #define FUZ_COMPRESSIBILITY_DEFAULT 50 static const U32 prime1 = 2654435761U; static const U32 prime2 = 2246822519U; /*-************************************ * Display Macros **************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } static U32 g_displayLevel = 2; #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ if ((FUZ_GetMilliSpan(g_displayTime) > g_refreshRate) || (g_displayLevel>=4)) \ { g_displayTime = FUZ_GetMilliStart(); DISPLAY(__VA_ARGS__); \ if (g_displayLevel>=4) fflush(stdout); } } static const U32 g_refreshRate = 150; static U32 g_displayTime = 0; static U32 g_testTime = 0; /*-******************************************************* * Fuzzer functions *********************************************************/ #define MAX(a,b) ((a)>(b)?(a):(b)) static U32 FUZ_GetMilliStart(void) { struct timeb tb; U32 nCount; ftime( &tb ); nCount = (U32) (((tb.time & 0xFFFFF) * 1000) + tb.millitm); return nCount; } static U32 FUZ_GetMilliSpan(U32 nTimeStart) { U32 const nCurrent = FUZ_GetMilliStart(); U32 nSpan = nCurrent - nTimeStart; if (nTimeStart > nCurrent) nSpan += 0x100000 * 1000; return nSpan; } /*! FUZ_rand() : @return : a 27 bits random value, from a 32-bits `seed`. `seed` is also modified */ # define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) unsigned int FUZ_rand(unsigned int* seedPtr) { U32 rand32 = *seedPtr; rand32 *= prime1; rand32 += prime2; rand32 = FUZ_rotl32(rand32, 13); *seedPtr = rand32; return rand32 >> 5; } /* static unsigned FUZ_highbit32(U32 v32) { unsigned nbBits = 0; if (v32==0) return 0; for ( ; v32 ; v32>>=1) nbBits++; return nbBits; } */ static void* allocFunction(void* opaque, size_t size) { void* address = malloc(size); (void)opaque; return address; } static void freeFunction(void* opaque, void* address) { (void)opaque; free(address); } static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem) { int testResult = 0; size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH; void* CNBuffer = malloc(CNBufferSize); size_t const skippableFrameSize = 11; size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH); void* compressedBuffer = malloc(compressedBufferSize); size_t const decodedBufferSize = CNBufferSize; void* decodedBuffer = malloc(decodedBufferSize); size_t cSize; U32 testNb=0; ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem); ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem); ZSTD_rCursor rCursor; ZSTD_wCursor wCursor; /* Create compressible test buffer */ if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) { DISPLAY("Not enough memory, aborting\n"); goto _output_error; } RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed); /* generate skippable frame */ MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START); MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize); cSize = skippableFrameSize + 8; /* Basic compression test */ DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZSTD_initCStream_usingDict(zc, CNBuffer, 128 KB, 1); wCursor.ptr = (char*)(compressedBuffer)+cSize; wCursor.size = compressedBufferSize; wCursor.nbBytesWritten = 0; rCursor.ptr = CNBuffer; rCursor.size = CNBufferSize; { size_t const r = ZSTD_compressStream(zc, &wCursor, &rCursor); if (ZSTD_isError(r)) goto _output_error; } if (rCursor.size != 0) goto _output_error; /* entire input should be consumed */ { size_t const r = ZSTD_endStream(zc, &wCursor); if (r != 0) goto _output_error; } /*< error, or some data not flushed */ cSize += wCursor.nbBytesWritten; DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); /* skippable frame test */ DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++); ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); rCursor.ptr = compressedBuffer; rCursor.size = cSize; wCursor.ptr = decodedBuffer; wCursor.size = CNBufferSize; wCursor.nbBytesWritten = 0; { size_t const r = ZSTD_decompressStream(zd, &wCursor, &rCursor); if (r != 0) goto _output_error; } if (wCursor.nbBytesWritten != 0) goto _output_error; /* skippable frame len is 0 */ DISPLAYLEVEL(4, "OK \n"); /* Basic decompression test */ DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB); { size_t const r = ZSTD_decompressStream(zd, &wCursor, &rCursor); if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ if (wCursor.nbBytesWritten != CNBufferSize) goto _output_error; /* should regenerate the same amount */ if (rCursor.size != 0) goto _output_error; /* should have read the entire frame */ DISPLAYLEVEL(4, "OK \n"); /* check regenerated data is byte exact */ DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); { size_t i; for (i=0; i "); DISPLAY(__VA_ARGS__); \ DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility) { static const U32 maxSrcLog = 24; static const U32 maxSampleLog = 19; BYTE* cNoiseBuffer[5]; size_t srcBufferSize = (size_t)1<= testNb) DISPLAYUPDATE(2, "/%6u ", nbTests); FUZ_rand(&coreSeed); lseed = coreSeed ^ prime1; /* states full reset (unsynchronized) */ /* some issues only happen when reusing states in a specific sequence of parameters */ if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZSTD_freeCStream(zc); zc = ZSTD_createCStream(); } if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZSTD_freeDStream(zd); zd = ZSTD_createDStream(); } /* srcBuffer selection [0-4] */ { U32 buffNb = FUZ_rand(&lseed) & 0x7F; if (buffNb & 7) buffNb=2; /* most common : compressible (P) */ else { buffNb >>= 3; if (buffNb & 7) { const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */ buffNb = tnb[buffNb >> 3]; } else { const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */ buffNb = tnb[buffNb >> 3]; } } srcBuffer = cNoiseBuffer[buffNb]; } /* compression init */ { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1; maxTestSize = FUZ_rLogLength(&lseed, testLog); dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0; /* random dictionary selection */ { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); dict = srcBuffer + dictStart; } { ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictSize); params.fParams.checksumFlag = FUZ_rand(&lseed) & 1; params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; { size_t const initError = ZSTD_initCStream_advanced(zc, dict, dictSize, params, 0); CHECK (ZSTD_isError(initError),"init error : %s", ZSTD_getErrorName(initError)); } } } /* multi-segments compression test */ XXH64_reset(&xxhState, 0); nbChunks = (FUZ_rand(&lseed) & 127) + 2; { ZSTD_wCursor wCursor = { cBuffer, cBufferSize, 0 } ; for (n=0, cSize=0, totalTestSize=0 ; (n= remainingToFlush); wCursor.size = adjustedDstSize; remainingToFlush = ZSTD_endStream(zc, &wCursor); CHECK (ZSTD_isError(remainingToFlush), "flush error : %s", ZSTD_getErrorName(remainingToFlush)); CHECK (enoughDstSize && remainingToFlush, "ZSTD_endStream() not fully flushed (%u remaining), but enough space available", (U32)remainingToFlush); } } crcOrig = XXH64_digest(&xxhState); cSize = wCursor.nbBytesWritten; } /* multi - fragments decompression test */ ZSTD_initDStream_usingDict(zd, dict, dictSize); { size_t decompressionResult = 1; ZSTD_rCursor rCursor = { cBuffer, cSize }; ZSTD_wCursor wCursor = { dstBuffer, dstBufferSize, 0 }; for (totalCSize = 0, totalGenSize = 0 ; decompressionResult ; ) { size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize); rCursor.size = readCSrcSize; wCursor.size = dstBuffSize; decompressionResult = ZSTD_decompressStream(zd, &wCursor, &rCursor); CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult)); totalCSize += readCSrcSize - rCursor.size; } CHECK (decompressionResult != 0, "frame not fully decoded"); CHECK (wCursor.nbBytesWritten != totalTestSize, "decompressed data : wrong size") CHECK (totalCSize != cSize, "compressed data should be fully read") { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); CHECK (crcDest!=crcOrig, "decompressed data corrupted"); } } /*===== noisy/erroneous src decompression test =====*/ /* add some noise */ { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2; U32 nn; for (nn=0; nn='0') && (*argument<='9')) { nbTests *= 10; nbTests += *argument - '0'; argument++; } break; case 'T': argument++; nbTests=0; g_testTime=0; while ((*argument>='0') && (*argument<='9')) { g_testTime *= 10; g_testTime += *argument - '0'; argument++; } if (*argument=='m') g_testTime *=60, argument++; if (*argument=='n') argument++; g_testTime *= 1000; break; case 's': argument++; seed=0; seedset=1; while ((*argument>='0') && (*argument<='9')) { seed *= 10; seed += *argument - '0'; argument++; } break; case 't': argument++; testNb=0; while ((*argument>='0') && (*argument<='9')) { testNb *= 10; testNb += *argument - '0'; argument++; } break; case 'P': /* compressibility % */ argument++; proba=0; while ((*argument>='0') && (*argument<='9')) { proba *= 10; proba += *argument - '0'; argument++; } if (proba<0) proba=0; if (proba>100) proba=100; break; default: return FUZ_usage(programName); } } } } /* for(argNb=1; argNb