diff --git a/lib/zstd_buffered.h b/lib/zstd_buffered.h index f79982a1..ecbc1df5 100644 --- a/lib/zstd_buffered.h +++ b/lib/zstd_buffered.h @@ -113,8 +113,8 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, void* dst, size_t* maxDstSizeP * Use ZBUFF_decompressContinue() repetitively to consume your input. * *srcSizePtr and *maxDstSizePtr can be any size. * The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr. -* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input. -* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst . +* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. +* The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst. * @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) * or 0 when a frame is completely decoded * or an error code, which can be tested using ZBUFF_isError(). diff --git a/programs/Makefile b/programs/Makefile index 22c2cd67..4683a946 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -35,7 +35,7 @@ VERSION?= 0.4.0 DESTDIR?= PREFIX ?= /usr/local CPPFLAGS= -I../lib -I../lib/legacy -I./legacy -DZSTD_VERSION=\"$(VERSION)\" -DZSTD_LEGACY_SUPPORT=1 -CFLAGS ?= -O3 # -falign-loops=32 # not always positive +CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) @@ -88,6 +88,11 @@ fuzzer32: $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse datagen.c xxhash.c fuzzer.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) +zbufftest : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/zstd_buffered.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \ + $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \ + datagen.c xxhash.c zbufftest.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + paramgrill : $(ZSTDDIR)/zstd_compress.c $(ZSTDDIR)/zstd_decompress.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c \ $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c \ datagen.c xxhash.c paramgrill.c @@ -100,7 +105,7 @@ clean: @rm -f core *.o tmp* \ zstd$(EXT) zstd32$(EXT) \ fullbench$(EXT) fullbench32$(EXT) \ - fuzzer$(EXT) fuzzer32$(EXT) \ + fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) \ datagen$(EXT) paramgrill$(EXT) @echo Cleaning completed diff --git a/programs/zbufftest.c b/programs/zbufftest.c new file mode 100644 index 00000000..0a10e4de --- /dev/null +++ b/programs/zbufftest.c @@ -0,0 +1,521 @@ +/* + Fuzzer test tool for zstd_buffered + Copyright (C) Yann Collet 2105 + + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - ZSTD source repository : https://github.com/Cyan4973/zstd + - ZSTD public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +/************************************** +* Compiler specific +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define _CRT_SECURE_NO_WARNINGS /* fgets */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ +#endif + +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wmissing-braces" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ +# pragma GCC diagnostic ignored "-Wmissing-field-initializers" /* GCC bug 53119 : doesn't accept { 0 } as initializer (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53119) */ +#endif + + +/************************************** +* Includes +**************************************/ +#include /* free */ +#include /* fgets, sscanf */ +#include /* timeb */ +#include /* strcmp */ +#include "mem.h" +#include "zstd_buffered.h" +#include "zstd.h" /* ZSTD_compressBound() */ +#include "datagen.h" /* RDG_genBuffer */ +#include "xxhash.h" /* XXH64 */ + + +/************************************** + Constants +**************************************/ +#ifndef ZSTD_VERSION +# define ZSTD_VERSION "" +#endif + +#define KB *(1U<<10) +#define MB *(1U<<20) +#define GB *(1U<<30) + +static const U32 nbTestsDefault = 30000; +#define COMPRESSIBLE_NOISE_LENGTH (10 MB) +#define FUZ_COMPRESSIBILITY_DEFAULT 50 +static const U32 prime1 = 2654435761U; +static const U32 prime2 = 2246822519U; + + + +/************************************** +* Display Macros +**************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static U32 g_displayLevel = 2; + +#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ + if ((FUZ_GetMilliSpan(g_time) > g_refreshRate) || (g_displayLevel>=4)) \ + { g_time = FUZ_GetMilliStart(); DISPLAY(__VA_ARGS__); \ + if (g_displayLevel>=4) fflush(stdout); } } +static const U32 g_refreshRate = 150; +static U32 g_time = 0; + + +/********************************************************* +* Fuzzer functions +*********************************************************/ +#define MAX(a,b) ((a)>(b)?(a):(b)) + +static U32 FUZ_GetMilliStart(void) +{ + struct timeb tb; + U32 nCount; + ftime( &tb ); + nCount = (U32) (((tb.time & 0xFFFFF) * 1000) + tb.millitm); + return nCount; +} + + +static U32 FUZ_GetMilliSpan(U32 nTimeStart) +{ + U32 nCurrent = FUZ_GetMilliStart(); + U32 nSpan = nCurrent - nTimeStart; + if (nTimeStart > nCurrent) + nSpan += 0x100000 * 1000; + return nSpan; +} + + +# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) +unsigned int FUZ_rand(unsigned int* src) +{ + U32 rand32 = *src; + rand32 *= prime1; + rand32 += prime2; + rand32 = FUZ_rotl32(rand32, 13); + *src = rand32; + return rand32 >> 5; +} + + +/* +static unsigned FUZ_highbit32(U32 v32) +{ + unsigned nbBits = 0; + if (v32==0) return 0; + for ( ; v32 ; v32>>=1) nbBits++; + return nbBits; +} +*/ + +static int basicUnitTests(U32 seed, double compressibility) +{ + int testResult = 0; + void* CNBuffer; + size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH; + void* compressedBuffer; + size_t compressedBufferSize = ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH); + void* decodedBuffer; + size_t decodedBufferSize = CNBufferSize; + U32 randState = seed; + size_t result, cSize, readSize, genSize; + U32 testNb=0; + ZBUFF_CCtx* zc = ZBUFF_createCCtx(); + ZBUFF_DCtx* zd = ZBUFF_createDCtx(); + + /* Create compressible test buffer */ + CNBuffer = malloc(CNBufferSize); + compressedBuffer = malloc(compressedBufferSize); + decodedBuffer = malloc(decodedBufferSize); + if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) + { + DISPLAY("Not enough memory, aborting\n"); + goto _output_error; + } + RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., randState); + + /* Basic compression test */ + DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); + ZBUFF_compressInit(zc, 1); + readSize = CNBufferSize; + genSize = compressedBufferSize; + result = ZBUFF_compressContinue(zc, compressedBuffer, &genSize, CNBuffer, &readSize); + if (ZBUFF_isError(result)) goto _output_error; + if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */ + cSize = genSize; + genSize = compressedBufferSize - cSize; + result = ZBUFF_compressEnd(zc, compressedBuffer+cSize, &genSize); + if (result != 0) goto _output_error; /* error, or some data not flushed */ + cSize += genSize; + DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); + + /* Basic decompression test */ + DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); + ZBUFF_decompressInit(zd); + readSize = cSize; + genSize = CNBufferSize; + result = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSize); + if (result != 0) goto _output_error; /* should reach end of frame == 0; otherwise, some data left, or an error */ + if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */ + if (readSize != cSize) goto _output_error; /* should have read the entire frame */ + DISPLAYLEVEL(4, "OK \n"); + + /* check regenerated data is byte exact */ + { + size_t i; + DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); + for (i=0; i "); DISPLAY(__VA_ARGS__); \ + DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } + +static const U32 maxSrcLog = 23; +static const U32 maxSampleLog = 19; + +int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility) +{ + BYTE* cNoiseBuffer[5]; + BYTE* srcBuffer; + size_t srcBufferSize = (size_t)1<>= 3; + if (buffNb & 7) + { + const U32 tnb[2] = { 1, 3 }; + buffNb = tnb[buffNb >> 3]; + } + else + { + const U32 tnb[2] = { 0, 4 }; + buffNb = tnb[buffNb >> 3]; + } + } + srcBuffer = cNoiseBuffer[buffNb]; + + /* Multi - segments compression test */ + XXH64_reset(&crc64, 0); + nbChunks = (FUZ_rand(&lseed) & 127) + 2; + sampleSizeLog = FUZ_rand(&lseed) % maxSrcLog; + maxTestSize = (size_t)1 << sampleSizeLog; + maxTestSize += FUZ_rand(&lseed) & (maxTestSize-1); + ZBUFF_compressInit(zc, (FUZ_rand(&lseed) % (20 - (sampleSizeLog/3))) + 1); + totalTestSize = 0; + cSize = 0; + for (n=0; n maxTestSize) break; + } + + genSize = cBufferSize - cSize; + errorCode = ZBUFF_compressEnd(zc, cBuffer+cSize, &genSize); + CHECK (ZBUFF_isError(errorCode), "compression error : %s", ZBUFF_getErrorName(errorCode)); + CHECK (errorCode != 0, "frame epilogue not fully consumed"); + cSize += genSize; + crcOrig = XXH64_digest(&crc64); + + /* multi - fragments decompression test */ + ZBUFF_decompressInit(zd); + genSize = dstBufferSize; + readSize = cBufferSize; + errorCode = ZBUFF_decompressContinue(zd, dstBuffer, &genSize, cBuffer, &readSize); + CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode)); + CHECK (errorCode != 0, "frame not fully decoded"); + CHECK (genSize != totalTestSize, "decompressed data : wrong size") + crcDest = XXH64(dstBuffer, totalTestSize, 0); + if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); + CHECK (crcDest!=crcOrig, "decompressed data corrupted"); + + /* noisy/erroneous src decompression test */ + /* TBD later */ + } + DISPLAY("\rAll fuzzer tests completed \n"); + +_cleanup: + ZBUFF_freeCCtx(zc); + ZBUFF_freeDCtx(zd); + free(cNoiseBuffer[0]); + free(cNoiseBuffer[1]); + free(cNoiseBuffer[2]); + free(cNoiseBuffer[3]); + free(cNoiseBuffer[4]); + free(copyBuffer); + free(cBuffer); + free(dstBuffer); + return result; + +_output_error: + result = 1; + goto _cleanup; +} + + +/********************************************************* +* Command line +*********************************************************/ +int FUZ_usage(char* programName) +{ + DISPLAY( "Usage :\n"); + DISPLAY( " %s [args]\n", programName); + DISPLAY( "\n"); + DISPLAY( "Arguments :\n"); + DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault); + DISPLAY( " -s# : Select seed (default:prompt user)\n"); + DISPLAY( " -t# : Select starting test number (default:0)\n"); + DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); + DISPLAY( " -v : verbose\n"); + DISPLAY( " -p : pause at the end\n"); + DISPLAY( " -h : display help and exit\n"); + return 0; +} + + +int main(int argc, char** argv) +{ + U32 seed=0; + int seedset=0; + int argNb; + int nbTests = nbTestsDefault; + int testNb = 0; + int proba = FUZ_COMPRESSIBILITY_DEFAULT; + int result=0; + U32 mainPause = 0; + char* programName; + + /* Check command line */ + programName = argv[0]; + for(argNb=1; argNb='0') && (*argument<='9')) + { + nbTests *= 10; + nbTests += *argument - '0'; + argument++; + } + break; + + case 's': + argument++; + seed=0; + seedset=1; + while ((*argument>='0') && (*argument<='9')) + { + seed *= 10; + seed += *argument - '0'; + argument++; + } + break; + + case 't': + argument++; + testNb=0; + while ((*argument>='0') && (*argument<='9')) + { + testNb *= 10; + testNb += *argument - '0'; + argument++; + } + break; + + case 'P': /* compressibility % */ + argument++; + proba=0; + while ((*argument>='0') && (*argument<='9')) + { + proba *= 10; + proba += *argument - '0'; + argument++; + } + if (proba<0) proba=0; + if (proba>100) proba=100; + break; + + default: + return FUZ_usage(programName); + } + } + } + } + + /* Get Seed */ + DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION); + + if (!seedset) seed = FUZ_GetMilliStart() % 10000; + DISPLAY("Seed = %u\n", seed); + if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba); + + if (nbTests<=0) nbTests=1; + + if (testNb==0) result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */ + if (!result) + result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100); + if (mainPause) + { + int unused; + DISPLAY("Press Enter \n"); + unused = getchar(); + (void)unused; + } + return result; +}