From 33fce03045bccbdac39a53a2aab1be0ce9d70ced Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 16 Jan 2017 19:46:22 -0800 Subject: [PATCH] added test checking dictID when using ZSTD_initCStream_usingCDict() It shows that dictID is not properly added into frame header --- Makefile | 2 +- tests/Makefile | 8 ++--- tests/zstreamtest.c | 72 ++++++++++++++++++++++++++++++++++++++------- 3 files changed, 65 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 19b12d0e..0a3634c3 100644 --- a/Makefile +++ b/Makefile @@ -88,7 +88,7 @@ travis-install: $(MAKE) install PREFIX=~/install_test_dir gpptest: clean - $(MAKE) -C programs all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" + CC=g++ $(MAKE) -C programs all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" gcc5test: clean gcc-5 -v diff --git a/tests/Makefile b/tests/Makefile index c080fe34..82c7af41 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -18,10 +18,6 @@ # zstreamtest32: Same as zstreamtest, but forced to compile in 32-bits mode # ########################################################################## -DESTDIR?= -PREFIX ?= /usr/local -BINDIR = $(PREFIX)/bin -MANDIR = $(PREFIX)/share/man/man1 ZSTDDIR = ../lib PRGDIR = ../programs PYTHON ?= python3 @@ -123,10 +119,10 @@ zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zbufftest.c $(MAKE) -C $(ZSTDDIR) libzstd $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT) -zstreamtest : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c +zstreamtest : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c zstreamtest.c $(CC) $(FLAGS) $^ -o $@$(EXT) -zstreamtest32 : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c +zstreamtest32 : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c zstreamtest.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) zstreamtest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index ce619308..4024e5ed 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -26,9 +26,10 @@ #include /* clock_t, clock() */ #include /* strcmp */ #include "mem.h" -#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel, ZSTD_customMem */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel, ZSTD_customMem, ZSTD_getDictID_fromFrame */ #include "zstd.h" /* ZSTD_compressBound */ #include "zstd_errors.h" /* ZSTD_error_srcSize_wrong */ +#include "zdict.h" /* ZDICT_trainFromBuffer */ #include "datagen.h" /* RDG_genBuffer */ #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ #include "xxhash.h" /* XXH64_* */ @@ -44,8 +45,7 @@ static const U32 nbTestsDefault = 10000; #define COMPRESSIBLE_NOISE_LENGTH (10 MB) #define FUZ_COMPRESSIBILITY_DEFAULT 50 -static const U32 prime1 = 2654435761U; -static const U32 prime2 = 2246822519U; +static const U32 prime32 = 2654435761U; /*-************************************ @@ -81,8 +81,9 @@ static clock_t FUZ_GetClockSpan(clock_t clockStart) #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) unsigned int FUZ_rand(unsigned int* seedPtr) { + static const U32 prime2 = 2246822519U; U32 rand32 = *seedPtr; - rand32 *= prime1; + rand32 *= prime32; rand32 += prime2; rand32 = FUZ_rotl32(rand32, 13); *seedPtr = rand32; @@ -107,6 +108,41 @@ static void freeFunction(void* opaque, void* address) * Basic Unit tests ======================================================*/ +typedef struct { + void* start; + size_t size; + size_t filled; +} buffer_t; + +static const buffer_t g_nullBuffer = { NULL, 0 , 0 }; + +static buffer_t FUZ_createDictionary(const void* src, size_t srcSize, size_t blockSize, size_t requestedDictSize) +{ + buffer_t dict = { NULL, 0, 0 }; + size_t const nbBlocks = (srcSize + (blockSize-1)) / blockSize; + size_t* const blockSizes = (size_t*) malloc(nbBlocks * sizeof(size_t)); + if (!blockSizes) return dict; + dict.start = malloc(requestedDictSize); + if (!dict.start) { free(blockSizes); return dict; } + { size_t nb; + for (nb=0; nb= 4 MB); + dictionary = FUZ_createDictionary(CNBuffer, 4 MB, 4 KB, 40 KB); + if (!dictionary.start) { + DISPLAY("Error creating dictionary, aborting \n"); + goto _output_error; + } + dictID = ZDICT_getDictID(dictionary.start, dictionary.filled); + /* generate skippable frame */ MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START); MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize); @@ -260,8 +307,6 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo { size_t const r = ZSTD_endStream(zc, &outBuff); if (r != 0) goto _output_error; } /* error, or some data not flushed */ { unsigned long long origSize = ZSTD_getDecompressedSize(outBuff.dst, outBuff.pos); - DISPLAY("outBuff.pos : %u \n", (U32)outBuff.pos); - DISPLAY("origSize = %u \n", (U32)origSize); if ((size_t)origSize != CNBufferSize) goto _output_error; } /* exact original size must be present */ DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); @@ -320,7 +365,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* CDict scenario */ DISPLAYLEVEL(4, "test%3i : digested dictionary : ", testNb++); - { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, 128 KB, 1); + { ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1); size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict); if (ZSTD_isError(initError)) goto _output_error; cSize = 0; @@ -346,9 +391,15 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo DISPLAYLEVEL(4, "OK (%u bytes) \n", (U32)s); } + DISPLAYLEVEL(4, "test%3i : check Dictionary ID : ", testNb++); + { unsigned const dID = ZSTD_getDictID_fromFrame(compressedBuffer, cSize); + if (dID != dictID) goto _output_error; + DISPLAYLEVEL(4, "OK (%u) \n", dID); + } + /* DDict scenario */ DISPLAYLEVEL(4, "test%3i : decompress %u bytes with digested dictionary : ", testNb++, (U32)CNBufferSize); - { ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, 128 KB); + { ZSTD_DDict* const ddict = ZSTD_createDDict(dictionary.start, dictionary.filled); size_t const initError = ZSTD_initDStream_usingDDict(zd, ddict); if (ZSTD_isError(initError)) goto _output_error; inBuff.src = compressedBuffer; @@ -388,6 +439,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo _end: + FUZ_freeDictionary(dictionary); ZSTD_freeCStream(zc); ZSTD_freeDStream(zd); free(CNBuffer); @@ -492,7 +544,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); } else { DISPLAYUPDATE(2, "\r%6u ", testNb); } FUZ_rand(&coreSeed); - lseed = coreSeed ^ prime1; + lseed = coreSeed ^ prime32; /* states full reset (deliberately not synchronized) */ /* some issues can only happen when reusing states */