AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

reduced zstreamtest --mt memory load

Browse Source 
adjust compression level, hence memory usage, depending on nb threads
in order to run correctly on memory-starved VM.
This commit is contained in:
Yann Collet 2017-09-27 17:27:38 -07:00
parent 02502191e5
commit bc32b40b98
2 changed files with 30 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
Makefile
View File

@ -296,7 +296,7 @@ endif
#------------------------------------------------------------------------ #------------------------------------------------------------------------
#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets # target specific tests
#------------------------------------------------------------------------ #------------------------------------------------------------------------
ifneq (,$(filter $(HOST_OS),MSYS POSIX)) ifneq (,$(filter $(HOST_OS),MSYS POSIX))
cmakebuild: cmakebuild:
@ -306,38 +306,38 @@ cmakebuild:
cd $(BUILDIR)/cmake/build ; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall cd $(BUILDIR)/cmake/build ; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
c90build: clean c90build: clean
gcc -v $(CC) -v
CFLAGS="-std=c90" $(MAKE) allmost # will fail, due to missing support for `long long` CFLAGS="-std=c90" $(MAKE) allmost # will fail, due to missing support for `long long`
gnu90build: clean gnu90build: clean
gcc -v $(CC) -v
CFLAGS="-std=gnu90" $(MAKE) allmost CFLAGS="-std=gnu90" $(MAKE) allmost
c99build: clean c99build: clean
gcc -v $(CC) -v
CFLAGS="-std=c99" $(MAKE) allmost CFLAGS="-std=c99" $(MAKE) allmost
gnu99build: clean gnu99build: clean
gcc -v $(CC) -v
CFLAGS="-std=gnu99" $(MAKE) allmost CFLAGS="-std=gnu99" $(MAKE) allmost
c11build: clean c11build: clean
gcc -v $(CC) -v
CFLAGS="-std=c11" $(MAKE) allmost CFLAGS="-std=c11" $(MAKE) allmost
bmix64build: clean bmix64build: clean
gcc -v $(CC) -v
CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(TESTDIR) test CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(TESTDIR) test
bmix32build: clean bmix32build: clean
gcc -v $(CC) -v
CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(TESTDIR) test CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(TESTDIR) test
bmi32build: clean bmi32build: clean
gcc -v $(CC) -v
CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test
staticAnalyze: clean staticAnalyze: clean
gcc -v $(CC) -v
CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all
endif endif

38
tests/zstreamtest.c
View File

@ -741,20 +741,20 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
static const U32 maxSampleLog = 19; static const U32 maxSampleLog = 19;
size_t const srcBufferSize = (size_t)1<<maxSrcLog; size_t const srcBufferSize = (size_t)1<<maxSrcLog;
BYTE* cNoiseBuffer[5]; BYTE* cNoiseBuffer[5];
size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog); size_t const copyBufferSize = srcBufferSize + (1<<maxSampleLog);
BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize); BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize);
size_t const cBufferSize = ZSTD_compressBound(srcBufferSize); size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
BYTE* const cBuffer = (BYTE*)malloc (cBufferSize); BYTE* const cBuffer = (BYTE*)malloc (cBufferSize);
size_t const dstBufferSize = srcBufferSize; size_t const dstBufferSize = srcBufferSize;
BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize); BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize);
U32 result = 0; U32 result = 0;
U32 testNb = 0; U32 testNb = 0;
U32 coreSeed = seed; U32 coreSeed = seed;
ZSTD_CStream* zc = ZSTD_createCStream(); /* will be reset sometimes */ ZSTD_CStream* zc = ZSTD_createCStream(); /* will be re-created sometimes */
ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */ ZSTD_DStream* zd = ZSTD_createDStream(); /* will be re-created sometimes */
ZSTD_DStream* const zd_noise = ZSTD_createDStream(); ZSTD_DStream* const zd_noise = ZSTD_createDStream();
clock_t const startClock = clock(); clock_t const startClock = clock();
const BYTE* dict=NULL; /* can keep same dict on 2 consecutive tests */ const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */
size_t dictSize = 0; size_t dictSize = 0;
U32 oldTestLog = 0; U32 oldTestLog = 0;
U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests; U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests;
@ -832,8 +832,7 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres
if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */ if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
&& oldTestLog /* at least one test happened */ && resetAllowed) { && oldTestLog /* at least one test happened */ && resetAllowed) {
maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2); maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
if (maxTestSize >= srcBufferSize) maxTestSize = MIN(maxTestSize, srcBufferSize-16);
maxTestSize = srcBufferSize-1;
{ U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize; { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize;
CHECK_Z( ZSTD_resetCStream(zc, pledgedSrcSize) ); CHECK_Z( ZSTD_resetCStream(zc, pledgedSrcSize) );
} }
@ -999,15 +998,16 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
U32 result = 0; U32 result = 0;
U32 testNb = 0; U32 testNb = 0;
U32 coreSeed = seed; U32 coreSeed = seed;
ZSTDMT_CCtx* zc = ZSTDMT_createCCtx(2); /* will be reset sometimes */ U32 nbThreads = 2;
ZSTDMT_CCtx* zc = ZSTDMT_createCCtx(nbThreads); /* will be reset sometimes */
ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */ ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */
ZSTD_DStream* const zd_noise = ZSTD_createDStream(); ZSTD_DStream* const zd_noise = ZSTD_createDStream();
clock_t const startClock = clock(); clock_t const startClock = clock();
const BYTE* dict=NULL; /* can keep same dict on 2 consecutive tests */ const BYTE* dict=NULL; /* can keep same dict on 2 consecutive tests */
size_t dictSize = 0; size_t dictSize = 0;
U32 oldTestLog = 0; U32 oldTestLog = 0;
U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests; int const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel()-1 : g_cLevelMax_smallTests;
U32 const nbThreadsMax = bigTests ? 5 : 2; U32 const nbThreadsMax = bigTests ? 4 : 2;
/* allocations */ /* allocations */
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
@ -1043,8 +1043,9 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
size_t maxTestSize; size_t maxTestSize;
/* init */ /* init */
if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); } if (testNb < nbTests) {
else { DISPLAYUPDATE(2, "\r%6u ", testNb); } DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests);
} else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
FUZ_rand(&coreSeed); FUZ_rand(&coreSeed);
lseed = coreSeed ^ prime32; lseed = coreSeed ^ prime32;
@ -1052,7 +1053,7 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
/* some issues can only happen when reusing states */ /* some issues can only happen when reusing states */
if ((FUZ_rand(&lseed) & 0xFF) == 131) { if ((FUZ_rand(&lseed) & 0xFF) == 131) {
U32 const nbThreadsCandidate = (FUZ_rand(&lseed) % 6) + 1; U32 const nbThreadsCandidate = (FUZ_rand(&lseed) % 6) + 1;
U32 const nbThreads = MIN(nbThreadsCandidate, nbThreadsMax); nbThreads = MIN(nbThreadsCandidate, nbThreadsMax);
DISPLAYLEVEL(5, "Creating new context with %u threads \n", nbThreads); DISPLAYLEVEL(5, "Creating new context with %u threads \n", nbThreads);
ZSTDMT_freeCCtx(zc); ZSTDMT_freeCCtx(zc);
zc = ZSTDMT_createCCtx(nbThreads); zc = ZSTDMT_createCCtx(nbThreads);
@ -1092,11 +1093,12 @@ static int fuzzerTests_MT(U32 seed, U32 nbTests, unsigned startTest, double comp
} else { } else {
U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const cLevelCandidate = (FUZ_rand(&lseed) % int const cLevelCandidate = ( FUZ_rand(&lseed)
(ZSTD_maxCLevel() - % (ZSTD_maxCLevel() - (MAX(testLog, dictLog) / 2)) )
(MAX(testLog, dictLog) / 3))) + + 1;
1; int const cLevelThreadAdjusted = cLevelCandidate - (nbThreads * 2) + 2; /* reduce cLevel when multiple threads to reduce memory consumption */
U32 const cLevel = MIN(cLevelCandidate, cLevelMax); int const cLevelMin = MAX(cLevelThreadAdjusted, 1); /* no negative cLevel yet */
int const cLevel = MIN(cLevelMin, cLevelMax);
maxTestSize = FUZ_rLogLength(&lseed, testLog); maxTestSize = FUZ_rLogLength(&lseed, testLog);
oldTestLog = testLog; oldTestLog = testLog;
/* random dictionary selection */ /* random dictionary selection */