AuroraMiddleware/zstd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #847 from terrelln/fuzzer

Browse Source 
[fuzzer] Fuzz long range matching & new API
This commit is contained in:
Yann Collet 2017-09-15 12:09:00 -07:00 committed by GitHub
commit 31829cb057
6 changed files with 209 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
tests/fuzz/Makefile
View File

@ -33,14 +33,19 @@ FUZZ_LDFLAGS := $(LDFLAGS)
FUZZ_ARFLAGS := $(ARFLAGS)
FUZZ_TARGET_FLAGS = $(FUZZ_CPPFLAGS) $(FUZZ_CXXFLAGS) $(FUZZ_LDFLAGS)
FUZZ_HEADERS := fuzz_helpers.h fuzz.h
FUZZ_HEADERS := fuzz_helpers.h fuzz.h zstd_helpers.h
FUZZ_SRC := zstd_helpers.c
ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c
ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c
ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
ZSTDCOMMON_SRC := $(ZSTDDIR)/common/*.c
ZSTDCOMP_SRC := $(ZSTDDIR)/compress/*.c
ZSTDDECOMP_SRC := $(ZSTDDIR)/decompress/*.c
FUZZ_SRC := \
$(FUZZ_SRC) \
$(ZSTDDECOMP_SRC) \
$(ZSTDCOMMON_SRC) \
$(ZSTDCOMP_SRC)
ZSTD_OBJ := $(patsubst %.c,%.o, $(wildcard $(ZSTD_FILES)))
FUZZ_OBJ := $(patsubst %.c,%.o, $(wildcard $(FUZZ_SRC)))
.PHONY: default all clean
@ -58,23 +63,23 @@ all: \
%.o: %.c
$(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $^ -c -o $@
simple_round_trip: $(FUZZ_HEADERS) $(ZSTD_OBJ) simple_round_trip.o
$(CXX) $(FUZZ_TARGET_FLAGS) $(ZSTD_OBJ) simple_round_trip.o $(LIB_FUZZING_ENGINE) -o $@
simple_round_trip: $(FUZZ_HEADERS) $(FUZZ_OBJ) simple_round_trip.o
$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) simple_round_trip.o $(LIB_FUZZING_ENGINE) -o $@
stream_round_trip: $(FUZZ_HEADERS) $(ZSTD_OBJ) stream_round_trip.o
$(CXX) $(FUZZ_TARGET_FLAGS) $(ZSTD_OBJ) stream_round_trip.o $(LIB_FUZZING_ENGINE) -o $@
stream_round_trip: $(FUZZ_HEADERS) $(FUZZ_OBJ) stream_round_trip.o
$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) stream_round_trip.o $(LIB_FUZZING_ENGINE) -o $@
block_round_trip: $(FUZZ_HEADERS) $(ZSTD_OBJ) block_round_trip.o
$(CXX) $(FUZZ_TARGET_FLAGS) $(ZSTD_OBJ) block_round_trip.o $(LIB_FUZZING_ENGINE) -o $@
block_round_trip: $(FUZZ_HEADERS) $(FUZZ_OBJ) block_round_trip.o
$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) block_round_trip.o $(LIB_FUZZING_ENGINE) -o $@
simple_decompress: $(FUZZ_HEADERS) $(ZSTD_OBJ) simple_decompress.o
$(CXX) $(FUZZ_TARGET_FLAGS) $(ZSTD_OBJ) simple_decompress.o $(LIB_FUZZING_ENGINE) -o $@
simple_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) simple_decompress.o
$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) simple_decompress.o $(LIB_FUZZING_ENGINE) -o $@
stream_decompress: $(FUZZ_HEADERS) $(ZSTD_OBJ) stream_decompress.o
$(CXX) $(FUZZ_TARGET_FLAGS) $(ZSTD_OBJ) stream_decompress.o $(LIB_FUZZING_ENGINE) -o $@
stream_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) stream_decompress.o
$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) stream_decompress.o $(LIB_FUZZING_ENGINE) -o $@
block_decompress: $(FUZZ_HEADERS) $(ZSTD_OBJ) block_decompress.o
$(CXX) $(FUZZ_TARGET_FLAGS) $(ZSTD_OBJ) block_decompress.o $(LIB_FUZZING_ENGINE) -o $@
block_decompress: $(FUZZ_HEADERS) $(FUZZ_OBJ) block_decompress.o
$(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_OBJ) block_decompress.o $(LIB_FUZZING_ENGINE) -o $@
libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h regression_driver.o
$(AR) $(FUZZ_ARFLAGS) $@ regression_driver.o

37
tests/fuzz/fuzz_helpers.h
View File

@ -16,8 +16,10 @@
#include "fuzz.h"
#include "xxhash.h"
#include "zstd.h"
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#ifdef __cplusplus
extern "C" {
@ -38,6 +40,8 @@ extern "C" {
__LINE__, FUZZ_QUOTE(cond), (msg)), \
abort()))
#define FUZZ_ASSERT(cond) FUZZ_ASSERT_MSG((cond), "");
#define FUZZ_ZASSERT(code) \
FUZZ_ASSERT_MSG(!ZSTD_isError(code), ZSTD_getErrorName(code))
#if defined(__GNUC__)
#define FUZZ_STATIC static __inline __attribute__((unused))
@ -55,23 +59,30 @@ extern "C" {
* Consumes up to the first FUZZ_RNG_SEED_SIZE bytes of the input.
*/
FUZZ_STATIC uint32_t FUZZ_seed(uint8_t const **src, size_t* size) {
uint8_t const *data = *src;
size_t const toHash = MIN(FUZZ_RNG_SEED_SIZE, *size);
*size -= toHash;
*src += toHash;
return XXH32(data, toHash, 0);
uint8_t const *data = *src;
size_t const toHash = MIN(FUZZ_RNG_SEED_SIZE, *size);
*size -= toHash;
*src += toHash;
return XXH32(data, toHash, 0);
}
#define FUZZ_rotl32(x, r) (((x) << (r)) | ((x) >> (32 - (r))))
FUZZ_STATIC uint32_t FUZZ_rand(uint32_t *state) {
static const uint32_t prime1 = 2654435761U;
static const uint32_t prime2 = 2246822519U;
uint32_t rand32 = *state;
rand32 *= prime1;
rand32 += prime2;
rand32 = FUZZ_rotl32(rand32, 13);
*state = rand32;
return rand32 >> 5;
static const uint32_t prime1 = 2654435761U;
static const uint32_t prime2 = 2246822519U;
uint32_t rand32 = *state;
rand32 *= prime1;
rand32 += prime2;
rand32 = FUZZ_rotl32(rand32, 13);
*state = rand32;
return rand32 >> 5;
}
/* Returns a random numer in the range [min, max]. */
FUZZ_STATIC uint32_t FUZZ_rand32(uint32_t *state, uint32_t min, uint32_t max) {
uint32_t random = FUZZ_rand(state);
return min + (random % (max - min + 1));
}
#ifdef __cplusplus

34
tests/fuzz/simple_round_trip.c
View File

@ -12,12 +12,14 @@
* compares the result with the original, and calls abort() on corruption.
*/
#define ZSTD_STATIC_LINKING_ONLY
#include <stddef.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "fuzz_helpers.h"
#include "zstd.h"
#include "zstd_helpers.h"
static const int kMaxClevel = 19;
@ -32,14 +34,28 @@ static size_t roundTripTest(void *result, size_t resultCapacity,
void *compressed, size_t compressedCapacity,
const void *src, size_t srcSize)
{
int const cLevel = FUZZ_rand(&seed) % kMaxClevel;
size_t const cSize = ZSTD_compressCCtx(cctx, compressed, compressedCapacity,
src, srcSize, cLevel);
if (ZSTD_isError(cSize)) {
fprintf(stderr, "Compression error: %s\n", ZSTD_getErrorName(cSize));
return cSize;
}
return ZSTD_decompressDCtx(dctx, result, resultCapacity, compressed, cSize);
size_t cSize;
if (FUZZ_rand(&seed) & 1) {
ZSTD_inBuffer in = {src, srcSize, 0};
ZSTD_outBuffer out = {compressed, compressedCapacity, 0};
ZSTD_CCtx_reset(cctx);
FUZZ_setRandomParameters(cctx, &seed);
size_t const err = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
if (err != 0) {
return err;
}
cSize = out.pos;
} else {
int const cLevel = FUZZ_rand(&seed) % kMaxClevel;
cSize = ZSTD_compressCCtx(
cctx, compressed, compressedCapacity, src, srcSize, cLevel);
}
if (ZSTD_isError(cSize)) {
fprintf(stderr, "Compression error: %s\n", ZSTD_getErrorName(cSize));
return cSize;
}
return ZSTD_decompressDCtx(dctx, result, resultCapacity, compressed, cSize);
}
int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)

115
tests/fuzz/stream_round_trip.c
View File

@ -12,16 +12,16 @@
* compares the result with the original, and calls abort() on corruption.
*/
#define ZSTD_STATIC_LINKING_ONLY
#include <stddef.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "fuzz_helpers.h"
#include "zstd.h"
#include "zstd_helpers.h"
static const int kMaxClevel = 19;
static ZSTD_CStream *cstream = NULL;
ZSTD_CCtx *cctx = NULL;
static ZSTD_DCtx *dctx = NULL;
static uint8_t* cBuf = NULL;
static uint8_t* rBuf = NULL;
@ -30,84 +30,89 @@ static uint32_t seed;
static ZSTD_outBuffer makeOutBuffer(uint8_t *dst, size_t capacity)
{
ZSTD_outBuffer buffer = { dst, 0, 0 };
ZSTD_outBuffer buffer = { dst, 0, 0 };
FUZZ_ASSERT(capacity > 0);
buffer.size = (FUZZ_rand(&seed) % capacity) + 1;
FUZZ_ASSERT(buffer.size <= capacity);
FUZZ_ASSERT(capacity > 0);
buffer.size = (FUZZ_rand(&seed) % capacity) + 1;
FUZZ_ASSERT(buffer.size <= capacity);
return buffer;
return buffer;
}
static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size)
{
ZSTD_inBuffer buffer = { *src, 0, 0 };
ZSTD_inBuffer buffer = { *src, 0, 0 };
FUZZ_ASSERT(*size > 0);
buffer.size = (FUZZ_rand(&seed) % *size) + 1;
FUZZ_ASSERT(buffer.size <= *size);
*src += buffer.size;
*size -= buffer.size;
FUZZ_ASSERT(*size > 0);
buffer.size = (FUZZ_rand(&seed) % *size) + 1;
FUZZ_ASSERT(buffer.size <= *size);
*src += buffer.size;
*size -= buffer.size;
return buffer;
return buffer;
}
static size_t compress(uint8_t *dst, size_t capacity,
const uint8_t *src, size_t srcSize)
{
int cLevel = FUZZ_rand(&seed) % kMaxClevel;
size_t dstSize = 0;
FUZZ_ASSERT(!ZSTD_isError(ZSTD_initCStream(cstream, cLevel)));
ZSTD_CCtx_reset(cctx);
FUZZ_setRandomParameters(cctx, &seed);
while (srcSize > 0) {
ZSTD_inBuffer in = makeInBuffer(&src, &srcSize);
/* Mode controls the action. If mode == -1 we pick a new mode */
int mode = -1;
while (in.pos < in.size) {
ZSTD_outBuffer out = makeOutBuffer(dst, capacity);
/* Previous action finished, pick a new mode. */
if (mode == -1) mode = FUZZ_rand(&seed) % 10;
switch (mode) {
case 0: /* fall-though */
case 1: /* fall-though */
case 2: {
size_t const ret = ZSTD_flushStream(cstream, &out);
FUZZ_ASSERT_MSG(!ZSTD_isError(ret), ZSTD_getErrorName(ret));
if (ret == 0) mode = -1;
break;
}
case 3: {
size_t ret = ZSTD_endStream(cstream, &out);
FUZZ_ASSERT_MSG(!ZSTD_isError(ret), ZSTD_getErrorName(ret));
/* Reset the compressor when the frame is finished */
if (ret == 0) {
cLevel = FUZZ_rand(&seed) % kMaxClevel;
ret = ZSTD_initCStream(cstream, cLevel);
FUZZ_ASSERT(!ZSTD_isError(ret));
ZSTD_outBuffer out = makeOutBuffer(dst, capacity);
/* Previous action finished, pick a new mode. */
if (mode == -1) mode = FUZZ_rand(&seed) % 10;
switch (mode) {
case 0: /* fall-though */
case 1: /* fall-though */
case 2: {
size_t const ret =
ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_flush);
FUZZ_ZASSERT(ret);
if (ret == 0)
mode = -1;
break;
}
case 3: {
size_t ret =
ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
FUZZ_ZASSERT(ret);
/* Reset the compressor when the frame is finished */
if (ret == 0) {
ZSTD_CCtx_reset(cctx);
FUZZ_setRandomParameters(cctx, &seed);
mode = -1;
}
break;
}
default: {
size_t const ret =
ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_continue);
FUZZ_ZASSERT(ret);
mode = -1;
}
break;
}
default: {
size_t const ret = ZSTD_compressStream(cstream, &out, &in);
FUZZ_ASSERT_MSG(!ZSTD_isError(ret), ZSTD_getErrorName(ret));
mode = -1;
}
}
dst += out.pos;
dstSize += out.pos;
capacity -= out.pos;
dst += out.pos;
dstSize += out.pos;
capacity -= out.pos;
}
}
for (;;) {
ZSTD_inBuffer in = {NULL, 0, 0};
ZSTD_outBuffer out = makeOutBuffer(dst, capacity);
size_t const ret = ZSTD_endStream(cstream, &out);
FUZZ_ASSERT_MSG(!ZSTD_isError(ret), ZSTD_getErrorName(ret));
size_t const ret = ZSTD_compress_generic(cctx, &out, &in, ZSTD_e_end);
FUZZ_ZASSERT(ret);
dst += out.pos;
dstSize += out.pos;
capacity -= out.pos;
if (ret == 0) break;
if (ret == 0)
break;
}
return dstSize;
}
@ -128,9 +133,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
bufSize = neededBufSize;
FUZZ_ASSERT(cBuf && rBuf);
}
if (!cstream) {
cstream = ZSTD_createCStream();
FUZZ_ASSERT(cstream);
if (!cctx) {
cctx = ZSTD_createCCtx();
FUZZ_ASSERT(cctx);
}
if (!dctx) {
dctx = ZSTD_createDCtx();
@ -147,7 +152,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
}
#ifndef STATEFUL_FUZZING
ZSTD_freeCStream(cstream); cstream = NULL;
ZSTD_freeCCtx(cctx); cctx = NULL;
ZSTD_freeDCtx(dctx); dctx = NULL;
#endif
return 0;

46
tests/fuzz/zstd_helpers.c Normal file
View File

@ -0,0 +1,46 @@
/*
* Copyright (c) 2016-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd_helpers.h"
#include "fuzz_helpers.h"
#include "zstd.h"
static void setRand(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned min,
unsigned max, uint32_t *state) {
unsigned const value = FUZZ_rand32(state, min, max);
FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value));
}
void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, uint32_t *state)
{
setRand(cctx, ZSTD_p_windowLog, ZSTD_WINDOWLOG_MIN, 23, state);
setRand(cctx, ZSTD_p_hashLog, ZSTD_HASHLOG_MIN, 23, state);
setRand(cctx, ZSTD_p_chainLog, ZSTD_CHAINLOG_MIN, 24, state);
setRand(cctx, ZSTD_p_searchLog, ZSTD_SEARCHLOG_MIN, 9, state);
setRand(cctx, ZSTD_p_minMatch, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX,
state);
setRand(cctx, ZSTD_p_targetLength, ZSTD_TARGETLENGTH_MIN,
ZSTD_TARGETLENGTH_MAX, state);
setRand(cctx, ZSTD_p_compressionStrategy, ZSTD_fast, ZSTD_btultra, state);
/* Select frame parameters */
setRand(cctx, ZSTD_p_contentSizeFlag, 0, 1, state);
setRand(cctx, ZSTD_p_checksumFlag, 0, 1, state);
setRand(cctx, ZSTD_p_dictIDFlag, 0, 1, state);
/* Select long distance matchig parameters */
setRand(cctx, ZSTD_p_enableLongDistanceMatching, 0, 1, state);
setRand(cctx, ZSTD_p_ldmHashLog, ZSTD_HASHLOG_MIN, 24, state);
setRand(cctx, ZSTD_p_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN,
ZSTD_LDM_MINMATCH_MAX, state);
setRand(cctx, ZSTD_p_ldmBucketSizeLog, 0, ZSTD_LDM_BUCKETSIZELOG_MAX,
state);
setRand(cctx, ZSTD_p_ldmHashEveryLog, 0,
ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN, state);
}

31
tests/fuzz/zstd_helpers.h Normal file
View File

@ -0,0 +1,31 @@
/*
* Copyright (c) 2016-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
/**
* Helper functions for fuzzing.
*/
#ifndef ZSTD_HELPERS_H
#define ZSTD_HELPERS_H
#include "zstd.h"
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, uint32_t *state);
#ifdef __cplusplus
}
#endif
#endif /* ZSTD_HELPERS_H */