From 4eee71e78f6f1f79885ee22dd2ee5da62b138252 Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Fri, 20 Sep 2019 09:43:26 +0100 Subject: [PATCH] Disable long-running fuzzer tests by default (#2887) spirv-fuzz has useful tests that run the fuzzer and shrinker, to give the whole tool a good shake up, effectively "fuzzing the fuzzer". The problems that this detects are sensitive to the source of randomness that is used, which can change from test platform to test platform. It is thus not a good idea to run these tests by default during continuous integration - they may end up failing due to environtal factors, making it look like an unrelated change has broken the fuzzer when really the fuzzer has revealed an already-existing bug in itself. This change makes the tests disabled by default; they can enabled during dedicated testing of the fuzzer. --- test/fuzz/CMakeLists.txt | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/test/fuzz/CMakeLists.txt b/test/fuzz/CMakeLists.txt index 8929cd056..715de6323 100644 --- a/test/fuzz/CMakeLists.txt +++ b/test/fuzz/CMakeLists.txt @@ -17,8 +17,6 @@ if (${SPIRV_BUILD_FUZZER}) set(SOURCES fuzz_test_util.h - fuzzer_replayer_test.cpp - fuzzer_shrinker_test.cpp fact_manager_test.cpp fuzz_test_util.cpp fuzzer_pass_add_useful_constructs_test.cpp @@ -38,6 +36,17 @@ if (${SPIRV_BUILD_FUZZER}) transformation_split_block_test.cpp uniform_buffer_element_descriptor_test.cpp) + if (${SPIRV_ENABLE_LONG_FUZZER_TESTS}) + # These are long-running tests that depend on random seeds. We do not want + # to run them during regular whole-project CI because they may reveal + # spirv-fuzz bugs in changes that are totally unrelated to spirv-fuzz, + # which would be counfounding. Instead, they should be run regularly but + # separately. + set(SOURCES + fuzzer_replayer_test.cpp + fuzzer_shrinker_test.cpp) + endif() + add_spvtools_unittest(TARGET fuzz SRCS ${SOURCES} LIBS SPIRV-Tools-fuzz