AuroraMiddleware/SPIRV-Tools
1
0
Fork 0
mirror of https://github.com/KhronosGroup/SPIRV-Tools synced 2024-10-18 19:20:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Limit fuzzer tests so that they take less time to run (#2763)

Browse Source 
The recently added fuzzer_replayer and fuzzer_shrinker tests were
rather heavyweight and were leading to CI timeouts.  This change
reduces the runtime of those tests by having them do fewer iterations.
This commit is contained in:
Alastair Donaldson 2019-07-25 18:09:49 +01:00 committed by alan-baker
parent bb0e2f65bb
commit f54b8653dd
2 changed files with 60 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
test/fuzz/fuzzer_replayer_test.cpp
View File

@ -240,9 +240,9 @@ TEST(FuzzerReplayerTest, Miscellaneous1) {
OpFunctionEnd OpFunctionEnd
)"; )";
// Do 10 fuzzer runs, starting from an initial seed of 0 (seed value chosen // Do 5 fuzzer runs, starting from an initial seed of 0 (seed value chosen
// arbitrarily). // arbitrarily).
RunFuzzerAndReplayer(shader, protobufs::FactSequence(), 0, 10); RunFuzzerAndReplayer(shader, protobufs::FactSequence(), 0, 5);
} }
TEST(FuzzerReplayerTest, Miscellaneous2) { TEST(FuzzerReplayerTest, Miscellaneous2) {
@ -485,9 +485,9 @@ TEST(FuzzerReplayerTest, Miscellaneous2) {
OpFunctionEnd OpFunctionEnd
)"; )";
// Do 10 fuzzer runs, starting from an initial seed of 10 (seed value chosen // Do 5 fuzzer runs, starting from an initial seed of 10 (seed value chosen
// arbitrarily). // arbitrarily).
RunFuzzerAndReplayer(shader, protobufs::FactSequence(), 10, 10); RunFuzzerAndReplayer(shader, protobufs::FactSequence(), 10, 5);
} }
TEST(FuzzerReplayerTest, Miscellaneous3) { TEST(FuzzerReplayerTest, Miscellaneous3) {
@ -970,9 +970,9 @@ TEST(FuzzerReplayerTest, Miscellaneous3) {
*facts.mutable_fact()->Add() = temp; *facts.mutable_fact()->Add() = temp;
} }
// Do 10 fuzzer runs, starting from an initial seed of 94 (seed value chosen // Do 5 fuzzer runs, starting from an initial seed of 94 (seed value chosen
// arbitrarily). // arbitrarily).
RunFuzzerAndReplayer(shader, facts, 94, 10); RunFuzzerAndReplayer(shader, facts, 94, 5);
} }
} // namespace } // namespace

109
test/fuzz/fuzzer_shrinker_test.cpp
View File

@ -111,20 +111,21 @@ class InterestingThenRandom : public InterestingnessTest {
// |transformation_sequence_in| gets performed with respect to // |transformation_sequence_in| gets performed with respect to
// |interestingness_function|. If |expected_binary_out| is non-empty, it must // |interestingness_function|. If |expected_binary_out| is non-empty, it must
// match the binary obtained by applying the final shrunk set of // match the binary obtained by applying the final shrunk set of
// transformations, in which case the number of such transforations should equal // transformations, in which case the number of such transformations should
// |expected_transformations_out_size|. // equal |expected_transformations_out_size|.
//
// The |step_limit| parameter restricts the number of steps that the shrinker
// will try; it can be set to something small for a faster (but less thorough)
// test.
void RunAndCheckShrinker( void RunAndCheckShrinker(
const spv_target_env& target_env, const std::vector<uint32_t>& binary_in, const spv_target_env& target_env, const std::vector<uint32_t>& binary_in,
const protobufs::FactSequence& initial_facts, const protobufs::FactSequence& initial_facts,
const protobufs::TransformationSequence& transformation_sequence_in, const protobufs::TransformationSequence& transformation_sequence_in,
const Shrinker::InterestingnessFunction& interestingness_function, const Shrinker::InterestingnessFunction& interestingness_function,
const std::vector<uint32_t>& expected_binary_out, const std::vector<uint32_t>& expected_binary_out,
uint32_t expected_transformations_out_size) { uint32_t expected_transformations_out_size, uint32_t step_limit) {
// We want tests to complete in reasonable time, so don't go on too long.
const uint32_t kStepLimit = 50;
// Run the shrinker. // Run the shrinker.
Shrinker shrinker(target_env, kStepLimit); Shrinker shrinker(target_env, step_limit);
shrinker.SetMessageConsumer(kSilentConsumer); shrinker.SetMessageConsumer(kSilentConsumer);
std::vector<uint32_t> binary_out; std::vector<uint32_t> binary_out;
@ -146,14 +147,13 @@ void RunAndCheckShrinker(
} }
} }
// Assembles the given |shader| text, and then does the following |num_runs| // Assembles the given |shader| text, and then:
// times, with successive seeds starting from |initial_seed|: // - Runs the fuzzer with |seed| to yield a set of transformations
// - Runs the fuzzer with the seed to yield a set of transformations
// - Shrinks the transformation with various interestingness functions, // - Shrinks the transformation with various interestingness functions,
// asserting some properties about the result each time // asserting some properties about the result each time
void RunFuzzerAndShrinker(const std::string& shader, void RunFuzzerAndShrinker(const std::string& shader,
const protobufs::FactSequence& initial_facts, const protobufs::FactSequence& initial_facts,
uint32_t initial_seed, uint32_t num_runs) { uint32_t seed) {
const auto env = SPV_ENV_UNIVERSAL_1_3; const auto env = SPV_ENV_UNIVERSAL_1_3;
std::vector<uint32_t> binary_in; std::vector<uint32_t> binary_in;
@ -161,47 +161,50 @@ void RunFuzzerAndShrinker(const std::string& shader,
ASSERT_TRUE(t.Assemble(shader, &binary_in, kFuzzAssembleOption)); ASSERT_TRUE(t.Assemble(shader, &binary_in, kFuzzAssembleOption));
ASSERT_TRUE(t.Validate(binary_in)); ASSERT_TRUE(t.Validate(binary_in));
for (uint32_t seed = initial_seed; seed < initial_seed + num_runs; seed++) { // Run the fuzzer and check that it successfully yields a valid binary.
// Run the fuzzer and check that it successfully yields a valid binary. std::vector<uint32_t> fuzzer_binary_out;
std::vector<uint32_t> fuzzer_binary_out; protobufs::TransformationSequence fuzzer_transformation_sequence_out;
protobufs::TransformationSequence fuzzer_transformation_sequence_out; spvtools::FuzzerOptions fuzzer_options;
spvtools::FuzzerOptions fuzzer_options; spvFuzzerOptionsSetRandomSeed(fuzzer_options, seed);
spvFuzzerOptionsSetRandomSeed(fuzzer_options, seed); Fuzzer fuzzer(env);
Fuzzer fuzzer(env); fuzzer.SetMessageConsumer(kSilentConsumer);
fuzzer.SetMessageConsumer(kSilentConsumer); auto fuzzer_result_status =
auto fuzzer_result_status = fuzzer.Run(binary_in, initial_facts, fuzzer_options, &fuzzer_binary_out,
fuzzer.Run(binary_in, initial_facts, fuzzer_options, &fuzzer_binary_out, &fuzzer_transformation_sequence_out);
&fuzzer_transformation_sequence_out); ASSERT_EQ(Fuzzer::FuzzerResultStatus::kComplete, fuzzer_result_status);
ASSERT_EQ(Fuzzer::FuzzerResultStatus::kComplete, fuzzer_result_status); ASSERT_TRUE(t.Validate(fuzzer_binary_out));
ASSERT_TRUE(t.Validate(fuzzer_binary_out));
// With the AlwaysInteresting test, we should quickly shrink to the original const uint32_t kReasonableStepLimit = 50;
// binary with no transformations remaining. const uint32_t kSmallStepLimit = 20;
RunAndCheckShrinker(env, binary_in, initial_facts,
fuzzer_transformation_sequence_out,
AlwaysInteresting().AsFunction(), binary_in, 0);
// With the OnlyInterestingFirstTime test, no shrinking should be achieved. // With the AlwaysInteresting test, we should quickly shrink to the original
RunAndCheckShrinker( // binary with no transformations remaining.
env, binary_in, initial_facts, fuzzer_transformation_sequence_out, RunAndCheckShrinker(
OnlyInterestingFirstTime().AsFunction(), fuzzer_binary_out, env, binary_in, initial_facts, fuzzer_transformation_sequence_out,
static_cast<uint32_t>( AlwaysInteresting().AsFunction(), binary_in, 0, kReasonableStepLimit);
fuzzer_transformation_sequence_out.transformation_size()));
// The PingPong test is unpredictable; passing an empty expected binary // With the OnlyInterestingFirstTime test, no shrinking should be achieved.
// means that we don't check anything beyond that shrinking completes RunAndCheckShrinker(
// successfully. env, binary_in, initial_facts, fuzzer_transformation_sequence_out,
RunAndCheckShrinker(env, binary_in, initial_facts, OnlyInterestingFirstTime().AsFunction(), fuzzer_binary_out,
fuzzer_transformation_sequence_out, static_cast<uint32_t>(
PingPong().AsFunction(), {}, 0); fuzzer_transformation_sequence_out.transformation_size()),
kReasonableStepLimit);
// The InterestingThenRandom test is unpredictable; passing an empty // The PingPong test is unpredictable; passing an empty expected binary
// expected binary means that we do not check anything about shrinking // means that we don't check anything beyond that shrinking completes
// results. // successfully.
RunAndCheckShrinker( RunAndCheckShrinker(env, binary_in, initial_facts,
env, binary_in, initial_facts, fuzzer_transformation_sequence_out, fuzzer_transformation_sequence_out,
InterestingThenRandom(PseudoRandomGenerator(seed)).AsFunction(), {}, 0); PingPong().AsFunction(), {}, 0, kSmallStepLimit);
}
// The InterestingThenRandom test is unpredictable; passing an empty
// expected binary means that we do not check anything about shrinking
// results.
RunAndCheckShrinker(
env, binary_in, initial_facts, fuzzer_transformation_sequence_out,
InterestingThenRandom(PseudoRandomGenerator(seed)).AsFunction(), {}, 0,
kSmallStepLimit);
} }
TEST(FuzzerShrinkerTest, Miscellaneous1) { TEST(FuzzerShrinkerTest, Miscellaneous1) {
@ -369,9 +372,7 @@ TEST(FuzzerShrinkerTest, Miscellaneous1) {
)"; )";
// Do 2 fuzz-and-shrink runs, starting from an initial seed of 2 (seed value RunFuzzerAndShrinker(shader, protobufs::FactSequence(), 2);
// chosen arbitrarily).
RunFuzzerAndShrinker(shader, protobufs::FactSequence(), 2, 2);
} }
TEST(FuzzerShrinkerTest, Miscellaneous2) { TEST(FuzzerShrinkerTest, Miscellaneous2) {
@ -614,9 +615,7 @@ TEST(FuzzerShrinkerTest, Miscellaneous2) {
OpFunctionEnd OpFunctionEnd
)"; )";
// Do 2 fuzzer runs, starting from an initial seed of 19 (seed value chosen RunFuzzerAndShrinker(shader, protobufs::FactSequence(), 19);
// arbitrarily).
RunFuzzerAndShrinker(shader, protobufs::FactSequence(), 19, 2);
} }
TEST(FuzzerShrinkerTest, Miscellaneous3) { TEST(FuzzerShrinkerTest, Miscellaneous3) {
@ -1101,7 +1100,7 @@ TEST(FuzzerShrinkerTest, Miscellaneous3) {
// Do 2 fuzzer runs, starting from an initial seed of 194 (seed value chosen // Do 2 fuzzer runs, starting from an initial seed of 194 (seed value chosen
// arbitrarily). // arbitrarily).
RunFuzzerAndShrinker(shader, facts, 194, 2); RunFuzzerAndShrinker(shader, facts, 194);
} }
} // namespace } // namespace