mirror of
https://github.com/KhronosGroup/SPIRV-Tools
synced 2024-10-18 19:20:05 +00:00
Limit fuzzer tests so that they take less time to run (#2763)
The recently added fuzzer_replayer and fuzzer_shrinker tests were rather heavyweight and were leading to CI timeouts. This change reduces the runtime of those tests by having them do fewer iterations.
This commit is contained in:
parent
bb0e2f65bb
commit
f54b8653dd
@ -240,9 +240,9 @@ TEST(FuzzerReplayerTest, Miscellaneous1) {
|
|||||||
OpFunctionEnd
|
OpFunctionEnd
|
||||||
)";
|
)";
|
||||||
|
|
||||||
// Do 10 fuzzer runs, starting from an initial seed of 0 (seed value chosen
|
// Do 5 fuzzer runs, starting from an initial seed of 0 (seed value chosen
|
||||||
// arbitrarily).
|
// arbitrarily).
|
||||||
RunFuzzerAndReplayer(shader, protobufs::FactSequence(), 0, 10);
|
RunFuzzerAndReplayer(shader, protobufs::FactSequence(), 0, 5);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(FuzzerReplayerTest, Miscellaneous2) {
|
TEST(FuzzerReplayerTest, Miscellaneous2) {
|
||||||
@ -485,9 +485,9 @@ TEST(FuzzerReplayerTest, Miscellaneous2) {
|
|||||||
OpFunctionEnd
|
OpFunctionEnd
|
||||||
)";
|
)";
|
||||||
|
|
||||||
// Do 10 fuzzer runs, starting from an initial seed of 10 (seed value chosen
|
// Do 5 fuzzer runs, starting from an initial seed of 10 (seed value chosen
|
||||||
// arbitrarily).
|
// arbitrarily).
|
||||||
RunFuzzerAndReplayer(shader, protobufs::FactSequence(), 10, 10);
|
RunFuzzerAndReplayer(shader, protobufs::FactSequence(), 10, 5);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(FuzzerReplayerTest, Miscellaneous3) {
|
TEST(FuzzerReplayerTest, Miscellaneous3) {
|
||||||
@ -970,9 +970,9 @@ TEST(FuzzerReplayerTest, Miscellaneous3) {
|
|||||||
*facts.mutable_fact()->Add() = temp;
|
*facts.mutable_fact()->Add() = temp;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Do 10 fuzzer runs, starting from an initial seed of 94 (seed value chosen
|
// Do 5 fuzzer runs, starting from an initial seed of 94 (seed value chosen
|
||||||
// arbitrarily).
|
// arbitrarily).
|
||||||
RunFuzzerAndReplayer(shader, facts, 94, 10);
|
RunFuzzerAndReplayer(shader, facts, 94, 5);
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace
|
} // namespace
|
||||||
|
@ -111,20 +111,21 @@ class InterestingThenRandom : public InterestingnessTest {
|
|||||||
// |transformation_sequence_in| gets performed with respect to
|
// |transformation_sequence_in| gets performed with respect to
|
||||||
// |interestingness_function|. If |expected_binary_out| is non-empty, it must
|
// |interestingness_function|. If |expected_binary_out| is non-empty, it must
|
||||||
// match the binary obtained by applying the final shrunk set of
|
// match the binary obtained by applying the final shrunk set of
|
||||||
// transformations, in which case the number of such transforations should equal
|
// transformations, in which case the number of such transformations should
|
||||||
// |expected_transformations_out_size|.
|
// equal |expected_transformations_out_size|.
|
||||||
|
//
|
||||||
|
// The |step_limit| parameter restricts the number of steps that the shrinker
|
||||||
|
// will try; it can be set to something small for a faster (but less thorough)
|
||||||
|
// test.
|
||||||
void RunAndCheckShrinker(
|
void RunAndCheckShrinker(
|
||||||
const spv_target_env& target_env, const std::vector<uint32_t>& binary_in,
|
const spv_target_env& target_env, const std::vector<uint32_t>& binary_in,
|
||||||
const protobufs::FactSequence& initial_facts,
|
const protobufs::FactSequence& initial_facts,
|
||||||
const protobufs::TransformationSequence& transformation_sequence_in,
|
const protobufs::TransformationSequence& transformation_sequence_in,
|
||||||
const Shrinker::InterestingnessFunction& interestingness_function,
|
const Shrinker::InterestingnessFunction& interestingness_function,
|
||||||
const std::vector<uint32_t>& expected_binary_out,
|
const std::vector<uint32_t>& expected_binary_out,
|
||||||
uint32_t expected_transformations_out_size) {
|
uint32_t expected_transformations_out_size, uint32_t step_limit) {
|
||||||
// We want tests to complete in reasonable time, so don't go on too long.
|
|
||||||
const uint32_t kStepLimit = 50;
|
|
||||||
|
|
||||||
// Run the shrinker.
|
// Run the shrinker.
|
||||||
Shrinker shrinker(target_env, kStepLimit);
|
Shrinker shrinker(target_env, step_limit);
|
||||||
shrinker.SetMessageConsumer(kSilentConsumer);
|
shrinker.SetMessageConsumer(kSilentConsumer);
|
||||||
|
|
||||||
std::vector<uint32_t> binary_out;
|
std::vector<uint32_t> binary_out;
|
||||||
@ -146,14 +147,13 @@ void RunAndCheckShrinker(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Assembles the given |shader| text, and then does the following |num_runs|
|
// Assembles the given |shader| text, and then:
|
||||||
// times, with successive seeds starting from |initial_seed|:
|
// - Runs the fuzzer with |seed| to yield a set of transformations
|
||||||
// - Runs the fuzzer with the seed to yield a set of transformations
|
|
||||||
// - Shrinks the transformation with various interestingness functions,
|
// - Shrinks the transformation with various interestingness functions,
|
||||||
// asserting some properties about the result each time
|
// asserting some properties about the result each time
|
||||||
void RunFuzzerAndShrinker(const std::string& shader,
|
void RunFuzzerAndShrinker(const std::string& shader,
|
||||||
const protobufs::FactSequence& initial_facts,
|
const protobufs::FactSequence& initial_facts,
|
||||||
uint32_t initial_seed, uint32_t num_runs) {
|
uint32_t seed) {
|
||||||
const auto env = SPV_ENV_UNIVERSAL_1_3;
|
const auto env = SPV_ENV_UNIVERSAL_1_3;
|
||||||
|
|
||||||
std::vector<uint32_t> binary_in;
|
std::vector<uint32_t> binary_in;
|
||||||
@ -161,47 +161,50 @@ void RunFuzzerAndShrinker(const std::string& shader,
|
|||||||
ASSERT_TRUE(t.Assemble(shader, &binary_in, kFuzzAssembleOption));
|
ASSERT_TRUE(t.Assemble(shader, &binary_in, kFuzzAssembleOption));
|
||||||
ASSERT_TRUE(t.Validate(binary_in));
|
ASSERT_TRUE(t.Validate(binary_in));
|
||||||
|
|
||||||
for (uint32_t seed = initial_seed; seed < initial_seed + num_runs; seed++) {
|
// Run the fuzzer and check that it successfully yields a valid binary.
|
||||||
// Run the fuzzer and check that it successfully yields a valid binary.
|
std::vector<uint32_t> fuzzer_binary_out;
|
||||||
std::vector<uint32_t> fuzzer_binary_out;
|
protobufs::TransformationSequence fuzzer_transformation_sequence_out;
|
||||||
protobufs::TransformationSequence fuzzer_transformation_sequence_out;
|
spvtools::FuzzerOptions fuzzer_options;
|
||||||
spvtools::FuzzerOptions fuzzer_options;
|
spvFuzzerOptionsSetRandomSeed(fuzzer_options, seed);
|
||||||
spvFuzzerOptionsSetRandomSeed(fuzzer_options, seed);
|
Fuzzer fuzzer(env);
|
||||||
Fuzzer fuzzer(env);
|
fuzzer.SetMessageConsumer(kSilentConsumer);
|
||||||
fuzzer.SetMessageConsumer(kSilentConsumer);
|
auto fuzzer_result_status =
|
||||||
auto fuzzer_result_status =
|
fuzzer.Run(binary_in, initial_facts, fuzzer_options, &fuzzer_binary_out,
|
||||||
fuzzer.Run(binary_in, initial_facts, fuzzer_options, &fuzzer_binary_out,
|
&fuzzer_transformation_sequence_out);
|
||||||
&fuzzer_transformation_sequence_out);
|
ASSERT_EQ(Fuzzer::FuzzerResultStatus::kComplete, fuzzer_result_status);
|
||||||
ASSERT_EQ(Fuzzer::FuzzerResultStatus::kComplete, fuzzer_result_status);
|
ASSERT_TRUE(t.Validate(fuzzer_binary_out));
|
||||||
ASSERT_TRUE(t.Validate(fuzzer_binary_out));
|
|
||||||
|
|
||||||
// With the AlwaysInteresting test, we should quickly shrink to the original
|
const uint32_t kReasonableStepLimit = 50;
|
||||||
// binary with no transformations remaining.
|
const uint32_t kSmallStepLimit = 20;
|
||||||
RunAndCheckShrinker(env, binary_in, initial_facts,
|
|
||||||
fuzzer_transformation_sequence_out,
|
|
||||||
AlwaysInteresting().AsFunction(), binary_in, 0);
|
|
||||||
|
|
||||||
// With the OnlyInterestingFirstTime test, no shrinking should be achieved.
|
// With the AlwaysInteresting test, we should quickly shrink to the original
|
||||||
RunAndCheckShrinker(
|
// binary with no transformations remaining.
|
||||||
env, binary_in, initial_facts, fuzzer_transformation_sequence_out,
|
RunAndCheckShrinker(
|
||||||
OnlyInterestingFirstTime().AsFunction(), fuzzer_binary_out,
|
env, binary_in, initial_facts, fuzzer_transformation_sequence_out,
|
||||||
static_cast<uint32_t>(
|
AlwaysInteresting().AsFunction(), binary_in, 0, kReasonableStepLimit);
|
||||||
fuzzer_transformation_sequence_out.transformation_size()));
|
|
||||||
|
|
||||||
// The PingPong test is unpredictable; passing an empty expected binary
|
// With the OnlyInterestingFirstTime test, no shrinking should be achieved.
|
||||||
// means that we don't check anything beyond that shrinking completes
|
RunAndCheckShrinker(
|
||||||
// successfully.
|
env, binary_in, initial_facts, fuzzer_transformation_sequence_out,
|
||||||
RunAndCheckShrinker(env, binary_in, initial_facts,
|
OnlyInterestingFirstTime().AsFunction(), fuzzer_binary_out,
|
||||||
fuzzer_transformation_sequence_out,
|
static_cast<uint32_t>(
|
||||||
PingPong().AsFunction(), {}, 0);
|
fuzzer_transformation_sequence_out.transformation_size()),
|
||||||
|
kReasonableStepLimit);
|
||||||
|
|
||||||
// The InterestingThenRandom test is unpredictable; passing an empty
|
// The PingPong test is unpredictable; passing an empty expected binary
|
||||||
// expected binary means that we do not check anything about shrinking
|
// means that we don't check anything beyond that shrinking completes
|
||||||
// results.
|
// successfully.
|
||||||
RunAndCheckShrinker(
|
RunAndCheckShrinker(env, binary_in, initial_facts,
|
||||||
env, binary_in, initial_facts, fuzzer_transformation_sequence_out,
|
fuzzer_transformation_sequence_out,
|
||||||
InterestingThenRandom(PseudoRandomGenerator(seed)).AsFunction(), {}, 0);
|
PingPong().AsFunction(), {}, 0, kSmallStepLimit);
|
||||||
}
|
|
||||||
|
// The InterestingThenRandom test is unpredictable; passing an empty
|
||||||
|
// expected binary means that we do not check anything about shrinking
|
||||||
|
// results.
|
||||||
|
RunAndCheckShrinker(
|
||||||
|
env, binary_in, initial_facts, fuzzer_transformation_sequence_out,
|
||||||
|
InterestingThenRandom(PseudoRandomGenerator(seed)).AsFunction(), {}, 0,
|
||||||
|
kSmallStepLimit);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(FuzzerShrinkerTest, Miscellaneous1) {
|
TEST(FuzzerShrinkerTest, Miscellaneous1) {
|
||||||
@ -369,9 +372,7 @@ TEST(FuzzerShrinkerTest, Miscellaneous1) {
|
|||||||
|
|
||||||
)";
|
)";
|
||||||
|
|
||||||
// Do 2 fuzz-and-shrink runs, starting from an initial seed of 2 (seed value
|
RunFuzzerAndShrinker(shader, protobufs::FactSequence(), 2);
|
||||||
// chosen arbitrarily).
|
|
||||||
RunFuzzerAndShrinker(shader, protobufs::FactSequence(), 2, 2);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(FuzzerShrinkerTest, Miscellaneous2) {
|
TEST(FuzzerShrinkerTest, Miscellaneous2) {
|
||||||
@ -614,9 +615,7 @@ TEST(FuzzerShrinkerTest, Miscellaneous2) {
|
|||||||
OpFunctionEnd
|
OpFunctionEnd
|
||||||
)";
|
)";
|
||||||
|
|
||||||
// Do 2 fuzzer runs, starting from an initial seed of 19 (seed value chosen
|
RunFuzzerAndShrinker(shader, protobufs::FactSequence(), 19);
|
||||||
// arbitrarily).
|
|
||||||
RunFuzzerAndShrinker(shader, protobufs::FactSequence(), 19, 2);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(FuzzerShrinkerTest, Miscellaneous3) {
|
TEST(FuzzerShrinkerTest, Miscellaneous3) {
|
||||||
@ -1101,7 +1100,7 @@ TEST(FuzzerShrinkerTest, Miscellaneous3) {
|
|||||||
|
|
||||||
// Do 2 fuzzer runs, starting from an initial seed of 194 (seed value chosen
|
// Do 2 fuzzer runs, starting from an initial seed of 194 (seed value chosen
|
||||||
// arbitrarily).
|
// arbitrarily).
|
||||||
RunFuzzerAndShrinker(shader, facts, 194, 2);
|
RunFuzzerAndShrinker(shader, facts, 194);
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace
|
} // namespace
|
||||||
|
Loading…
Reference in New Issue
Block a user