From 7275a71654abf01daffa59b48427c4f717c7049f Mon Sep 17 00:00:00 2001 From: Alastair Donaldson Date: Fri, 20 Sep 2019 10:54:09 +0100 Subject: [PATCH] Allow validation during spirv-fuzz replay (#2873) To aid in debugging issues in spirv-fuzz, this change adds an option whereby the SPIR-V module is validated after each transformation is applied during replay. This can assist in finding a transformation that erroneously makes the module invalid, so that said transformation can be debugged. --- include/spirv-tools/libspirv.h | 5 ++++ include/spirv-tools/libspirv.hpp | 5 ++++ source/fuzz/replayer.cpp | 33 ++++++++++++++++++++-- source/fuzz/replayer.h | 3 +- source/fuzz/shrinker.cpp | 22 +++++++++------ source/fuzz/shrinker.h | 3 +- source/spirv_fuzzer_options.cpp | 6 ++++ source/spirv_fuzzer_options.h | 3 ++ test/fuzz/fuzzer_replayer_test.cpp | 2 +- test/fuzz/fuzzer_shrinker_test.cpp | 2 +- tools/fuzz/fuzz.cpp | 44 ++++++++++++++++++++++++++++-- 11 files changed, 110 insertions(+), 18 deletions(-) diff --git a/include/spirv-tools/libspirv.h b/include/spirv-tools/libspirv.h index 672848ea1..dd2526bf3 100644 --- a/include/spirv-tools/libspirv.h +++ b/include/spirv-tools/libspirv.h @@ -612,6 +612,11 @@ SPIRV_TOOLS_EXPORT spv_fuzzer_options spvFuzzerOptionsCreate(); // Destroys the given fuzzer options object. SPIRV_TOOLS_EXPORT void spvFuzzerOptionsDestroy(spv_fuzzer_options options); +// Enables running the validator after every transformation is applied during +// a replay. +SPIRV_TOOLS_EXPORT void spvFuzzerOptionsEnableReplayValidation( + spv_fuzzer_options options); + // Sets the seed with which the random number generator used by the fuzzer // should be initialized. SPIRV_TOOLS_EXPORT void spvFuzzerOptionsSetRandomSeed( diff --git a/include/spirv-tools/libspirv.hpp b/include/spirv-tools/libspirv.hpp index 2da1152aa..b09fdd457 100644 --- a/include/spirv-tools/libspirv.hpp +++ b/include/spirv-tools/libspirv.hpp @@ -214,6 +214,11 @@ class FuzzerOptions { return options_; } + // See spvFuzzerOptionsEnableReplayValidation. + void enable_replay_validation() { + spvFuzzerOptionsEnableReplayValidation(options_); + } + // See spvFuzzerOptionsSetRandomSeed. void set_random_seed(uint32_t seed) { spvFuzzerOptionsSetRandomSeed(options_, seed); diff --git a/source/fuzz/replayer.cpp b/source/fuzz/replayer.cpp index b0d4ee2ec..398ce5950 100644 --- a/source/fuzz/replayer.cpp +++ b/source/fuzz/replayer.cpp @@ -37,13 +37,18 @@ namespace spvtools { namespace fuzz { struct Replayer::Impl { - explicit Impl(spv_target_env env) : target_env(env) {} + explicit Impl(spv_target_env env, bool validate) + : target_env(env), validate_during_replay(validate) {} const spv_target_env target_env; // Target environment. MessageConsumer consumer; // Message consumer. + + const bool validate_during_replay; // Controls whether the validator should + // be run after every replay step. }; -Replayer::Replayer(spv_target_env env) : impl_(MakeUnique(env)) {} +Replayer::Replayer(spv_target_env env, bool validate_during_replay) + : impl_(MakeUnique(env, validate_during_replay)) {} Replayer::~Replayer() = default; @@ -80,6 +85,13 @@ Replayer::ReplayerResultStatus Replayer::Run( impl_->target_env, impl_->consumer, binary_in.data(), binary_in.size()); assert(ir_context); + // For replay validation, we track the last valid SPIR-V binary that was + // observed. Initially this is the input binary. + std::vector last_valid_binary; + if (impl_->validate_during_replay) { + last_valid_binary = binary_in; + } + FactManager fact_manager; fact_manager.AddFacts(impl_->consumer, initial_facts, ir_context.get()); @@ -93,6 +105,23 @@ Replayer::ReplayerResultStatus Replayer::Run( // sequence of transformations that were applied. transformation->Apply(ir_context.get(), &fact_manager); *transformation_sequence_out->add_transformation() = message; + + if (impl_->validate_during_replay) { + std::vector binary_to_validate; + ir_context->module()->ToBinary(&binary_to_validate, false); + + // Check whether the latest transformation led to a valid binary. + if (!tools.Validate(&binary_to_validate[0], + binary_to_validate.size())) { + impl_->consumer(SPV_MSG_INFO, nullptr, {}, + "Binary became invalid during replay (set a " + "breakpoint to inspect); stopping."); + return Replayer::ReplayerResultStatus::kReplayValidationFailure; + } + + // The binary was valid, so it becomes the latest valid binary. + last_valid_binary = std::move(binary_to_validate); + } } } diff --git a/source/fuzz/replayer.h b/source/fuzz/replayer.h index 13391d07c..1d58baeb7 100644 --- a/source/fuzz/replayer.h +++ b/source/fuzz/replayer.h @@ -33,10 +33,11 @@ class Replayer { kComplete, kFailedToCreateSpirvToolsInterface, kInitialBinaryInvalid, + kReplayValidationFailure, }; // Constructs a replayer from the given target environment. - explicit Replayer(spv_target_env env); + explicit Replayer(spv_target_env env, bool validate_during_replay); // Disables copy/move constructor/assignment operations. Replayer(const Replayer&) = delete; diff --git a/source/fuzz/shrinker.cpp b/source/fuzz/shrinker.cpp index f8d8aa309..1bb92f100 100644 --- a/source/fuzz/shrinker.cpp +++ b/source/fuzz/shrinker.cpp @@ -60,16 +60,20 @@ protobufs::TransformationSequence RemoveChunk( } // namespace struct Shrinker::Impl { - explicit Impl(spv_target_env env, uint32_t limit) - : target_env(env), step_limit(limit) {} + explicit Impl(spv_target_env env, uint32_t limit, bool validate) + : target_env(env), step_limit(limit), validate_during_replay(validate) {} - const spv_target_env target_env; // Target environment. - MessageConsumer consumer; // Message consumer. - const uint32_t step_limit; // Step limit for reductions. + const spv_target_env target_env; // Target environment. + MessageConsumer consumer; // Message consumer. + const uint32_t step_limit; // Step limit for reductions. + const bool validate_during_replay; // Determines whether to check for + // validity during the replaying of + // transformations. }; -Shrinker::Shrinker(spv_target_env env, uint32_t step_limit) - : impl_(MakeUnique(env, step_limit)) {} +Shrinker::Shrinker(spv_target_env env, uint32_t step_limit, + bool validate_during_replay) + : impl_(MakeUnique(env, step_limit, validate_during_replay)) {} Shrinker::~Shrinker() = default; @@ -109,7 +113,7 @@ Shrinker::ShrinkerResultStatus Shrinker::Run( // succeeds, (b) get the binary that results from running these // transformations, and (c) get the subsequence of the initial transformations // that actually apply (in principle this could be a strict subsequence). - if (Replayer(impl_->target_env) + if (Replayer(impl_->target_env, impl_->validate_during_replay) .Run(binary_in, initial_facts, transformation_sequence_in, ¤t_best_binary, ¤t_best_transformations) != Replayer::ReplayerResultStatus::kComplete) { @@ -180,7 +184,7 @@ Shrinker::ShrinkerResultStatus Shrinker::Run( // transformations inapplicable. std::vector next_binary; protobufs::TransformationSequence next_transformation_sequence; - if (Replayer(impl_->target_env) + if (Replayer(impl_->target_env, false) .Run(binary_in, initial_facts, transformations_with_chunk_removed, &next_binary, &next_transformation_sequence) != Replayer::ReplayerResultStatus::kComplete) { diff --git a/source/fuzz/shrinker.h b/source/fuzz/shrinker.h index 72dd470b0..0163a53ab 100644 --- a/source/fuzz/shrinker.h +++ b/source/fuzz/shrinker.h @@ -50,7 +50,8 @@ class Shrinker { const std::vector& binary, uint32_t counter)>; // Constructs a shrinker from the given target environment. - Shrinker(spv_target_env env, uint32_t step_limit); + Shrinker(spv_target_env env, uint32_t step_limit, + bool validate_during_replay); // Disables copy/move constructor/assignment operations. Shrinker(const Shrinker&) = delete; diff --git a/source/spirv_fuzzer_options.cpp b/source/spirv_fuzzer_options.cpp index 9a2cb9fda..ab8903e1e 100644 --- a/source/spirv_fuzzer_options.cpp +++ b/source/spirv_fuzzer_options.cpp @@ -22,6 +22,7 @@ const uint32_t kDefaultStepLimit = 250; spv_fuzzer_options_t::spv_fuzzer_options_t() : has_random_seed(false), random_seed(0), + replay_validation_enabled(false), shrinker_step_limit(kDefaultStepLimit) {} SPIRV_TOOLS_EXPORT spv_fuzzer_options spvFuzzerOptionsCreate() { @@ -32,6 +33,11 @@ SPIRV_TOOLS_EXPORT void spvFuzzerOptionsDestroy(spv_fuzzer_options options) { delete options; } +SPIRV_TOOLS_EXPORT void spvFuzzerOptionsEnableReplayValidation( + spv_fuzzer_options options) { + options->replay_validation_enabled = true; +} + SPIRV_TOOLS_EXPORT void spvFuzzerOptionsSetRandomSeed( spv_fuzzer_options options, uint32_t seed) { options->has_random_seed = true; diff --git a/source/spirv_fuzzer_options.h b/source/spirv_fuzzer_options.h index 3b8e15ca1..7bb16c738 100644 --- a/source/spirv_fuzzer_options.h +++ b/source/spirv_fuzzer_options.h @@ -29,6 +29,9 @@ struct spv_fuzzer_options_t { bool has_random_seed; uint32_t random_seed; + // See spvFuzzerOptionsEnableReplayValidation. + bool replay_validation_enabled; + // See spvFuzzerOptionsSetShrinkerStepLimit. uint32_t shrinker_step_limit; }; diff --git a/test/fuzz/fuzzer_replayer_test.cpp b/test/fuzz/fuzzer_replayer_test.cpp index 550c4fc8d..3c3660286 100644 --- a/test/fuzz/fuzzer_replayer_test.cpp +++ b/test/fuzz/fuzzer_replayer_test.cpp @@ -54,7 +54,7 @@ void RunFuzzerAndReplayer(const std::string& shader, std::vector replayer_binary_out; protobufs::TransformationSequence replayer_transformation_sequence_out; - Replayer replayer(env); + Replayer replayer(env, true); replayer.SetMessageConsumer(kSilentConsumer); auto replayer_result_status = replayer.Run( binary_in, initial_facts, fuzzer_transformation_sequence_out, diff --git a/test/fuzz/fuzzer_shrinker_test.cpp b/test/fuzz/fuzzer_shrinker_test.cpp index 3796bbfbd..933038dc6 100644 --- a/test/fuzz/fuzzer_shrinker_test.cpp +++ b/test/fuzz/fuzzer_shrinker_test.cpp @@ -125,7 +125,7 @@ void RunAndCheckShrinker( const std::vector& expected_binary_out, uint32_t expected_transformations_out_size, uint32_t step_limit) { // Run the shrinker. - Shrinker shrinker(target_env, step_limit); + Shrinker shrinker(target_env, step_limit, true); shrinker.SetMessageConsumer(kSilentConsumer); std::vector binary_out; diff --git a/tools/fuzz/fuzz.cpp b/tools/fuzz/fuzz.cpp index 24e4ac683..fdea1af17 100644 --- a/tools/fuzz/fuzz.cpp +++ b/tools/fuzz/fuzz.cpp @@ -104,6 +104,11 @@ Options (in lexicographical order): Path to an interestingness function to guide shrinking: a script that returns 0 if and only if a given binary is interesting. Required if --shrink is provided; disallowed otherwise. + --replay-validation + Run the validator after applying each transformation during + replay (including the replay that occurs during shrinking). + Aborts if an invalid binary is created. Useful for debugging + spirv-fuzz. --version Display fuzzer version information. @@ -161,6 +166,9 @@ FuzzStatus ParseFlags(int argc, const char** argv, std::string* in_binary_file, sizeof("--interestingness=") - 1)) { const auto split_flag = spvtools::utils::SplitFlagArgs(cur_arg); *interestingness_function_file = std::string(split_flag.second); + } else if (0 == strncmp(cur_arg, "--replay-validation", + sizeof("--replay-validation") - 1)) { + fuzzer_options->enable_replay_validation(); } else if (0 == strncmp(cur_arg, "--shrink=", sizeof("--shrink=") - 1)) { const auto split_flag = spvtools::utils::SplitFlagArgs(cur_arg); *shrink_transformations_file = std::string(split_flag.second); @@ -221,6 +229,16 @@ FuzzStatus ParseFlags(int argc, const char** argv, std::string* in_binary_file, return {FuzzActions::STOP, 1}; } + if (replay_transformations_file->empty() && + shrink_transformations_file->empty() && + static_cast(*fuzzer_options) + ->replay_validation_enabled) { + spvtools::Error(FuzzDiagnostic, nullptr, {}, + "The --replay-validation argument can only be used with " + "one of the --replay or --shrink arguments."); + return {FuzzActions::STOP, 1}; + } + if (!replay_transformations_file->empty()) { // A replay transformations file was given, thus the tool is being invoked // in replay mode. @@ -278,6 +296,7 @@ bool ParseTransformations( } bool Replay(const spv_target_env& target_env, + spv_const_fuzzer_options fuzzer_options, const std::vector& binary_in, const spvtools::fuzz::protobufs::FactSequence& initial_facts, const std::string& replay_transformations_file, @@ -289,7 +308,8 @@ bool Replay(const spv_target_env& target_env, &transformation_sequence)) { return false; } - spvtools::fuzz::Replayer replayer(target_env); + spvtools::fuzz::Replayer replayer(target_env, + fuzzer_options->replay_validation_enabled); replayer.SetMessageConsumer(spvtools::utils::CLIMessageConsumer); auto replay_result_status = replayer.Run(binary_in, initial_facts, transformation_sequence, @@ -313,7 +333,8 @@ bool Shrink(const spv_target_env& target_env, return false; } spvtools::fuzz::Shrinker shrinker(target_env, - fuzzer_options->shrinker_step_limit); + fuzzer_options->shrinker_step_limit, + fuzzer_options->replay_validation_enabled); shrinker.SetMessageConsumer(spvtools::utils::CLIMessageConsumer); spvtools::fuzz::Shrinker::InterestingnessFunction interestingness_function = @@ -362,6 +383,23 @@ bool Fuzz(const spv_target_env& target_env, } // namespace +// Dumps |binary| to file |filename|. Useful for interactive debugging. +void DumpShader(const std::vector& binary, const char* filename) { + auto write_file_succeeded = + WriteFile(filename, "wb", &binary[0], binary.size()); + if (!write_file_succeeded) { + std::cerr << "Failed to dump shader" << std::endl; + } +} + +// Dumps the SPIRV-V module in |context| to file |filename|. Useful for +// interactive debugging. +void DumpShader(spvtools::opt::IRContext* context, const char* filename) { + std::vector binary; + context->module()->ToBinary(&binary, false); + DumpShader(binary, filename); +} + const auto kDefaultEnvironment = SPV_ENV_UNIVERSAL_1_3; int main(int argc, const char** argv) { @@ -418,7 +456,7 @@ int main(int argc, const char** argv) { } break; case FuzzActions::REPLAY: - if (!Replay(target_env, binary_in, initial_facts, + if (!Replay(target_env, fuzzer_options, binary_in, initial_facts, replay_transformations_file, &binary_out, &transformations_applied)) { return 1;