Allow validation during spirv-fuzz replay (#2873)

To aid in debugging issues in spirv-fuzz, this change adds an option whereby the SPIR-V module is validated after each transformation is applied during replay.  This can assist in finding a transformation that erroneously makes the module invalid, so that said transformation can be debugged.

include/spirv-tools/libspirv.h

@@ -612,6 +612,11 @@ SPIRV_TOOLS_EXPORT spv_fuzzer_options spvFuzzerOptionsCreate();
 // Destroys the given fuzzer options object.
 SPIRV_TOOLS_EXPORT void spvFuzzerOptionsDestroy(spv_fuzzer_options options);
 
+// Enables running the validator after every transformation is applied during
+// a replay.
+SPIRV_TOOLS_EXPORT void spvFuzzerOptionsEnableReplayValidation(
+    spv_fuzzer_options options);
+
 // Sets the seed with which the random number generator used by the fuzzer
 // should be initialized.
 SPIRV_TOOLS_EXPORT void spvFuzzerOptionsSetRandomSeed(

include/spirv-tools/libspirv.hpp

@@ -214,6 +214,11 @@ class FuzzerOptions {
     return options_;
   }
 
+  // See spvFuzzerOptionsEnableReplayValidation.
+  void enable_replay_validation() {
+    spvFuzzerOptionsEnableReplayValidation(options_);
+  }
+
   // See spvFuzzerOptionsSetRandomSeed.
   void set_random_seed(uint32_t seed) {
     spvFuzzerOptionsSetRandomSeed(options_, seed);

source/fuzz/replayer.cpp

@@ -37,13 +37,18 @@ namespace spvtools {
 namespace fuzz {
 
 struct Replayer::Impl {
-  explicit Impl(spv_target_env env) : target_env(env) {}
+  explicit Impl(spv_target_env env, bool validate)
+      : target_env(env), validate_during_replay(validate) {}
 
   const spv_target_env target_env;  // Target environment.
   MessageConsumer consumer;         // Message consumer.
+
+  const bool validate_during_replay;  // Controls whether the validator should
+                                      // be run after every replay step.
 };
 
-Replayer::Replayer(spv_target_env env) : impl_(MakeUnique<Impl>(env)) {}
+Replayer::Replayer(spv_target_env env, bool validate_during_replay)
+    : impl_(MakeUnique<Impl>(env, validate_during_replay)) {}
 
 Replayer::~Replayer() = default;
 
@@ -80,6 +85,13 @@ Replayer::ReplayerResultStatus Replayer::Run(
       impl_->target_env, impl_->consumer, binary_in.data(), binary_in.size());
   assert(ir_context);
 
+  // For replay validation, we track the last valid SPIR-V binary that was
+  // observed. Initially this is the input binary.
+  std::vector<uint32_t> last_valid_binary;
+  if (impl_->validate_during_replay) {
+    last_valid_binary = binary_in;
+  }
+
   FactManager fact_manager;
   fact_manager.AddFacts(impl_->consumer, initial_facts, ir_context.get());
 
@@ -93,6 +105,23 @@ Replayer::ReplayerResultStatus Replayer::Run(
       // sequence of transformations that were applied.
       transformation->Apply(ir_context.get(), &fact_manager);
       *transformation_sequence_out->add_transformation() = message;
+
+      if (impl_->validate_during_replay) {
+        std::vector<uint32_t> binary_to_validate;
+        ir_context->module()->ToBinary(&binary_to_validate, false);
+
+        // Check whether the latest transformation led to a valid binary.
+        if (!tools.Validate(&binary_to_validate[0],
+                            binary_to_validate.size())) {
+          impl_->consumer(SPV_MSG_INFO, nullptr, {},
+                          "Binary became invalid during replay (set a "
+                          "breakpoint to inspect); stopping.");
+          return Replayer::ReplayerResultStatus::kReplayValidationFailure;
+        }
+
+        // The binary was valid, so it becomes the latest valid binary.
+        last_valid_binary = std::move(binary_to_validate);
+      }
     }
   }
 

source/fuzz/replayer.h

@@ -33,10 +33,11 @@ class Replayer {
     kComplete,
     kFailedToCreateSpirvToolsInterface,
     kInitialBinaryInvalid,
+    kReplayValidationFailure,
   };
 
   // Constructs a replayer from the given target environment.
-  explicit Replayer(spv_target_env env);
+  explicit Replayer(spv_target_env env, bool validate_during_replay);
 
   // Disables copy/move constructor/assignment operations.
   Replayer(const Replayer&) = delete;

source/fuzz/shrinker.cpp

@@ -60,16 +60,20 @@ protobufs::TransformationSequence RemoveChunk(
 }  // namespace
 
 struct Shrinker::Impl {
-  explicit Impl(spv_target_env env, uint32_t limit)
-      : target_env(env), step_limit(limit) {}
+  explicit Impl(spv_target_env env, uint32_t limit, bool validate)
+      : target_env(env), step_limit(limit), validate_during_replay(validate) {}
 
-  const spv_target_env target_env;  // Target environment.
-  MessageConsumer consumer;         // Message consumer.
-  const uint32_t step_limit;        // Step limit for reductions.
+  const spv_target_env target_env;    // Target environment.
+  MessageConsumer consumer;           // Message consumer.
+  const uint32_t step_limit;          // Step limit for reductions.
+  const bool validate_during_replay;  // Determines whether to check for
+                                      // validity during the replaying of
+                                      // transformations.
 };
 
-Shrinker::Shrinker(spv_target_env env, uint32_t step_limit)
-    : impl_(MakeUnique<Impl>(env, step_limit)) {}
+Shrinker::Shrinker(spv_target_env env, uint32_t step_limit,
+                   bool validate_during_replay)
+    : impl_(MakeUnique<Impl>(env, step_limit, validate_during_replay)) {}
 
 Shrinker::~Shrinker() = default;
 
@@ -109,7 +113,7 @@ Shrinker::ShrinkerResultStatus Shrinker::Run(
   // succeeds, (b) get the binary that results from running these
   // transformations, and (c) get the subsequence of the initial transformations
   // that actually apply (in principle this could be a strict subsequence).
-  if (Replayer(impl_->target_env)
+  if (Replayer(impl_->target_env, impl_->validate_during_replay)
           .Run(binary_in, initial_facts, transformation_sequence_in,
                &current_best_binary, &current_best_transformations) !=
       Replayer::ReplayerResultStatus::kComplete) {
@@ -180,7 +184,7 @@ Shrinker::ShrinkerResultStatus Shrinker::Run(
       // transformations inapplicable.
       std::vector<uint32_t> next_binary;
       protobufs::TransformationSequence next_transformation_sequence;
-      if (Replayer(impl_->target_env)
+      if (Replayer(impl_->target_env, false)
               .Run(binary_in, initial_facts, transformations_with_chunk_removed,
                    &next_binary, &next_transformation_sequence) !=
           Replayer::ReplayerResultStatus::kComplete) {

source/fuzz/shrinker.h

@@ -50,7 +50,8 @@ class Shrinker {
       const std::vector<uint32_t>& binary, uint32_t counter)>;
 
   // Constructs a shrinker from the given target environment.
-  Shrinker(spv_target_env env, uint32_t step_limit);
+  Shrinker(spv_target_env env, uint32_t step_limit,
+           bool validate_during_replay);
 
   // Disables copy/move constructor/assignment operations.
   Shrinker(const Shrinker&) = delete;

source/spirv_fuzzer_options.cpp

@@ -22,6 +22,7 @@ const uint32_t kDefaultStepLimit = 250;
 spv_fuzzer_options_t::spv_fuzzer_options_t()
     : has_random_seed(false),
       random_seed(0),
+      replay_validation_enabled(false),
       shrinker_step_limit(kDefaultStepLimit) {}
 
 SPIRV_TOOLS_EXPORT spv_fuzzer_options spvFuzzerOptionsCreate() {
@@ -32,6 +33,11 @@ SPIRV_TOOLS_EXPORT void spvFuzzerOptionsDestroy(spv_fuzzer_options options) {
   delete options;
 }
 
+SPIRV_TOOLS_EXPORT void spvFuzzerOptionsEnableReplayValidation(
+    spv_fuzzer_options options) {
+  options->replay_validation_enabled = true;
+}
+
 SPIRV_TOOLS_EXPORT void spvFuzzerOptionsSetRandomSeed(
     spv_fuzzer_options options, uint32_t seed) {
   options->has_random_seed = true;

source/spirv_fuzzer_options.h

@@ -29,6 +29,9 @@ struct spv_fuzzer_options_t {
   bool has_random_seed;
   uint32_t random_seed;
 
+  // See spvFuzzerOptionsEnableReplayValidation.
+  bool replay_validation_enabled;
+
   // See spvFuzzerOptionsSetShrinkerStepLimit.
   uint32_t shrinker_step_limit;
 };

test/fuzz/fuzzer_replayer_test.cpp

@@ -54,7 +54,7 @@ void RunFuzzerAndReplayer(const std::string& shader,
     std::vector<uint32_t> replayer_binary_out;
     protobufs::TransformationSequence replayer_transformation_sequence_out;
 
-    Replayer replayer(env);
+    Replayer replayer(env, true);
     replayer.SetMessageConsumer(kSilentConsumer);
     auto replayer_result_status = replayer.Run(
         binary_in, initial_facts, fuzzer_transformation_sequence_out,

test/fuzz/fuzzer_shrinker_test.cpp

@@ -125,7 +125,7 @@ void RunAndCheckShrinker(
     const std::vector<uint32_t>& expected_binary_out,
     uint32_t expected_transformations_out_size, uint32_t step_limit) {
   // Run the shrinker.
-  Shrinker shrinker(target_env, step_limit);
+  Shrinker shrinker(target_env, step_limit, true);
   shrinker.SetMessageConsumer(kSilentConsumer);
 
   std::vector<uint32_t> binary_out;

tools/fuzz/fuzz.cpp

@@ -104,6 +104,11 @@ Options (in lexicographical order):
                Path to an interestingness function to guide shrinking: a script
                that returns 0 if and only if a given binary is interesting.
                Required if --shrink is provided; disallowed otherwise.
+  --replay-validation
+               Run the validator after applying each transformation during
+               replay (including the replay that occurs during shrinking).
+               Aborts if an invalid binary is created.  Useful for debugging
+               spirv-fuzz.
   --version
                Display fuzzer version information.
 
@@ -161,6 +166,9 @@ FuzzStatus ParseFlags(int argc, const char** argv, std::string* in_binary_file,
                               sizeof("--interestingness=") - 1)) {
         const auto split_flag = spvtools::utils::SplitFlagArgs(cur_arg);
         *interestingness_function_file = std::string(split_flag.second);
+      } else if (0 == strncmp(cur_arg, "--replay-validation",
+                              sizeof("--replay-validation") - 1)) {
+        fuzzer_options->enable_replay_validation();
       } else if (0 == strncmp(cur_arg, "--shrink=", sizeof("--shrink=") - 1)) {
         const auto split_flag = spvtools::utils::SplitFlagArgs(cur_arg);
         *shrink_transformations_file = std::string(split_flag.second);
@@ -221,6 +229,16 @@ FuzzStatus ParseFlags(int argc, const char** argv, std::string* in_binary_file,
     return {FuzzActions::STOP, 1};
   }
 
+  if (replay_transformations_file->empty() &&
+      shrink_transformations_file->empty() &&
+      static_cast<spv_const_fuzzer_options>(*fuzzer_options)
+          ->replay_validation_enabled) {
+    spvtools::Error(FuzzDiagnostic, nullptr, {},
+                    "The --replay-validation argument can only be used with "
+                    "one of the --replay or --shrink arguments.");
+    return {FuzzActions::STOP, 1};
+  }
+
   if (!replay_transformations_file->empty()) {
     // A replay transformations file was given, thus the tool is being invoked
     // in replay mode.
@@ -278,6 +296,7 @@ bool ParseTransformations(
 }
 
 bool Replay(const spv_target_env& target_env,
+            spv_const_fuzzer_options fuzzer_options,
             const std::vector<uint32_t>& binary_in,
             const spvtools::fuzz::protobufs::FactSequence& initial_facts,
             const std::string& replay_transformations_file,
@@ -289,7 +308,8 @@ bool Replay(const spv_target_env& target_env,
                             &transformation_sequence)) {
     return false;
   }
-  spvtools::fuzz::Replayer replayer(target_env);
+  spvtools::fuzz::Replayer replayer(target_env,
+                                    fuzzer_options->replay_validation_enabled);
   replayer.SetMessageConsumer(spvtools::utils::CLIMessageConsumer);
   auto replay_result_status =
       replayer.Run(binary_in, initial_facts, transformation_sequence,
@@ -313,7 +333,8 @@ bool Shrink(const spv_target_env& target_env,
     return false;
   }
   spvtools::fuzz::Shrinker shrinker(target_env,
-                                    fuzzer_options->shrinker_step_limit);
+                                    fuzzer_options->shrinker_step_limit,
+                                    fuzzer_options->replay_validation_enabled);
   shrinker.SetMessageConsumer(spvtools::utils::CLIMessageConsumer);
 
   spvtools::fuzz::Shrinker::InterestingnessFunction interestingness_function =
@@ -362,6 +383,23 @@ bool Fuzz(const spv_target_env& target_env,
 
 }  // namespace
 
+// Dumps |binary| to file |filename|. Useful for interactive debugging.
+void DumpShader(const std::vector<uint32_t>& binary, const char* filename) {
+  auto write_file_succeeded =
+      WriteFile(filename, "wb", &binary[0], binary.size());
+  if (!write_file_succeeded) {
+    std::cerr << "Failed to dump shader" << std::endl;
+  }
+}
+
+// Dumps the SPIRV-V module in |context| to file |filename|. Useful for
+// interactive debugging.
+void DumpShader(spvtools::opt::IRContext* context, const char* filename) {
+  std::vector<uint32_t> binary;
+  context->module()->ToBinary(&binary, false);
+  DumpShader(binary, filename);
+}
+
 const auto kDefaultEnvironment = SPV_ENV_UNIVERSAL_1_3;
 
 int main(int argc, const char** argv) {
@@ -418,7 +456,7 @@ int main(int argc, const char** argv) {
       }
       break;
     case FuzzActions::REPLAY:
-      if (!Replay(target_env, binary_in, initial_facts,
+      if (!Replay(target_env, fuzzer_options, binary_in, initial_facts,
                   replay_transformations_file, &binary_out,
                   &transformations_applied)) {
         return 1;