Stop consuming input in fuzzers to select target environment (#4544)

Instead calculate a hash based on the input and use that as a seed
into random data generation for the target env.

Also fixes issue where input data was not actually being fed into
one fuzzer.

Fixes #4450

source/spirv_target_env.cpp

@@ -286,6 +286,38 @@ bool spvIsOpenGLEnv(spv_target_env env) {
   return false;
 }
 
+bool spvIsValidEnv(spv_target_env env) {
+  switch (env) {
+    case SPV_ENV_UNIVERSAL_1_0:
+    case SPV_ENV_VULKAN_1_0:
+    case SPV_ENV_UNIVERSAL_1_1:
+    case SPV_ENV_UNIVERSAL_1_2:
+    case SPV_ENV_UNIVERSAL_1_3:
+    case SPV_ENV_VULKAN_1_1:
+    case SPV_ENV_OPENCL_1_2:
+    case SPV_ENV_OPENCL_EMBEDDED_1_2:
+    case SPV_ENV_OPENCL_2_0:
+    case SPV_ENV_OPENCL_EMBEDDED_2_0:
+    case SPV_ENV_OPENCL_EMBEDDED_2_1:
+    case SPV_ENV_OPENCL_EMBEDDED_2_2:
+    case SPV_ENV_OPENCL_2_1:
+    case SPV_ENV_OPENCL_2_2:
+    case SPV_ENV_UNIVERSAL_1_4:
+    case SPV_ENV_VULKAN_1_1_SPIRV_1_4:
+    case SPV_ENV_UNIVERSAL_1_5:
+    case SPV_ENV_VULKAN_1_2:
+    case SPV_ENV_OPENGL_4_0:
+    case SPV_ENV_OPENGL_4_1:
+    case SPV_ENV_OPENGL_4_2:
+    case SPV_ENV_OPENGL_4_3:
+    case SPV_ENV_OPENGL_4_5:
+      return true;
+    case SPV_ENV_WEBGPU_0:
+      break;
+  }
+  return false;
+}
+
 std::string spvLogStringForEnv(spv_target_env env) {
   switch (env) {
     case SPV_ENV_OPENCL_1_2:

source/spirv_target_env.h

@@ -28,6 +28,9 @@ bool spvIsOpenCLEnv(spv_target_env env);
 // Returns true if |env| is an OPENGL environment, false otherwise.
 bool spvIsOpenGLEnv(spv_target_env env);
 
+// Returns true if |env| is an implemented/valid environment, false otherwise.
+bool spvIsValidEnv(spv_target_env env);
+
 // Returns the version number for the given SPIR-V target environment.
 uint32_t spvVersionForTargetEnv(spv_target_env env);
 

test/fuzzers/BUILD.gn

@@ -48,6 +48,7 @@ template("spvtools_fuzzer") {
   source_set(target_name) {
     testonly = true
     sources = invoker.sources
+    sources += [ "random_generator.cpp" ]
     deps = [
       "../..:spvtools",
       "../..:spvtools_opt",

test/fuzzers/CMakeLists.txt

@@ -43,11 +43,11 @@ if (${SPIRV_BUILD_LIBFUZZER_TARGETS})
   if(NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
     message(FATAL_ERROR "The libFuzzer targets are only supported with the Clang compiler. Compiler '${CMAKE_CXX_COMPILER_ID}' is not supported!")
   endif()
-  add_spvtools_libfuzzer_target(TARGET spvtools_as_fuzzer SRCS spvtools_as_fuzzer.cpp LIBS ${SPIRV_TOOLS_FULL_VISIBILITY})
-  add_spvtools_libfuzzer_target(TARGET spvtools_binary_parser_fuzzer SRCS spvtools_binary_parser_fuzzer.cpp LIBS ${SPIRV_TOOLS_FULL_VISIBILITY})
-  add_spvtools_libfuzzer_target(TARGET spvtools_dis_fuzzer SRCS spvtools_dis_fuzzer.cpp LIBS ${SPIRV_TOOLS_FULL_VISIBILITY})
-  add_spvtools_libfuzzer_target(TARGET spvtools_opt_legalization_fuzzer SRCS spvtools_opt_legalization_fuzzer.cpp LIBS SPIRV-Tools-opt ${SPIRV_TOOLS_FULL_VISIBILITY})
-  add_spvtools_libfuzzer_target(TARGET spvtools_opt_performance_fuzzer SRCS spvtools_opt_performance_fuzzer.cpp LIBS SPIRV-Tools-opt ${SPIRV_TOOLS_FULL_VISIBILITY})
-  add_spvtools_libfuzzer_target(TARGET spvtools_opt_size_fuzzer SRCS spvtools_opt_size_fuzzer.cpp LIBS SPIRV-Tools-opt ${SPIRV_TOOLS_FULL_VISIBILITY})
-  add_spvtools_libfuzzer_target(TARGET spvtools_val_fuzzer SRCS spvtools_val_fuzzer.cpp LIBS ${SPIRV_TOOLS_FULL_VISIBILITY})
+  add_spvtools_libfuzzer_target(TARGET spvtools_as_fuzzer SRCS spvtools_as_fuzzer.cpp random_generator.cpp LIBS ${SPIRV_TOOLS_FULL_VISIBILITY})
+  add_spvtools_libfuzzer_target(TARGET spvtools_binary_parser_fuzzer SRCS spvtools_binary_parser_fuzzer.cpp random_generator.cpp LIBS ${SPIRV_TOOLS_FULL_VISIBILITY})
+  add_spvtools_libfuzzer_target(TARGET spvtools_dis_fuzzer SRCS spvtools_dis_fuzzer.cpp random_generator.cpp LIBS ${SPIRV_TOOLS_FULL_VISIBILITY})
+  add_spvtools_libfuzzer_target(TARGET spvtools_opt_legalization_fuzzer SRCS spvtools_opt_legalization_fuzzer.cpp random_generator.cpp LIBS SPIRV-Tools-opt ${SPIRV_TOOLS_FULL_VISIBILITY})
+  add_spvtools_libfuzzer_target(TARGET spvtools_opt_performance_fuzzer SRCS spvtools_opt_performance_fuzzer.cpp random_generator.cpp LIBS SPIRV-Tools-opt ${SPIRV_TOOLS_FULL_VISIBILITY})
+  add_spvtools_libfuzzer_target(TARGET spvtools_opt_size_fuzzer SRCS spvtools_opt_size_fuzzer.cpp random_generator.cpp LIBS SPIRV-Tools-opt ${SPIRV_TOOLS_FULL_VISIBILITY})
+  add_spvtools_libfuzzer_target(TARGET spvtools_val_fuzzer SRCS spvtools_val_fuzzer.cpp random_generator.cpp LIBS ${SPIRV_TOOLS_FULL_VISIBILITY})
 endif()

test/fuzzers/random_generator.cpp

@@ -0,0 +1,133 @@
+// Copyright (c) 2021 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test/fuzzers/random_generator.h"
+
+#include <algorithm>
+#include <array>
+#include <cassert>
+
+namespace spvtools {
+namespace fuzzers {
+
+namespace {
+/// Generate integer from uniform distribution
+/// @tparam I - integer type
+/// @param engine - random number engine to use
+/// @param lower - Lower bound of integer generated
+/// @param upper - Upper bound of integer generated
+/// @returns i, where lower <= i < upper
+template <typename I>
+I RandomUInt(std::mt19937* engine, I lower, I upper) {
+  assert(lower < upper && "|lower| must be stictly less than |upper|");
+  return std::uniform_int_distribution<I>(lower, upper - 1)(*engine);
+}
+
+/// Helper for obtaining a seed bias value for HashCombine with a bit-width
+/// dependent on the size of size_t.
+template <int SIZE_OF_SIZE_T>
+struct HashCombineOffset {};
+/// Specialization of HashCombineOffset for size_t == 4.
+template <>
+struct HashCombineOffset<4> {
+  /// @returns the seed bias value for HashCombine()
+  static constexpr inline uint32_t value() {
+    return 0x9e3779b9;  // Fractional portion of Golden Ratio, suggested by
+                        // Linux Kernel and Knuth's Art of Computer Programming
+  }
+};
+/// Specialization of HashCombineOffset for size_t == 8.
+template <>
+struct HashCombineOffset<8> {
+  /// @returns the seed bias value for HashCombine()
+  static constexpr inline uint64_t value() {
+    return 0x9e3779b97f4a7c16;  // Fractional portion of Golden Ratio, suggested
+                                // by Linux Kernel and Knuth's Art of Computer
+                                // Programming
+  }
+};
+
+/// HashCombine "hashes" together an existing hash and hashable values.
+template <typename T>
+void HashCombine(size_t* hash, const T& value) {
+  constexpr size_t offset = HashCombineOffset<sizeof(size_t)>::value();
+  *hash ^= std::hash<T>()(value) + offset + (*hash << 6) + (*hash >> 2);
+}
+
+/// Calculate the hash for the contents of a C-style data buffer
+/// @param data - pointer to buffer to be hashed
+/// @param size - number of elements in buffer
+/// @returns hash of the data in the buffer
+size_t HashBuffer(const uint8_t* data, const size_t size) {
+  size_t hash = 0xCA8945571519E991;  // seed with an arbitrary prime
+  HashCombine(&hash, size);
+  for (size_t i = 0; i < size; i++) {
+    HashCombine(&hash, data[i]);
+  }
+  return hash;
+}
+
+}  // namespace
+
+RandomGenerator::RandomGenerator(uint64_t seed) : engine_(seed) {}
+
+RandomGenerator::RandomGenerator(const uint8_t* data, size_t size) {
+  RandomGenerator(RandomGenerator::CalculateSeed(data, size));
+}
+
+spv_target_env RandomGenerator::GetTargetEnv() {
+  // SPV_ENV_WEBGPU_0 is intentionally omitted here, since it is deprecated and
+  // using it will cause asserts.
+  static const std::array<spv_target_env, 23> envs = {
+      SPV_ENV_UNIVERSAL_1_0,       SPV_ENV_VULKAN_1_0,
+      SPV_ENV_UNIVERSAL_1_1,       SPV_ENV_UNIVERSAL_1_2,
+      SPV_ENV_UNIVERSAL_1_3,       SPV_ENV_VULKAN_1_1,
+      SPV_ENV_OPENCL_1_2,          SPV_ENV_OPENCL_EMBEDDED_1_2,
+      SPV_ENV_OPENCL_2_0,          SPV_ENV_OPENCL_EMBEDDED_2_0,
+      SPV_ENV_OPENCL_EMBEDDED_2_1, SPV_ENV_OPENCL_EMBEDDED_2_2,
+      SPV_ENV_OPENCL_2_1,          SPV_ENV_OPENCL_2_2,
+      SPV_ENV_UNIVERSAL_1_4,       SPV_ENV_VULKAN_1_1_SPIRV_1_4,
+      SPV_ENV_UNIVERSAL_1_5,       SPV_ENV_VULKAN_1_2,
+      SPV_ENV_OPENGL_4_0,          SPV_ENV_OPENGL_4_1,
+      SPV_ENV_OPENGL_4_2,          SPV_ENV_OPENGL_4_3,
+      SPV_ENV_OPENGL_4_5};
+
+  return envs[RandomUInt(&engine_, 0lu, envs.size())];
+}
+
+uint64_t RandomGenerator::CalculateSeed(const uint8_t* data, size_t size) {
+  assert(data != nullptr && "|data| must be !nullptr");
+
+  // Number of bytes we want to skip at the start of data for the hash.
+  // Fewer bytes may be skipped when `size` is small.
+  // Has lower precedence than kHashDesiredMinBytes.
+  static const int64_t kHashDesiredLeadingSkipBytes = 5;
+  // Minimum number of bytes we want to use in the hash.
+  // Used for short buffers.
+  static const int64_t kHashDesiredMinBytes = 4;
+  // Maximum number of bytes we want to use in the hash.
+  static const int64_t kHashDesiredMaxBytes = 32;
+  int64_t size_i64 = static_cast<int64_t>(size);
+  int64_t hash_begin_i64 =
+      std::min(kHashDesiredLeadingSkipBytes,
+               std::max<int64_t>(size_i64 - kHashDesiredMinBytes, 0));
+  int64_t hash_end_i64 =
+      std::min(hash_begin_i64 + kHashDesiredMaxBytes, size_i64);
+  size_t hash_begin = static_cast<size_t>(hash_begin_i64);
+  size_t hash_size = static_cast<size_t>(hash_end_i64) - hash_begin;
+  return HashBuffer(data + hash_begin, hash_size);
+}
+
+}  // namespace fuzzers
+}  // namespace spvtools

test/fuzzers/random_generator.h

@@ -0,0 +1,57 @@
+// Copyright (c) 2021 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TEST_FUZZERS_RANDOM_GENERATOR_H_
+#define TEST_FUZZERS_RANDOM_GENERATOR_H_
+
+#include <cstdint>
+#include <random>
+
+#include "source/spirv_target_env.h"
+
+namespace spvtools {
+namespace fuzzers {
+
+/// Pseudo random generator utility class for fuzzing
+class RandomGenerator {
+ public:
+  /// @brief Initializes the internal engine
+  /// @param seed - seed value passed to engine
+  explicit RandomGenerator(uint64_t seed);
+
+  /// @brief Initializes the internal engine
+  /// @param data - data to calculate the seed from
+  /// @param size - size of the data
+  explicit RandomGenerator(const uint8_t* data, size_t size);
+
+  ~RandomGenerator() {}
+
+  /// Calculate a seed value based on a blob of data.
+  /// Currently hashes bytes near the front of the buffer, after skipping N
+  /// bytes.
+  /// @param data - pointer to data to base calculation off of, must be !nullptr
+  /// @param size - number of elements in |data|, must be > 0
+  static uint64_t CalculateSeed(const uint8_t* data, size_t size);
+
+  /// Get random target env.
+  spv_target_env GetTargetEnv();
+
+ private:
+  std::mt19937 engine_;
+};  // class RandomGenerator
+
+}  // namespace fuzzers
+}  // namespace spvtools
+
+#endif  // TEST_FUZZERS_RANDOM_GENERATOR_UTILS_H_

test/fuzzers/spvtools_as_fuzzer.cpp

@@ -18,18 +18,27 @@
 
 #include "source/spirv_target_env.h"
 #include "spirv-tools/libspirv.hpp"
+#include "test/fuzzers/random_generator.h"
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
-  if (size < sizeof(spv_target_env) + 1) return 0;
+  spv_target_env target_env = SPV_ENV_UNIVERSAL_1_0;
+  if (size > 0) {
+    spvtools::fuzzers::RandomGenerator random_gen(data, size);
+    target_env = random_gen.GetTargetEnv();
+  }
 
-  const spv_context context =
-      spvContextCreate(*reinterpret_cast<const spv_target_env*>(data));
-  if (context == nullptr) return 0;
-
-  data += sizeof(spv_target_env);
-  size -= sizeof(spv_target_env);
+  const spv_context context = spvContextCreate(target_env);
+  if (context == nullptr) {
+    return 0;
+  }
 
   std::vector<uint32_t> input;
+  input.resize(size >> 2);
+  size_t count = 0;
+  for (size_t i = 0; (i + 3) < size; i += 4) {
+    input[count++] = data[i] | (data[i + 1] << 8) | (data[i + 2] << 16) |
+                     (data[i + 3]) << 24;
+  }
 
   std::vector<char> input_str;
   size_t char_count = input.size() * sizeof(uint32_t) / sizeof(char);

test/fuzzers/spvtools_binary_parser_fuzzer.cpp

@@ -16,16 +16,18 @@
 #include <vector>
 
 #include "spirv-tools/libspirv.hpp"
+#include "test/fuzzers/random_generator.h"
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
-  if (size < sizeof(spv_target_env) + 1) return 0;
+  if (size < 1) {
+    return 0;
+  }
 
-  const spv_context context =
-      spvContextCreate(*reinterpret_cast<const spv_target_env*>(data));
-  if (context == nullptr) return 0;
-
-  data += sizeof(spv_target_env);
-  size -= sizeof(spv_target_env);
+  spvtools::fuzzers::RandomGenerator random_gen(data, size);
+  const spv_context context = spvContextCreate(random_gen.GetTargetEnv());
+  if (context == nullptr) {
+    return 0;
+  }
 
   std::vector<uint32_t> input;
   input.resize(size >> 2);

test/fuzzers/spvtools_dis_fuzzer.cpp

@@ -18,28 +18,21 @@
 
 #include "source/spirv_target_env.h"
 #include "spirv-tools/libspirv.hpp"
+#include "test/fuzzers/random_generator.h"
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
-  if (size < sizeof(spv_target_env) + 1) return 0;
-
-  // TODO(https://github.com/KhronosGroup/SPIRV-Tools/issues/4450): A more
-  //  general solution to choosing the target environment based on the input
-  //  buffer should ultimately be used.
-  uint32_t first_data_word = *reinterpret_cast<const uint32_t*>(data);
-  spv_target_env target_env = static_cast<spv_target_env>(
-      first_data_word % (static_cast<uint32_t>(SPV_ENV_VULKAN_1_2) + 1));
-  const spv_context context = spvContextCreate(target_env);
-  if (context == nullptr) return 0;
-
-  data += sizeof(spv_target_env);
-  size -= sizeof(spv_target_env);
-
   if (size < 4) {
     // There are not enough bytes to constitute a binary that can be
     // disassembled.
     return 0;
   }
 
+  spvtools::fuzzers::RandomGenerator random_gen(data, size);
+  const spv_context context = spvContextCreate(random_gen.GetTargetEnv());
+  if (context == nullptr) {
+    return 0;
+  }
+
   std::vector<uint32_t> input;
   input.resize(size >> 2);
   size_t count = 0;

test/fuzzers/spvtools_opt_legalization_fuzzer.cpp

@@ -16,9 +16,15 @@
 #include <vector>
 
 #include "spirv-tools/optimizer.hpp"
+#include "test/fuzzers/random_generator.h"
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
-  spvtools::Optimizer optimizer(SPV_ENV_UNIVERSAL_1_3);
+  if (size < 1) {
+    return 0;
+  }
+
+  spvtools::fuzzers::RandomGenerator random_gen(data, size);
+  spvtools::Optimizer optimizer(random_gen.GetTargetEnv());
   optimizer.SetMessageConsumer([](spv_message_level_t, const char*,
                                   const spv_position_t&, const char*) {});
 

test/fuzzers/spvtools_opt_performance_fuzzer.cpp

@@ -16,9 +16,15 @@
 #include <vector>
 
 #include "spirv-tools/optimizer.hpp"
+#include "test/fuzzers/random_generator.h"
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
-  spvtools::Optimizer optimizer(SPV_ENV_UNIVERSAL_1_3);
+  if (size < 1) {
+    return 0;
+  }
+
+  spvtools::fuzzers::RandomGenerator random_gen(data, size);
+  spvtools::Optimizer optimizer(random_gen.GetTargetEnv());
   optimizer.SetMessageConsumer([](spv_message_level_t, const char*,
                                   const spv_position_t&, const char*) {});
 

test/fuzzers/spvtools_opt_size_fuzzer.cpp

@@ -16,9 +16,15 @@
 #include <vector>
 
 #include "spirv-tools/optimizer.hpp"
+#include "test/fuzzers/random_generator.h"
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
-  spvtools::Optimizer optimizer(SPV_ENV_UNIVERSAL_1_3);
+  if (size < 1) {
+    return 0;
+  }
+
+  spvtools::fuzzers::RandomGenerator random_gen(data, size);
+  spvtools::Optimizer optimizer(random_gen.GetTargetEnv());
   optimizer.SetMessageConsumer([](spv_message_level_t, const char*,
                                   const spv_position_t&, const char*) {});
 

test/fuzzers/spvtools_val_fuzzer.cpp

@@ -16,9 +16,15 @@
 #include <vector>
 
 #include "spirv-tools/libspirv.hpp"
+#include "test/fuzzers/random_generator.h"
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
-  spvtools::SpirvTools tools(SPV_ENV_UNIVERSAL_1_3);
+  if (size < 1) {
+    return 0;
+  }
+
+  spvtools::fuzzers::RandomGenerator random_gen(data, size);
+  spvtools::SpirvTools tools(random_gen.GetTargetEnv());
   tools.SetMessageConsumer([](spv_message_level_t, const char*,
                               const spv_position_t&, const char*) {});