[js-fuzzer] Make mutation composition more flexible

Before this change, the mutation order of js-fuzzer was hard-coded
and always executed in the same order. This e.g. prevents certain
mutation interactions. E.g. in the typical V8 pattern:

%Prep(foo);foo(N);foo(N);%Opt(foo);foo(N);

This gets typically inserted by the FunctionCallMutator, but none
of the arguments N would get mutated later, since e.g. the
NumberMutator is executed earlier.

This change adds an experiment that makes the top-level mutation
flow more flexible. With a probability of 20% each we now also:
- Shuffle the different mutators.
- Run a few random extra mutators after the first round.

We annotate the output files with comments if the experiment was
chosen to easier analyze later if interesting new bugs were found.

Change-Id: I581d43b41a8e1d87ff1e8cab435a1b6e834db0f1
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/4096477
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Liviu Rau <liviurau@chromium.org>
Cr-Commit-Position: refs/heads/main@{#84863}

tools/clusterfuzz/js_fuzzer/script_mutator.js

@@ -13,6 +13,7 @@ const path = require('path');
 
 const common = require('./mutators/common.js');
 const db = require('./db.js');
+const random = require('./random.js');
 const sourceHelpers = require('./source_helpers.js');
 
 const { AddTryCatchMutator } = require('./mutators/try_catch.js');
@@ -26,6 +27,8 @@ const { ObjectMutator } = require('./mutators/object_mutator.js');
 const { VariableMutator } = require('./mutators/variable_mutator.js');
 const { VariableOrObjectMutator } = require('./mutators/variable_or_object_mutation.js');
 
+const MAX_EXTRA_MUTATIONS = 5;
+
 function defaultSettings() {
   return {
     ADD_VAR_OR_OBJ_MUTATIONS: 0.1,
@@ -38,6 +41,8 @@ function defaultSettings() {
     MUTATE_NUMBERS: 0.05,
     MUTATE_OBJECTS: 0.1,
     MUTATE_VARIABLES: 0.075,
+    SCRIPT_MUTATOR_EXTRA_MUTATIONS: 0.2,
+    SCRIPT_MUTATOR_SHUFFLE: 0.2,
   };
 }
 
@@ -61,8 +66,9 @@ class ScriptMutator {
       new ExpressionMutator(settings),
       new FunctionCallMutator(settings),
       new VariableOrObjectMutator(settings),
-      new AddTryCatchMutator(settings),
     ];
+    this.trycatch = new AddTryCatchMutator(settings);
+    this.settings = settings;
   }
 
   _addMjsunitIfNeeded(dependencies, input) {
@@ -129,9 +135,31 @@ class ScriptMutator {
   }
 
   mutate(source) {
-    for (const mutator of this.mutators) {
+    let mutators = this.mutators.slice();
+    let annotations = [];
+    if (random.choose(this.settings.SCRIPT_MUTATOR_SHUFFLE)){
+      annotations.push(' Script mutator: using shuffled mutators');
+      random.shuffle(mutators);
+    }
+
+    if (random.choose(this.settings.SCRIPT_MUTATOR_EXTRA_MUTATIONS)){
+      for (let i = random.randInt(1, MAX_EXTRA_MUTATIONS); i > 0; i--) {
+        let mutator = random.single(this.mutators);
+        mutators.push(mutator);
+        annotations.push(` Script mutator: extra ${mutator.constructor.name}`);
+      }
+    }
+
+    // Try-catch wrapping should always be the last mutation.
+    mutators.push(this.trycatch);
+
+    for (const mutator of mutators) {
       mutator.mutate(source);
     }
+
+    for (const annotation of annotations.reverse()) {
+      sourceHelpers.annotateWithComment(source.ast, annotation);
+    }
   }
 
   // Returns parsed dependencies for inputs.

tools/clusterfuzz/js_fuzzer/source_helpers.js

@@ -374,14 +374,21 @@ function cleanAsserts(ast) {
   });
 }
 
+/**
+ * Annotate code with top-level comment.
+ */
+function annotateWithComment(ast, comment) {
+  if (ast.program && ast.program.body && ast.program.body.length > 0) {
+    babelTypes.addComment(
+        ast.program.body[0], 'leading', comment, true);
+  }
+}
+
 /**
  * Annotate code with original file path.
  */
 function annotateWithOriginalPath(ast, relPath) {
-  if (ast.program && ast.program.body && ast.program.body.length > 0) {
-    babelTypes.addComment(
-        ast.program.body[0], 'leading', ' Original: ' + relPath, true);
-  }
+  annotateWithComment(ast, ' Original: ' + relPath);
 }
 
 // TODO(machenbach): Move this into the V8 corpus. Other test suites don't
@@ -449,6 +456,7 @@ function generateCode(source, dependencies=[]) {
 module.exports = {
   BABYLON_OPTIONS: BABYLON_OPTIONS,
   BABYLON_REPLACE_VAR_OPTIONS: BABYLON_REPLACE_VAR_OPTIONS,
+  annotateWithComment: annotateWithComment,
   generateCode: generateCode,
   loadDependencyAbs: loadDependencyAbs,
   loadResource: loadResource,

tools/clusterfuzz/js_fuzzer/test/test_mutation_order.js

@@ -0,0 +1,56 @@
+// Copyright 2022 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+/**
+ * @fileoverview Test shuffling mutators and extra mutations.
+ *
+ * Use minimal probability settings to demonstrate order changes of top-level
+ * mutators. Which mutations are used exactly is not relevant to the test and
+ * handled pseudo-randomly.
+ */
+
+'use strict';
+
+const sinon = require('sinon');
+
+const helpers = require('./helpers.js');
+const scriptMutator = require('../script_mutator.js');
+const sourceHelpers = require('../source_helpers.js');
+const random = require('../random.js');
+
+const sandbox = sinon.createSandbox();
+
+describe('Toplevel mutations', () => {
+  afterEach(() => {
+    sandbox.restore();
+  });
+
+  it('shuffle their order', () => {
+    // Make random operations deterministic.
+    helpers.deterministicRandom(sandbox);
+
+    this.settings = {
+      ADD_VAR_OR_OBJ_MUTATIONS: 0.0,
+      MUTATE_CROSSOVER_INSERT: 0.0,
+      MUTATE_EXPRESSIONS: 0.0,
+      MUTATE_FUNCTION_CALLS: 1.0,
+      MUTATE_NUMBERS: 1.0,
+      MUTATE_VARIABLES: 0.0,
+      SCRIPT_MUTATOR_SHUFFLE: 1.0,
+      SCRIPT_MUTATOR_EXTRA_MUTATIONS: 1.0,
+      engine: 'V8',
+      testing: true,
+    };
+
+    const source = helpers.loadTestData('mutation_order/input.js');
+    const mutator = new scriptMutator.ScriptMutator(this.settings, helpers.DB_DIR);
+    const mutated = mutator.mutateInputs([source]);
+    const code = sourceHelpers.generateCode(mutated);
+
+    // The test data should be rich enough to produce a pattern from the
+    // FunctionCallMutator that afterwards gets mutated by the NumberMutator.
+    helpers.assertExpectedResult(
+        'mutation_order/output_expected.js', code);
+  });
+});

tools/clusterfuzz/js_fuzzer/test_data/mutation_order/input.js

@@ -0,0 +1,23 @@
+// Copyright 2022 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+var i = 1;
+var j = 'str';
+var k = undefined;
+var l = {0: 1};
+
+function foo(a, b) {
+  return a + b;
+}
+
+foo(i, 3);
+
+function bar(a) {
+  return foo(a, a);
+}
+
+foo('foo', j);
+bar(2, foo(i, j));
+foo(i, j);
+bar(j, 3);

tools/clusterfuzz/js_fuzzer/test_data/mutation_order/output_expected.js

@@ -0,0 +1,99 @@
+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Script mutator: using shuffled mutators
+// Script mutator: extra ArrayMutator
+// Script mutator: extra VariableMutator
+// Script mutator: extra ExpressionMutator
+// Script mutator: extra ArrayMutator
+
+// Original: mutation_order/input.js
+try {
+  var __v_0 =
+  /* NumberMutator: Replaced 1 with 17 */
+  17;
+} catch (e) {}
+
+try {
+  var __v_1 = 'str';
+} catch (e) {}
+
+try {
+  var __v_2 = undefined;
+} catch (e) {}
+
+try {
+  var __v_3 = {
+    /* NumberMutator: Replaced 0 with 5 */
+    5:
+    /* NumberMutator: Replaced 1 with 13 */
+    13
+  };
+} catch (e) {}
+
+function __f_0(__v_4, __v_5) {
+  return __v_4 + __v_5;
+}
+
+try {
+  %PrepareFunctionForOptimization(__f_0);
+} catch (e) {}
+
+try {
+  __f_0(__v_0,
+  /* NumberMutator: Replaced 3 with -77 */
+  -77);
+} catch (e) {}
+
+try {
+  __f_0(__v_0,
+  /* NumberMutator: Replaced 3 with 12 */
+  12);
+} catch (e) {}
+
+try {
+  %OptimizeFunctionOnNextCall(__f_0);
+} catch (e) {}
+
+try {
+  /* FunctionCallMutator: Optimizing __f_0 */
+  __f_0(__v_0,
+  /* NumberMutator: Replaced 3 with -7 */
+  -7);
+} catch (e) {}
+
+function __f_1(__v_6) {
+  return (
+    /* FunctionCallMutator: Replaced __f_0 with __f_0 */
+    __f_0(__v_6, __v_6)
+  );
+}
+
+try {
+  /* FunctionCallMutator: Replaced __f_0 with __f_0 */
+  __f_0('foo', __v_1);
+} catch (e) {}
+
+try {
+  /* FunctionCallMutator: Compiling baseline __f_1 */
+  %CompileBaseline(__f_1);
+} catch (e) {}
+
+try {
+  __f_1(
+  /* NumberMutator: Replaced 2 with -13 */
+  -13, __f_0(__v_0, __v_1));
+} catch (e) {}
+
+try {
+  /* FunctionCallMutator: Replaced __f_0 with __f_1 */
+  __f_1(__v_0, __v_1);
+} catch (e) {}
+
+try {
+  /* FunctionCallMutator: Replaced __f_1 with __f_1 */
+  __f_1(__v_1,
+  /* NumberMutator: Replaced 3 with 7 */
+  7);
+} catch (e) {}

tools/clusterfuzz/js_fuzzer/tools/run_one.py

@@ -56,9 +56,10 @@ def run(fuzz_file, flag_file):
   cmd = ' '.join(args)
   try:
     output = subprocess.check_output(cmd, stderr=subprocess.PIPE, shell=True)
-    return (cmd, output)
+    return (cmd, output.decode('utf-8'))
   except Exception as e:
-    return (cmd, e.output)
+    return (cmd, e.output.decode('utf-8'))
+
 
 def list_tests():
   """Iterates all fuzz tests and corresponding flags in the given base dir."""