[turbofan] Initial support for polymorphic inlining.

For call sites where the target is not a known constant, but potentially
a list of known constants (i.e. a Phi with all HeapConstant inputs), we
still record the call site as a potential candidate for inlining.
In case the heuristic picks that candidate for inlining, we
expand the call site to a dispatched call site and invoke the
actual inlining logic for all the nested call sites.

Like Crankshaft, we currently allow up to 4 targets for polymorphic inlining,
although we might want to refine that later.

This approach is different from what Crankshaft does in
that we don't duplicate the evaluation of the parameters per polymorphic
case. Instead we first perform the load of the target (which usually
dispatches based on the receiver map), then we evaluate all the
parameters, and then we dispatch again based on the known targets. This
might generate better or worse code compared to what Crankshaft does,
and for the cases where we generate worse code (i.e. because we have
only trivial parameters or no parameters at all), we might want to
investigate optimizing away the double dispatch in the
future.

R=mvstanton@chromium.org
BUG=v8:5267,v8:5365

Review-Url: https://codereview.chromium.org/2325943002
Cr-Commit-Position: refs/heads/master@{#39302}

src/compilation-info.cc

@@ -51,7 +51,6 @@ CompilationInfo::CompilationInfo(ParseInfo* parse_info,
   if (isolate_->serializer_enabled()) EnableDeoptimizationSupport();
 
   if (FLAG_function_context_specialization) MarkAsFunctionContextSpecializing();
-  if (FLAG_turbo_inlining) MarkAsInliningEnabled();
   if (FLAG_turbo_source_positions) MarkAsSourcePositionsEnabled();
   if (FLAG_turbo_splitting) MarkAsSplittingEnabled();
 }

src/compiler/js-inlining-heuristic.cc

@@ -5,13 +5,67 @@
 #include "src/compiler/js-inlining-heuristic.h"
 
 #include "src/compilation-info.h"
+#include "src/compiler/common-operator.h"
 #include "src/compiler/node-matchers.h"
+#include "src/compiler/simplified-operator.h"
 #include "src/objects-inl.h"
 
 namespace v8 {
 namespace internal {
 namespace compiler {
 
+#define TRACE(...)                                      \
+  do {                                                  \
+    if (FLAG_trace_turbo_inlining) PrintF(__VA_ARGS__); \
+  } while (false)
+
+namespace {
+
+int CollectFunctions(Node* node, Handle<JSFunction>* functions,
+                     int functions_size) {
+  DCHECK_NE(0u, functions_size);
+  HeapObjectMatcher m(node);
+  if (m.HasValue() && m.Value()->IsJSFunction()) {
+    functions[0] = Handle<JSFunction>::cast(m.Value());
+    return 1;
+  }
+  if (m.IsPhi()) {
+    int const value_input_count = m.node()->op()->ValueInputCount();
+    if (value_input_count > functions_size) return 0;
+    for (int n = 0; n < value_input_count; ++n) {
+      HeapObjectMatcher m(node->InputAt(n));
+      if (!m.HasValue() || !m.Value()->IsJSFunction()) return 0;
+      functions[n] = Handle<JSFunction>::cast(m.Value());
+    }
+    return value_input_count;
+  }
+  return 0;
+}
+
+bool CanInlineFunction(Handle<JSFunction> function) {
+  // Built-in functions are handled by the JSBuiltinReducer.
+  if (function->shared()->HasBuiltinFunctionId()) return false;
+
+  // Don't inline builtins.
+  if (function->shared()->IsBuiltin()) return false;
+
+  // Quick check on source code length to avoid parsing large candidate.
+  if (function->shared()->SourceSize() > FLAG_max_inlined_source_size) {
+    return false;
+  }
+
+  // Quick check on the size of the AST to avoid parsing large candidate.
+  if (function->shared()->ast_node_count() > FLAG_max_inlined_nodes) {
+    return false;
+  }
+
+  // Avoid inlining across the boundary of asm.js code.
+  if (function->shared()->asm_function()) return false;
+  return true;
+}
+
+}  // namespace
+
 Reduction JSInliningHeuristic::Reduce(Node* node) {
   if (!IrOpcode::IsInlineeOpcode(node->opcode())) return NoChange();
 
@@ -19,14 +73,72 @@ Reduction JSInliningHeuristic::Reduce(Node* node) {
   if (seen_.find(node->id()) != seen_.end()) return NoChange();
   seen_.insert(node->id());
 
+  // Check if the {node} is an appropriate candidate for inlining.
   Node* callee = node->InputAt(0);
-  HeapObjectMatcher match(callee);
+  Candidate candidate;
-  if (!match.HasValue() || !match.Value()->IsJSFunction()) return NoChange();
+  candidate.node = node;
-  Handle<JSFunction> function = Handle<JSFunction>::cast(match.Value());
+  candidate.num_functions =
+      CollectFunctions(callee, candidate.functions, kMaxCallPolymorphism);
+  if (candidate.num_functions == 0) {
+    return NoChange();
+  } else if (candidate.num_functions > 1 && !FLAG_polymorphic_inlining) {
+    TRACE(
+        "Not considering call site #%d:%s, because polymorphic inlining "
+        "is disabled\n",
+        node->id(), node->op()->mnemonic());
+    return NoChange();
+  }
 
   // Functions marked with %SetForceInlineFlag are immediately inlined.
-  if (function->shared()->force_inline()) {
+  bool can_inline = false, force_inline = true;
-    return inliner_.ReduceJSCall(node, function);
+  for (int i = 0; i < candidate.num_functions; ++i) {
+    Handle<JSFunction> function = candidate.functions[i];
+    if (!function->shared()->force_inline()) {
+      force_inline = false;
+    }
+    if (CanInlineFunction(function)) {
+      can_inline = true;
+    }
+  }
+  if (force_inline) return InlineCandidate(candidate);
+  if (!can_inline) return NoChange();
+
+  // Stop inlining once the maximum allowed level is reached.
+  int level = 0;
+  for (Node* frame_state = NodeProperties::GetFrameStateInput(node);
+       frame_state->opcode() == IrOpcode::kFrameState;
+       frame_state = NodeProperties::GetFrameStateInput(frame_state)) {
+    FrameStateInfo const& frame_info = OpParameter<FrameStateInfo>(frame_state);
+    if (frame_info.type() == FrameStateType::kJavaScriptFunction) {
+      if (++level > FLAG_max_inlining_levels) {
+        TRACE(
+            "Not considering call site #%d:%s, because inlining depth "
+            "%d exceeds maximum allowed level %d\n",
+            node->id(), node->op()->mnemonic(), level,
+            FLAG_max_inlining_levels);
+        return NoChange();
+      }
+    }
+  }
+
+  // Gather feedback on how often this call site has been hit before.
+  if (node->opcode() == IrOpcode::kJSCallFunction) {
+    CallFunctionParameters p = CallFunctionParametersOf(node->op());
+    if (p.feedback().IsValid()) {
+      CallICNexus nexus(p.feedback().vector(), p.feedback().slot());
+      candidate.calls = nexus.ExtractCallCount();
+    }
+  } else {
+    DCHECK_EQ(IrOpcode::kJSCallConstruct, node->opcode());
+    CallConstructParameters p = CallConstructParametersOf(node->op());
+    if (p.feedback().IsValid()) {
+      int const extra_index =
+          p.feedback().vector()->GetIndex(p.feedback().slot()) + 1;
+      Object* feedback_extra = p.feedback().vector()->get(extra_index);
+      if (feedback_extra->IsSmi()) {
+        candidate.calls = Smi::cast(feedback_extra)->value();
+      }
+    }
   }
 
   // Handling of special inlining modes right away:
@@ -36,75 +148,16 @@ Reduction JSInliningHeuristic::Reduce(Node* node) {
     case kRestrictedInlining:
       return NoChange();
     case kStressInlining:
-      return inliner_.ReduceJSCall(node, function);
+      return InlineCandidate(candidate);
     case kGeneralInlining:
       break;
   }
 
-  // ---------------------------------------------------------------------------
-  // Everything below this line is part of the inlining heuristic.
-  // ---------------------------------------------------------------------------
-
-  // Built-in functions are handled by the JSBuiltinReducer.
-  if (function->shared()->HasBuiltinFunctionId()) return NoChange();
-
-  // Don't inline builtins.
-  if (function->shared()->IsBuiltin()) return NoChange();
-
-  // Quick check on source code length to avoid parsing large candidate.
-  if (function->shared()->SourceSize() > FLAG_max_inlined_source_size) {
-    return NoChange();
-  }
-
-  // Quick check on the size of the AST to avoid parsing large candidate.
-  if (function->shared()->ast_node_count() > FLAG_max_inlined_nodes) {
-    return NoChange();
-  }
-
-  // Avoid inlining within or across the boundary of asm.js code.
-  if (info_->shared_info()->asm_function()) return NoChange();
-  if (function->shared()->asm_function()) return NoChange();
-
-  // Stop inlining once the maximum allowed level is reached.
-  int level = 0;
-  for (Node* frame_state = NodeProperties::GetFrameStateInput(node);
-       frame_state->opcode() == IrOpcode::kFrameState;
-       frame_state = NodeProperties::GetFrameStateInput(frame_state)) {
-    if (++level > FLAG_max_inlining_levels) return NoChange();
-  }
-
-  // Gather feedback on how often this call site has been hit before.
-  int calls = -1;  // Same default as CallICNexus::ExtractCallCount.
-  if (node->opcode() == IrOpcode::kJSCallFunction) {
-    CallFunctionParameters p = CallFunctionParametersOf(node->op());
-    if (p.feedback().IsValid()) {
-      CallICNexus nexus(p.feedback().vector(), p.feedback().slot());
-      calls = nexus.ExtractCallCount();
-    }
-  } else {
-    DCHECK_EQ(IrOpcode::kJSCallConstruct, node->opcode());
-    CallConstructParameters p = CallConstructParametersOf(node->op());
-    if (p.feedback().IsValid()) {
-      int const extra_index =
-          p.feedback().vector()->GetIndex(p.feedback().slot()) + 1;
-      Handle<Object> feedback_extra(p.feedback().vector()->get(extra_index),
-                                    function->GetIsolate());
-      if (feedback_extra->IsSmi()) {
-        calls = Handle<Smi>::cast(feedback_extra)->value();
-      }
-    }
-  }
-
-  // ---------------------------------------------------------------------------
-  // Everything above this line is part of the inlining heuristic.
-  // ---------------------------------------------------------------------------
-
   // In the general case we remember the candidate for later.
-  candidates_.insert({function, node, calls});
+  candidates_.insert(candidate);
   return NoChange();
 }
 
-
 void JSInliningHeuristic::Finalize() {
   if (candidates_.empty()) return;  // Nothing to do without candidates.
   if (FLAG_trace_turbo_inlining) PrintCandidates();
@@ -120,15 +173,110 @@ void JSInliningHeuristic::Finalize() {
     candidates_.erase(i);
     // Make sure we don't try to inline dead candidate nodes.
     if (!candidate.node->IsDead()) {
-      Reduction r = inliner_.ReduceJSCall(candidate.node, candidate.function);
+      Reduction const reduction = InlineCandidate(candidate);
-      if (r.Changed()) {
+      if (reduction.Changed()) return;
-        cumulative_count_ += candidate.function->shared()->ast_node_count();
-        return;
-      }
     }
   }
 }
 
+Reduction JSInliningHeuristic::InlineCandidate(Candidate const& candidate) {
+  int const num_calls = candidate.num_functions;
+  Node* const node = candidate.node;
+  if (num_calls == 1) {
+    Handle<JSFunction> function = candidate.functions[0];
+    Reduction const reduction = inliner_.ReduceJSCall(node, function);
+    if (reduction.Changed()) {
+      cumulative_count_ += function->shared()->ast_node_count();
+    }
+    return reduction;
+  }
+
+  // Expand the JSCallFunction/JSCallConstruct node to a subgraph first if
+  // we have multiple known target functions.
+  DCHECK_LT(1, num_calls);
+  Node* calls[kMaxCallPolymorphism + 1];
+  Node* if_successes[kMaxCallPolymorphism];
+  Node* callee = NodeProperties::GetValueInput(node, 0);
+  Node* control = NodeProperties::GetControlInput(node);
+
+  // Setup the inputs for the cloned call nodes.
+  int const input_count = node->InputCount();
+  Node** inputs = graph()->zone()->NewArray<Node*>(input_count);
+  for (int i = 0; i < input_count; ++i) {
+    inputs[i] = node->InputAt(i);
+  }
+
+  // Create the appropriate control flow to dispatch to the cloned calls.
+  for (int i = 0; i < num_calls; ++i) {
+    Node* target = jsgraph()->HeapConstant(candidate.functions[i]);
+    if (i != (num_calls - 1)) {
+      Node* check =
+          graph()->NewNode(simplified()->ReferenceEqual(), callee, target);
+      Node* branch = graph()->NewNode(common()->Branch(), check, control);
+      control = graph()->NewNode(common()->IfFalse(), branch);
+      if_successes[i] = graph()->NewNode(common()->IfTrue(), branch);
+    } else {
+      if_successes[i] = control;
+    }
+
+    // The first input to the call is the actual target (which we specialize
+    // to the known {target}); the last input is the control dependency.
+    inputs[0] = target;
+    inputs[input_count - 1] = if_successes[i];
+    calls[i] = graph()->NewNode(node->op(), input_count, inputs);
+    if_successes[i] = graph()->NewNode(common()->IfSuccess(), calls[i]);
+  }
+
+  // Check if we have an exception projection for the call {node}.
+  Node* if_exception = nullptr;
+  for (Edge const edge : node->use_edges()) {
+    if (NodeProperties::IsControlEdge(edge) &&
+        edge.from()->opcode() == IrOpcode::kIfException) {
+      if_exception = edge.from();
+      break;
+    }
+  }
+  if (if_exception != nullptr) {
+    // Morph the {if_exception} projection into a join.
+    Node* if_exceptions[kMaxCallPolymorphism + 1];
+    for (int i = 0; i < num_calls; ++i) {
+      if_exceptions[i] =
+          graph()->NewNode(common()->IfException(), calls[i], calls[i]);
+    }
+    Node* control =
+        graph()->NewNode(common()->Merge(num_calls), num_calls, if_exceptions);
+    if_exceptions[num_calls] = control;
+    Node* effect = graph()->NewNode(common()->EffectPhi(num_calls),
+                                    num_calls + 1, if_exceptions);
+    Node* value = graph()->NewNode(
+        common()->Phi(MachineRepresentation::kTagged, num_calls), num_calls + 1,
+        if_exceptions);
+    ReplaceWithValue(if_exception, value, effect, control);
+  }
+
+  // Morph the call site into the dispatched call sites.
+  control =
+      graph()->NewNode(common()->Merge(num_calls), num_calls, if_successes);
+  calls[num_calls] = control;
+  Node* effect =
+      graph()->NewNode(common()->EffectPhi(num_calls), num_calls + 1, calls);
+  Node* value =
+      graph()->NewNode(common()->Phi(MachineRepresentation::kTagged, num_calls),
+                       num_calls + 1, calls);
+  ReplaceWithValue(node, value, effect, control);
+
+  // Inline the individual, cloned call sites.
+  for (int i = 0; i < num_calls; ++i) {
+    Handle<JSFunction> function = candidate.functions[i];
+    Node* node = calls[i];
+    Reduction const reduction = inliner_.ReduceJSCall(node, function);
+    if (reduction.Changed()) {
+      cumulative_count_ += function->shared()->ast_node_count();
+    }
+  }
+
+  return Replace(value);
+}
 
 bool JSInliningHeuristic::CandidateCompare::operator()(
     const Candidate& left, const Candidate& right) const {
@@ -138,18 +286,31 @@ bool JSInliningHeuristic::CandidateCompare::operator()(
   return left.node < right.node;
 }
 
-
 void JSInliningHeuristic::PrintCandidates() {
   PrintF("Candidates for inlining (size=%zu):\n", candidates_.size());
   for (const Candidate& candidate : candidates_) {
-    PrintF("  id:%d, calls:%d, size[source]:%d, size[ast]:%d / %s\n",
+    PrintF("  #%d:%s, calls:%d\n", candidate.node->id(),
-           candidate.node->id(), candidate.calls,
+           candidate.node->op()->mnemonic(), candidate.calls);
-           candidate.function->shared()->SourceSize(),
+    for (int i = 0; i < candidate.num_functions; ++i) {
-           candidate.function->shared()->ast_node_count(),
+      Handle<JSFunction> function = candidate.functions[i];
-           candidate.function->shared()->DebugName()->ToCString().get());
+      PrintF("  - size[source]:%d, size[ast]:%d, name: %s\n",
+             function->shared()->SourceSize(),
+             function->shared()->ast_node_count(),
+             function->shared()->DebugName()->ToCString().get());
+    }
   }
 }
 
+Graph* JSInliningHeuristic::graph() const { return jsgraph()->graph(); }
+
+CommonOperatorBuilder* JSInliningHeuristic::common() const {
+  return jsgraph()->common();
+}
+
+SimplifiedOperatorBuilder* JSInliningHeuristic::simplified() const {
+  return jsgraph()->simplified();
+}
+
 }  // namespace compiler
 }  // namespace internal
 }  // namespace v8

src/compiler/js-inlining-heuristic.h

@@ -21,7 +21,7 @@ class JSInliningHeuristic final : public AdvancedReducer {
         inliner_(editor, local_zone, info, jsgraph),
         candidates_(local_zone),
         seen_(local_zone),
-        info_(info) {}
+        jsgraph_(jsgraph) {}
 
   Reduction Reduce(Node* node) final;
 
@@ -30,10 +30,15 @@ class JSInliningHeuristic final : public AdvancedReducer {
   void Finalize() final;
 
  private:
+  // This limit currently matches what Crankshaft does. We may want to
+  // re-evaluate and come up with a proper limit for TurboFan.
+  static const int kMaxCallPolymorphism = 4;
+
   struct Candidate {
-    Handle<JSFunction> function;  // The call target being inlined.
+    Handle<JSFunction> functions[kMaxCallPolymorphism];
-    Node* node;                   // The call site at which to inline.
+    int num_functions;
-    int calls;                    // Number of times the call site was hit.
+    Node* node = nullptr;  // The call site at which to inline.
+    int calls = -1;        // Number of times the call site was hit.
   };
 
   // Comparator for candidates.
@@ -46,12 +51,18 @@ class JSInliningHeuristic final : public AdvancedReducer {
 
   // Dumps candidates to console.
   void PrintCandidates();
+  Reduction InlineCandidate(Candidate const& candidate);
+
+  CommonOperatorBuilder* common() const;
+  Graph* graph() const;
+  JSGraph* jsgraph() const { return jsgraph_; }
+  SimplifiedOperatorBuilder* simplified() const;
 
   Mode const mode_;
   JSInliner inliner_;
   Candidates candidates_;
   ZoneSet<NodeId> seen_;
-  CompilationInfo* info_;
+  JSGraph* const jsgraph_;
   int cumulative_count_ = 0;
 };
 

src/compiler/pipeline.cc

@@ -605,6 +605,9 @@ PipelineCompilationJob::Status PipelineCompilationJob::PrepareJobImpl() {
     if (FLAG_native_context_specialization) {
       info()->MarkAsNativeContextSpecializing();
     }
+    if (FLAG_turbo_inlining) {
+      info()->MarkAsInliningEnabled();
+    }
   }
   if (!info()->shared_info()->asm_function() || FLAG_turbo_asm_deoptimization) {
     info()->MarkAsDeoptimizationEnabled();