[turbofan] Add a nesting limit for the child serializer

Add a --max-serializer-nesting flag which defaults to 25.

Fixed: chromium:1034768
Change-Id: Ib68f26ce4bf53db297b25d16a046d275beaec642
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1969895
Commit-Queue: Maya Lekova <mslekova@chromium.org>
Reviewed-by: Michael Stanton <mvstanton@chromium.org>
Cr-Commit-Position: refs/heads/master@{#65486}

src/compiler/serializer-for-background-compilation.cc

@@ -378,7 +378,7 @@ class SerializerForBackgroundCompilation {
       CompilationDependencies* dependencies, CompilationSubject function,
       base::Optional<Hints> new_target, const HintsVector& arguments,
       MissingArgumentsPolicy padding,
-      SerializerForBackgroundCompilationFlags flags);
+      SerializerForBackgroundCompilationFlags flags, int nesting_level);
 
   bool BailoutOnUninitialized(ProcessedFeedback const& feedback);
 
@@ -546,6 +546,8 @@ class SerializerForBackgroundCompilation {
   HintsVector const arguments_;
   Hints return_value_hints_;
   Hints closure_hints_;
+
+  int nesting_level_ = 0;
 };
 
 void RunSerializerForBackgroundCompilation(
@@ -1009,7 +1011,7 @@ SerializerForBackgroundCompilation::SerializerForBackgroundCompilation(
     CompilationDependencies* dependencies, CompilationSubject function,
     base::Optional<Hints> new_target, const HintsVector& arguments,
     MissingArgumentsPolicy padding,
-    SerializerForBackgroundCompilationFlags flags)
+    SerializerForBackgroundCompilationFlags flags, int nesting_level)
     : broker_(broker),
       dependencies_(dependencies),
       zone_scope_(zone_stats, ZONE_NAME),
@@ -1020,7 +1022,8 @@ SerializerForBackgroundCompilation::SerializerForBackgroundCompilation(
       environment_(new (zone())
                        Environment(zone(), broker_->isolate(), function,
                                    new_target, arguments, padding)),
-      arguments_(arguments) {
+      arguments_(arguments),
+      nesting_level_(nesting_level) {
   Handle<JSFunction> closure;
   if (function.closure().ToHandle(&closure)) {
     closure_hints_.AddConstant(closure, zone());
@@ -1054,15 +1057,23 @@ bool SerializerForBackgroundCompilation::BailoutOnUninitialized(
 
 Hints SerializerForBackgroundCompilation::Run() {
   TraceScope tracer(broker(), this, "SerializerForBackgroundCompilation::Run");
+  if (nesting_level_ >= FLAG_max_serializer_nesting) {
+    TRACE_BROKER_MISSING(
+        broker(),
+        "opportunity - Reached max nesting level for "
+        "SerializerForBackgroundCompilation::Run, bailing out.\n");
+    return Hints();
+  }
+
   TRACE_BROKER_MEMORY(broker(), "[serializer start] Broker zone usage: "
                                     << broker()->zone()->allocation_size());
   SharedFunctionInfoRef shared(broker(), function().shared());
   FeedbackVectorRef feedback_vector_ref(broker(), feedback_vector());
   if (!broker()->ShouldBeSerializedForCompilation(shared, feedback_vector_ref,
                                                   arguments_)) {
-    TRACE_BROKER(broker(), "Already ran serializer for SharedFunctionInfo "
-                               << Brief(*shared.object())
-                               << ", bailing out.\n");
+    TRACE_BROKER_MISSING(
+        broker(), "opportunity - Already ran serializer for SharedFunctionInfo "
+                      << Brief(*shared.object()) << ", bailing out.\n");
     return Hints();
   }
 
@@ -1919,7 +1930,7 @@ Hints SerializerForBackgroundCompilation::RunChildSerializer(
     const HintsVector& arguments, MissingArgumentsPolicy padding) {
   SerializerForBackgroundCompilation child_serializer(
       zone_scope_.zone_stats(), broker(), dependencies(), function, new_target,
-      arguments, padding, flags());
+      arguments, padding, flags(), nesting_level_ + 1);
   Hints result = child_serializer.Run();
   // The Hints returned by the call to Run are allocated in the zone
   // created by the child serializer. Adding those hints to a hints

src/flags/flag-definitions.h

@@ -506,6 +506,8 @@ DEFINE_BOOL(block_concurrent_recompilation, false,
             "block queued jobs until released")
 DEFINE_BOOL(concurrent_inlining, false,
             "run optimizing compiler's inlining phase on a separate thread")
+DEFINE_INT(max_serializer_nesting, 25,
+           "maximum levels for nesting child serializers")
 DEFINE_IMPLICATION(future, concurrent_inlining)
 DEFINE_BOOL(trace_heap_broker_verbose, false,
             "trace the heap broker verbosely (all reports)")

test/mjsunit/compiler/regress-1034768

@@ -0,0 +1,24 @@
+// Copyright 2019 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Flags: --allow-natives-syntax --max-serializer-nesting=4
+function f1() {
+  return 1;
+}
+
+function f2() { return f1(); }
+function f3() { return f2(); }
+function f4() { return f3(); }
+function f5() { return f4(); }
+
+%PrepareFunctionForOptimization(f1);
+%PrepareFunctionForOptimization(f2);
+%PrepareFunctionForOptimization(f3);
+%PrepareFunctionForOptimization(f4);
+%PrepareFunctionForOptimization(f5);
+
+f5();
+f5();
+%OptimizeFunctionOnNextCall(f5);
+f5();