[d8] Fix stack overflow issues with async hooks

This fix consists of 2 parts:
a) Fix async hooks:
 - Allow initialising the promise hook properties
 - Do not call async hooks if we're overflowing the stack

b) Avoid some more recursion when reporting the stack trace

Bug: chromium:1240723
Change-Id: Icedfc8b48655bacc3f79591944e3869b85f1c4de
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3103321
Reviewed-by: Igor Sheludko <ishell@chromium.org>
Commit-Queue: Camillo Bruni <cbruni@chromium.org>
Cr-Commit-Position: refs/heads/main@{#76383}

src/d8/async-hooks-wrapper.cc

@@ -123,8 +123,6 @@ void AsyncHooks::ShellPromiseHook(PromiseHookType type, Local<Promise> promise,
   v8::Isolate* isolate = promise->GetIsolate();
   i::Isolate* i_isolate = reinterpret_cast<i::Isolate*>(isolate);
 
-  if (i::StackLimitCheck{i_isolate}.HasOverflowed()) return;
-
   AsyncHooks* hooks = PerIsolateData::Get(isolate)->GetAsyncHooks();
   HandleScope handle_scope(isolate);
   // Temporarily clear any scheduled_exception to allow evaluating JS that can
@@ -180,11 +178,13 @@ void AsyncHooks::ShellPromiseHook(PromiseHookType type, Local<Promise> promise,
     } else if (type == PromiseHookType::kAfter) {
       hooks->asyncContexts.pop();
     }
-    for (AsyncHooksWrap* wrap : hooks->async_wraps_) {
-      PromiseHookDispatch(type, promise, parent, wrap, hooks);
-      if (try_catch.HasCaught()) break;
+    if (!i::StackLimitCheck{i_isolate}.HasOverflowed()) {
+      for (AsyncHooksWrap* wrap : hooks->async_wraps_) {
+        PromiseHookDispatch(type, promise, parent, wrap, hooks);
+        if (try_catch.HasCaught()) break;
+      }
+      if (try_catch.HasCaught()) Shell::ReportException(isolate, &try_catch);
     }
-    if (try_catch.HasCaught()) Shell::ReportException(isolate, &try_catch);
   }
   if (!scheduled_exception.is_null()) {
     i_isolate->set_scheduled_exception(*scheduled_exception);

src/execution/messages.cc

@@ -301,6 +301,7 @@ MaybeHandle<Object> ErrorUtils::FormatStackTrace(Isolate* isolate,
   Handle<FixedArray> elems = Handle<FixedArray>::cast(raw_stack);
 
   const bool in_recursion = isolate->formatting_stack_trace();
+  const bool has_overflowed = i::StackLimitCheck{isolate}.HasOverflowed();
   Handle<Context> error_context;
   if (!in_recursion && error->GetCreationContext().ToHandle(&error_context)) {
     DCHECK(error_context->IsNativeContext());
@@ -318,7 +319,7 @@ MaybeHandle<Object> ErrorUtils::FormatStackTrace(Isolate* isolate,
           isolate->RunPrepareStackTraceCallback(error_context, error, sites),
           Object);
       return result;
-    } else {
+    } else if (!has_overflowed) {
       Handle<JSFunction> global_error =
           handle(error_context->error_function(), isolate);
 
@@ -359,7 +360,6 @@ MaybeHandle<Object> ErrorUtils::FormatStackTrace(Isolate* isolate,
   }
 
   // Otherwise, run our internal formatting logic.
-
   IncrementalStringBuilder builder(isolate);
 
   RETURN_ON_EXCEPTION(isolate, AppendErrorString(isolate, error, &builder),

test/mjsunit/regress/regress-crbug-1239907.js

@@ -14,3 +14,23 @@ function target() {
   })
 }
 target();
+
+function runNearStackLimit(f) {
+  function t() {
+    try {
+      return t();
+    } catch (e) {
+      return f();
+    }
+  }
+  return t();
+}
+
+function __f_2() {
+  return runNearStackLimit(() => {
+      return new Promise(function () {
+    });
+  });
+}
+__f_2().then();
+__f_2().then();