[TurboFan] Move FunctionTemplateInfo to never serialized

This CL moves FunctionTemplateInfo to the list of never serialized
objects, allowing direct heap reads. To make this threadsafe, the CL:
- adds necessary atomic (relaxed/acquire-release) operations to the
  accessors of FunctionTemplateInfo.
- changes FunctionTemplateInfoRef::LookupHolderOfExpectedType to be
  usable from the background thread (e.g. no handle construction) with
  the caveat of skipping optimization in some cases where necessary
  JSObjects are not serialized.

Drive-by: Add missing serialization of objects possibly reachable
through CallHandlerInfo::data.

Bug: v8:7790
Change-Id: I49cf4f328ecfab368dff9076fde8f5783ead3246
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2679687
Reviewed-by: Ross McIlroy <rmcilroy@chromium.org>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Reviewed-by: Georg Neis <neis@chromium.org>
Commit-Queue: Nico Hartmann <nicohartmann@chromium.org>
Cr-Commit-Position: refs/heads/master@{#73364}

src/compiler-dispatcher/optimizing-compile-dispatcher.cc

@@ -58,12 +58,13 @@ class OptimizingCompileDispatcher::CompileTask : public CancelableTask {
  private:
   // v8::Task overrides.
   void RunInternal() override {
-    LocalIsolate local_isolate(isolate_, ThreadKind::kBackground);
+    WorkerThreadRuntimeCallStatsScope runtime_call_stats_scope(
+        worker_thread_runtime_call_stats_);
+    LocalIsolate local_isolate(isolate_, ThreadKind::kBackground,
+                               runtime_call_stats_scope.Get());
     DCHECK(local_isolate.heap()->IsParked());
 
     {
-      WorkerThreadRuntimeCallStatsScope runtime_call_stats_scope(
-          worker_thread_runtime_call_stats_);
       RuntimeCallTimerScope runtimeTimer(
           runtime_call_stats_scope.Get(),
           RuntimeCallCounterId::kOptimizeBackgroundDispatcherJob);

src/compiler/heap-refs.h

@@ -87,6 +87,7 @@ enum class OddballType : uint8_t {
   V(DescriptorArray)                                \
   V(FeedbackCell)                                   \
   V(FeedbackVector)                                 \
+  V(FunctionTemplateInfo)                           \
   V(Name)                                           \
   V(RegExpBoilerplateDescription)                   \
   V(SharedFunctionInfo)                             \
@@ -136,7 +137,6 @@ enum class OddballType : uint8_t {
   /* Subtypes of HeapObject */                \
   V(AllocationSite)                           \
   V(FixedArrayBase)                           \
-  V(FunctionTemplateInfo)                     \
   V(JSReceiver)                               \
   V(SourceTextModule)                         \
   /* Subtypes of Object */                    \
@@ -192,6 +192,7 @@ class V8_EXPORT_PRIVATE ObjectRef {
   HEAP_BROKER_NEVER_SERIALIZED_OBJECT_LIST(HEAP_AS_METHOD_DECL)
 #undef HEAP_AS_METHOD_DECL
 
+  bool IsNull() const;
   bool IsNullOrUndefined() const;
   bool IsTheHole() const;
 

src/compiler/js-heap-broker.cc

@@ -164,6 +164,7 @@ class ObjectData : public ZoneObject {
            kind_ == kNeverSerializedHeapObject ||
            kind_ == kUnserializedReadOnlyHeapObject;
   }
+  bool IsNull() const { return object_->IsNull(); }
 
 #ifdef DEBUG
   enum class Usage{kUnused, kOnlyIdentityUsed, kDataUsed};
@@ -271,6 +272,7 @@ FunctionTemplateInfoData::FunctionTemplateInfoData(
       c_function_(v8::ToCData<Address>(object->GetCFunction())),
       c_signature_(v8::ToCData<CFunctionInfo*>(object->GetCSignature())),
       known_receivers_(broker->zone()) {
+  DCHECK(!FLAG_turbo_direct_heap_access);
   auto function_template_info = Handle<FunctionTemplateInfo>::cast(object);
   is_signature_undefined_ =
       function_template_info->signature().IsUndefined(broker->isolate());
@@ -3511,8 +3513,10 @@ BIMODAL_ACCESSOR_C(RegExpBoilerplateDescription, int, flags)
 
 base::Optional<CallHandlerInfoRef> FunctionTemplateInfoRef::call_code() const {
   if (data_->should_access_heap()) {
-    return CallHandlerInfoRef(broker(), broker()->CanonicalPersistentHandle(
+    HeapObject call_code = object()->call_code(kAcquireLoad);
-                                            object()->call_code(kAcquireLoad)));
+    if (call_code.IsUndefined()) return base::nullopt;
+    return CallHandlerInfoRef(broker(),
+                              broker()->CanonicalPersistentHandle(call_code));
   }
   ObjectData* call_code = data()->AsFunctionTemplateInfo()->call_code();
   if (!call_code) return base::nullopt;
@@ -3528,40 +3532,72 @@ bool FunctionTemplateInfoRef::is_signature_undefined() const {
 
 bool FunctionTemplateInfoRef::has_call_code() const {
   if (data_->should_access_heap()) {
-    CallOptimization call_optimization(broker()->isolate(), object());
+    HeapObject call_code = object()->call_code(kAcquireLoad);
-    return call_optimization.is_simple_api_call();
+    return !call_code.IsUndefined();
   }
   return data()->AsFunctionTemplateInfo()->has_call_code();
 }
 
-BIMODAL_ACCESSOR_C(FunctionTemplateInfo, bool, accept_any_receiver)
+bool FunctionTemplateInfoRef ::accept_any_receiver() const {
+  if (data_->should_access_heap()) {
+    return object()->accept_any_receiver();
+  }
+  return ObjectRef ::data()->AsFunctionTemplateInfo()->accept_any_receiver();
+}
 
 HolderLookupResult FunctionTemplateInfoRef::LookupHolderOfExpectedType(
     MapRef receiver_map, SerializationPolicy policy) {
   const HolderLookupResult not_found;
 
   if (data_->should_access_heap()) {
-    CallOptimization call_optimization(broker()->isolate(), object());
+    // There are currently two ways we can see a FunctionTemplateInfo on the
-    Handle<Map> receiver_map_ref(receiver_map.object());
+    // background thread: 1.) As part of a SharedFunctionInfo and 2.) in an
-    if (!receiver_map_ref->IsJSReceiverMap() ||
+    // AccessorPair. In both cases, the FTI is fully constructed on the main
-        (receiver_map_ref->is_access_check_needed() &&
+    // thread before.
+    // TODO(nicohartmann@, v8:7790): Once the above no longer holds, we might
+    // have to use the GC predicate to check whether objects are fully
+    // initialized and safe to read.
+    if (!receiver_map.IsJSReceiverMap() ||
+        (receiver_map.is_access_check_needed() &&
          !object()->accept_any_receiver())) {
       return not_found;
     }
 
-    HolderLookupResult result;
+    if (!receiver_map.IsJSObjectMap()) return not_found;
-    Handle<JSObject> holder = call_optimization.LookupHolderOfExpectedType(
-        receiver_map_ref, &result.lookup);
 
-    switch (result.lookup) {
+    DCHECK(has_call_code());
-      case CallOptimization::kHolderFound:
+
-        result.holder = JSObjectRef(broker(), holder);
+    DisallowGarbageCollection no_gc;
-        break;
+    HeapObject signature = object()->signature();
-      default:
+    if (signature.IsUndefined()) {
-        DCHECK_EQ(result.holder, base::nullopt);
+      return HolderLookupResult(CallOptimization::kHolderIsReceiver);
-        break;
     }
-    return result;
+    auto expected_receiver_type = FunctionTemplateInfo::cast(signature);
+    if (expected_receiver_type.IsTemplateFor(*receiver_map.object())) {
+      return HolderLookupResult(CallOptimization::kHolderIsReceiver);
+    }
+
+    if (!receiver_map.IsJSGlobalProxyMap()) return not_found;
+    if (policy == SerializationPolicy::kSerializeIfNeeded) {
+      receiver_map.SerializePrototype();
+    }
+    if (!receiver_map.serialized_prototype()) return not_found;
+    if (receiver_map.prototype().IsNull()) return not_found;
+
+    JSObject raw_prototype = JSObject::cast(*receiver_map.prototype().object());
+    if (!expected_receiver_type.IsTemplateFor(raw_prototype.map())) {
+      return not_found;
+    }
+    Handle<JSObject> prototype =
+        broker()->CanonicalPersistentHandle(raw_prototype);
+    if (ObjectData* data = broker()->TryGetOrCreateData(prototype)) {
+      return HolderLookupResult(CallOptimization::kHolderFound,
+                                JSObjectRef(broker(), data));
+    }
+
+    TRACE_BROKER_MISSING(broker(),
+                         "holder for receiver with map " << receiver_map);
+    return not_found;
   }
 
   FunctionTemplateInfoData* fti_data = data()->AsFunctionTemplateInfo();
@@ -3898,6 +3934,8 @@ base::Optional<JSFunctionRef> NativeContextRef::GetConstructorFunction(
   }
 }
 
+bool ObjectRef::IsNull() const { return object()->IsNull(); }
+
 bool ObjectRef::IsNullOrUndefined() const {
   if (IsSmi()) return false;
   OddballType type = AsHeapObject().map().oddball_type();
@@ -4519,8 +4557,8 @@ void MapRef::SerializePrototype() {
 }
 
 bool MapRef::serialized_prototype() const {
-  CHECK_NE(broker()->mode(), JSHeapBroker::kDisabled);
   if (data_->should_access_heap()) return true;
+  CHECK_NE(broker()->mode(), JSHeapBroker::kDisabled);
   return data()->AsMap()->serialized_prototype();
 }
 

src/compiler/serializer-for-background-compilation.cc

@@ -2260,21 +2260,27 @@ void SerializerForBackgroundCompilation::ProcessCallVarArgs(
 
 void SerializerForBackgroundCompilation::ProcessApiCall(
     Handle<SharedFunctionInfo> target, const HintsVector& arguments) {
-  ObjectRef(broker(), broker()->isolate()->builtins()->builtin_handle(
+  for (const auto b :
-                          Builtins::kCallFunctionTemplate_CheckAccess));
+       {Builtins::kCallFunctionTemplate_CheckAccess,
-  ObjectRef(broker(),
+        Builtins::kCallFunctionTemplate_CheckCompatibleReceiver,
-            broker()->isolate()->builtins()->builtin_handle(
+        Builtins::kCallFunctionTemplate_CheckAccessAndCompatibleReceiver}) {
-                Builtins::kCallFunctionTemplate_CheckCompatibleReceiver));
+    ObjectRef(broker(), broker()->isolate()->builtins()->builtin_handle(b));
-  ObjectRef(
+  }
-      broker(),
-      broker()->isolate()->builtins()->builtin_handle(
-          Builtins::kCallFunctionTemplate_CheckAccessAndCompatibleReceiver));
-
   FunctionTemplateInfoRef target_template_info(
       broker(),
       broker()->CanonicalPersistentHandle(target->function_data(kAcquireLoad)));
   if (!target_template_info.has_call_code()) return;
-  target_template_info.SerializeCallCode();
+  if (FLAG_turbo_direct_heap_access) {
+    // The object stored in CallHandlerInfo::data may still be
+    // SerializedHeapObject, so we keep serializing this here.
+    // TODO(v8:7790): Remove this once all JSObjects are NeverSerialized.
+    Handle<CallHandlerInfo> call_handler_info =
+        target_template_info.call_code()->object();
+    ObjectRef(broker(),
+              broker()->CanonicalPersistentHandle(call_handler_info->data()));
+  } else {
+    target_template_info.SerializeCallCode();
+  }
 
   SharedFunctionInfoRef target_ref(broker(), target);
   target_ref.SerializeFunctionTemplateInfo();

src/diagnostics/objects-printer.cc

@@ -2097,7 +2097,7 @@ void FunctionTemplateInfo::FunctionTemplateInfoPrint(
   os << "\n - undetectable: " << (undetectable() ? "true" : "false");
   os << "\n - need_access_check: " << (needs_access_check() ? "true" : "false");
   os << "\n - instantiated: " << (instantiated() ? "true" : "false");
-  os << "\n - rare_data: " << Brief(rare_data());
+  os << "\n - rare_data: " << Brief(rare_data(kAcquireLoad));
   os << "\n";
 }
 

src/execution/local-isolate.cc

@@ -12,7 +12,8 @@
 namespace v8 {
 namespace internal {
 
-LocalIsolate::LocalIsolate(Isolate* isolate, ThreadKind kind)
+LocalIsolate::LocalIsolate(Isolate* isolate, ThreadKind kind,
+                           RuntimeCallStats* runtime_call_stats)
     : HiddenLocalFactory(isolate),
       heap_(isolate->heap(), kind),
       isolate_(isolate),
@@ -20,7 +21,8 @@ LocalIsolate::LocalIsolate(Isolate* isolate, ThreadKind kind)
       thread_id_(ThreadId::Current()),
       stack_limit_(kind == ThreadKind::kMain
                        ? isolate->stack_guard()->real_climit()
-                       : GetCurrentStackPosition() - FLAG_stack_size * KB) {}
+                       : GetCurrentStackPosition() - FLAG_stack_size * KB),
+      runtime_call_stats_(runtime_call_stats) {}
 
 LocalIsolate::~LocalIsolate() = default;
 

src/execution/local-isolate.h

@@ -19,6 +19,7 @@ namespace internal {
 
 class Isolate;
 class LocalLogger;
+class RuntimeCallStats;
 
 // HiddenLocalFactory parallels Isolate's HiddenFactory
 class V8_EXPORT_PRIVATE HiddenLocalFactory : private LocalFactory {
@@ -37,7 +38,8 @@ class V8_EXPORT_PRIVATE LocalIsolate final : private HiddenLocalFactory {
  public:
   using HandleScopeType = LocalHandleScope;
 
-  explicit LocalIsolate(Isolate* isolate, ThreadKind kind);
+  explicit LocalIsolate(Isolate* isolate, ThreadKind kind,
+                        RuntimeCallStats* runtime_call_stats = nullptr);
   ~LocalIsolate();
 
   // Kinda sketchy.
@@ -84,6 +86,7 @@ class V8_EXPORT_PRIVATE LocalIsolate final : private HiddenLocalFactory {
   LocalLogger* logger() const { return logger_.get(); }
   ThreadId thread_id() const { return thread_id_; }
   Address stack_limit() const { return stack_limit_; }
+  RuntimeCallStats* runtime_call_stats() const { return runtime_call_stats_; }
 
   bool is_main_thread() const { return heap_.is_main_thread(); }
 
@@ -101,6 +104,8 @@ class V8_EXPORT_PRIVATE LocalIsolate final : private HiddenLocalFactory {
   std::unique_ptr<LocalLogger> logger_;
   ThreadId const thread_id_;
   Address const stack_limit_;
+
+  RuntimeCallStats* runtime_call_stats_;
 };
 
 template <base::MutexSharedType kIsShared>

src/ic/call-optimization.cc

@@ -88,16 +88,14 @@ bool CallOptimization::IsCompatibleReceiverMap(
 
 void CallOptimization::Initialize(
     Isolate* isolate, Handle<FunctionTemplateInfo> function_template_info) {
-  if (function_template_info->call_code(kAcquireLoad).IsUndefined(isolate))
+  HeapObject call_code = function_template_info->call_code(kAcquireLoad);
-    return;
+  if (call_code.IsUndefined(isolate)) return;
-  api_call_info_ = handle(
+  api_call_info_ = handle(CallHandlerInfo::cast(call_code), isolate);
-      CallHandlerInfo::cast(function_template_info->call_code(kAcquireLoad)),
-      isolate);
 
-  if (!function_template_info->signature().IsUndefined(isolate)) {
+  HeapObject signature = function_template_info->signature();
+  if (!signature.IsUndefined(isolate)) {
     expected_receiver_type_ =
-        handle(FunctionTemplateInfo::cast(function_template_info->signature()),
+        handle(FunctionTemplateInfo::cast(signature), isolate);
-               isolate);
   }
   is_simple_api_call_ = true;
 }

src/objects/object-macros.h

@@ -351,6 +351,19 @@
     set_##field(BooleanBit::set(field(), offset, value));                \
   }
 
+#define DECL_RELAXED_BOOL_ACCESSORS(name) \
+  inline bool name(RelaxedLoadTag) const; \
+  inline void set_##name(bool value, RelaxedStoreTag);
+
+#define RELAXED_BOOL_ACCESSORS(holder, field, name, offset)          \
+  bool holder::name(RelaxedLoadTag) const {                          \
+    return BooleanBit::get(field(kRelaxedLoad), offset);             \
+  }                                                                  \
+  void holder::set_##name(bool value, RelaxedStoreTag) {             \
+    set_##field(BooleanBit::set(field(kRelaxedLoad), offset, value), \
+                kRelaxedStore);                                      \
+  }
+
 #define BIT_FIELD_ACCESSORS(holder, field, name, BitField)      \
   typename BitField::FieldType holder::name() const {           \
     return BitField::decode(field());                           \

src/objects/objects.cc

@@ -1314,8 +1314,12 @@ Handle<SharedFunctionInfo> FunctionTemplateInfo::GetOrCreateSharedFunctionInfo(
 }
 
 bool FunctionTemplateInfo::IsTemplateFor(Map map) {
-  RuntimeCallTimerScope timer(GetIsolate(),
+  RuntimeCallTimerScope timer(
-                              RuntimeCallCounterId::kIsTemplateFor);
+      LocalHeap::Current() == nullptr
+          ? GetIsolate()->counters()->runtime_call_stats()
+          : LocalIsolate::FromHeap(LocalHeap::Current())->runtime_call_stats(),
+      RuntimeCallCounterId::kIsTemplateFor);
+
   // There is a constraint on the object; check.
   if (!map.IsJSObjectMap()) return false;
   // Fetch the constructor function of the object.
@@ -1342,14 +1346,14 @@ bool FunctionTemplateInfo::IsTemplateFor(Map map) {
 // static
 FunctionTemplateRareData FunctionTemplateInfo::AllocateFunctionTemplateRareData(
     Isolate* isolate, Handle<FunctionTemplateInfo> function_template_info) {
-  DCHECK(function_template_info->rare_data().IsUndefined(isolate));
+  DCHECK(function_template_info->rare_data(kAcquireLoad).IsUndefined(isolate));
   Handle<Struct> struct_obj = isolate->factory()->NewStruct(
       FUNCTION_TEMPLATE_RARE_DATA_TYPE, AllocationType::kOld);
   Handle<FunctionTemplateRareData> rare_data =
       i::Handle<FunctionTemplateRareData>::cast(struct_obj);
   rare_data->set_c_function(Smi(0));
   rare_data->set_c_signature(Smi(0));
-  function_template_info->set_rare_data(*rare_data);
+  function_template_info->set_rare_data(*rare_data, kReleaseStore);
   return *rare_data;
 }
 

src/objects/templates-inl.h

@@ -39,13 +39,33 @@ BOOL_ACCESSORS(FunctionTemplateInfo, flag, accept_any_receiver,
                AcceptAnyReceiverBit::kShift)
 BOOL_ACCESSORS(FunctionTemplateInfo, flag, published, PublishedBit::kShift)
 
+// TODO(nicohartmann@, v8:11122): Let Torque generate this accessor.
 RELEASE_ACQUIRE_ACCESSORS(FunctionTemplateInfo, call_code, HeapObject,
                           kCallCodeOffset)
 
+// TODO(nicohartmann@, v8:11122): Let Torque generate this accessor.
+HeapObject FunctionTemplateInfo::rare_data(AcquireLoadTag) const {
+  IsolateRoot isolate = GetIsolateForPtrCompr(*this);
+  return rare_data(isolate, kAcquireLoad);
+}
+HeapObject FunctionTemplateInfo::rare_data(IsolateRoot isolate,
+                                           AcquireLoadTag) const {
+  HeapObject value =
+      TaggedField<HeapObject>::Acquire_Load(isolate, *this, kRareDataOffset);
+  DCHECK(value.IsUndefined() || value.IsFunctionTemplateRareData());
+  return value;
+}
+void FunctionTemplateInfo::set_rare_data(HeapObject value, ReleaseStoreTag,
+                                         WriteBarrierMode mode) {
+  DCHECK(value.IsUndefined() || value.IsFunctionTemplateRareData());
+  RELEASE_WRITE_FIELD(*this, kRareDataOffset, value);
+  CONDITIONAL_WRITE_BARRIER(*this, kRareDataOffset, value, mode);
+}
+
 // static
 FunctionTemplateRareData FunctionTemplateInfo::EnsureFunctionTemplateRareData(
     Isolate* isolate, Handle<FunctionTemplateInfo> function_template_info) {
-  HeapObject extra = function_template_info->rare_data(isolate);
+  HeapObject extra = function_template_info->rare_data(isolate, kAcquireLoad);
   if (extra.IsUndefined(isolate)) {
     return AllocateFunctionTemplateRareData(isolate, function_template_info);
   } else {
@@ -55,7 +75,7 @@ FunctionTemplateRareData FunctionTemplateInfo::EnsureFunctionTemplateRareData(
 
 #define RARE_ACCESSORS(Name, CamelName, Type, Default)                        \
   DEF_GETTER(FunctionTemplateInfo, Get##CamelName, Type) {                    \
-    HeapObject extra = rare_data(isolate);                                    \
+    HeapObject extra = rare_data(isolate, kAcquireLoad);                      \
     HeapObject undefined = GetReadOnlyRoots(isolate).undefined_value();       \
     return extra == undefined ? Default                                       \
                               : FunctionTemplateRareData::cast(extra).Name(); \

src/objects/templates.h

@@ -87,8 +87,16 @@ class FunctionTemplateInfo
   DECL_RARE_ACCESSORS(c_signature, CSignature, Object)
 #undef DECL_RARE_ACCESSORS
 
+  // TODO(nicohartmann@, v8:11122): Let Torque generate the following accessor.
   DECL_RELEASE_ACQUIRE_ACCESSORS(call_code, HeapObject)
 
+  // TODO(nicohartmann@, v8:11122): Let Torque generate the following accessor.
+  inline HeapObject rare_data(AcquireLoadTag) const;
+  inline HeapObject rare_data(IsolateRoot isolate, AcquireLoadTag) const;
+  inline void set_rare_data(
+      HeapObject value, ReleaseStoreTag,
+      WriteBarrierMode mode = WriteBarrierMode::UPDATE_WRITE_BARRIER);
+
   // Begin flag bits ---------------------
   DECL_BOOLEAN_ACCESSORS(undetectable)
 

src/objects/templates.tq

@@ -52,7 +52,7 @@ extern class FunctionTemplateInfo extends TemplateInfo {
   // If any of the setters declared by DECL_RARE_ACCESSORS are used then a
   // FunctionTemplateRareData will be stored here. Until then this contains
   // undefined.
-  @acquireRead @releaseWrite rare_data: FunctionTemplateRareData|Undefined;
+  rare_data: FunctionTemplateRareData|Undefined;
   shared_function_info: SharedFunctionInfo|Undefined;
   // Internal field to store a flag bitfield.
   flag: SmiTagged<FunctionTemplateInfoFlags>;