[ext-code-space] Support embedding of CodeT references

In order to avoid some of the unnecessary Code <-> CodeDataContainer
conversions in builtins we need to be able to embed CodeDataContainer
references to builtins.

This CL makes it possible by introducing a table of builtins' CDCs.
Eventually, usages of the builtins table containing Code objects will
be replaced by usages of this table.

Bug: v8:11880
Change-Id: Iffffd1507d5c7b38af34c42071a4577a8d18e8eb
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3257710
Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Reviewed-by: Jakob Gruber <jgruber@chromium.org>
Reviewed-by: Camillo Bruni <cbruni@chromium.org>
Commit-Queue: Camillo Bruni <cbruni@chromium.org>
Auto-Submit: Igor Sheludko <ishell@chromium.org>
Cr-Commit-Position: refs/heads/main@{#77678}

src/builtins/builtins.cc

@@ -194,6 +194,37 @@ Handle<Code> Builtins::code_handle(Builtin builtin) {
   return Handle<Code>(location);
 }
 
+FullObjectSlot Builtins::builtin_code_data_container_slot(Builtin builtin) {
+  CHECK(V8_EXTERNAL_CODE_SPACE_BOOL);
+  Address* location =
+      &isolate_->builtin_code_data_container_table()[Builtins::ToInt(builtin)];
+  return FullObjectSlot(location);
+}
+
+void Builtins::set_codet(Builtin builtin, CodeT code) {
+  CHECK(V8_EXTERNAL_CODE_SPACE_BOOL);
+  // TODO(v8:11880): add DCHECK_EQ(builtin, code.builtin_id()); once CodeT
+  // has respective field.
+  DCHECK(Internals::HasHeapObjectTag(code.ptr()));
+  // The given builtin may be uninitialized thus we cannot check its type here.
+  isolate_->builtin_code_data_container_table()[Builtins::ToInt(builtin)] =
+      code.ptr();
+}
+
+CodeT Builtins::codet(Builtin builtin) {
+  CHECK(V8_EXTERNAL_CODE_SPACE_BOOL);
+  Address ptr =
+      isolate_->builtin_code_data_container_table()[Builtins::ToInt(builtin)];
+  return CodeT::cast(Object(ptr));
+}
+
+Handle<CodeT> Builtins::codet_handle(Builtin builtin) {
+  CHECK(V8_EXTERNAL_CODE_SPACE_BOOL);
+  Address* location =
+      &isolate_->builtin_code_data_container_table()[Builtins::ToInt(builtin)];
+  return Handle<CodeT>(location);
+}
+
 // static
 int Builtins::GetStackParameterCount(Builtin builtin) {
   DCHECK(Builtins::KindOf(builtin) == TFJ);
@@ -296,6 +327,17 @@ bool Builtins::IsBuiltinHandle(Handle<HeapObject> maybe_code,
   return true;
 }
 
+bool Builtins::IsBuiltinCodeDataContainerHandle(Handle<HeapObject> maybe_code,
+                                                Builtin* builtin) const {
+  Address* handle_location = maybe_code.location();
+  Address* builtins_table = isolate_->builtin_code_data_container_table();
+  if (handle_location < builtins_table) return false;
+  Address* builtins_table_end = &builtins_table[Builtins::kBuiltinCount];
+  if (handle_location >= builtins_table_end) return false;
+  *builtin = FromInt(static_cast<int>(handle_location - builtins_table));
+  return true;
+}
+
 // static
 bool Builtins::IsIsolateIndependentBuiltin(const Code code) {
   const Builtin builtin = code.builtin_id();

src/builtins/builtins.h

@@ -36,6 +36,13 @@ static constexpr T FirstFromVarArgs(T x, ...) noexcept {
 #define BUILTIN_CODE(isolate, name) \
   (isolate)->builtins()->code_handle(i::Builtin::k##name)
 
+#if V8_EXTERNAL_CODE_SPACE
+#define BUILTIN_CODET(isolate, name) \
+  (isolate)->builtins()->codet_handle(i::Builtin::k##name)
+#else
+#define BUILTIN_CODET(isolate, name) BUILTIN_CODE(isolate, name)
+#endif  // V8_EXTERNAL_CODE_SPACE
+
 enum class Builtin : int32_t {
   kNoBuiltinId = -1,
 #define DEF_ENUM(Name, ...) k##Name,
@@ -158,10 +165,14 @@ class Builtins {
 
   // Used by CreateOffHeapTrampolines in isolate.cc.
   void set_code(Builtin builtin, Code code);
+  void set_codet(Builtin builtin, CodeT code);
 
   V8_EXPORT_PRIVATE Code code(Builtin builtin);
   V8_EXPORT_PRIVATE Handle<Code> code_handle(Builtin builtin);
 
+  V8_EXPORT_PRIVATE CodeT codet(Builtin builtin);
+  V8_EXPORT_PRIVATE Handle<CodeT> codet_handle(Builtin builtin);
+
   static CallInterfaceDescriptor CallInterfaceDescriptorFor(Builtin builtin);
   V8_EXPORT_PRIVATE static Callable CallableFor(Isolate* isolate,
                                                 Builtin builtin);
@@ -192,6 +203,11 @@ class Builtins {
   // by handle location. Similar to Heap::IsRootHandle.
   bool IsBuiltinHandle(Handle<HeapObject> maybe_code, Builtin* index) const;
 
+  // Similar to IsBuiltinHandle but for respective CodeDataContainer handle.
+  // Can be used only when external code space is enabled.
+  bool IsBuiltinCodeDataContainerHandle(Handle<HeapObject> maybe_code,
+                                        Builtin* index) const;
+
   // True, iff the given code object is a builtin with off-heap embedded code.
   static bool IsIsolateIndependentBuiltin(const Code code);
 
@@ -280,6 +296,8 @@ class Builtins {
   FullObjectSlot builtin_slot(Builtin builtin);
   // Returns given builtin's slot in the tier0 builtin table.
   FullObjectSlot builtin_tier0_slot(Builtin builtin);
+  // Returns given builtin's slot in the builtin code data container table.
+  FullObjectSlot builtin_code_data_container_slot(Builtin builtin);
 
  private:
   static void Generate_CallFunction(MacroAssembler* masm,

src/builtins/setup-builtins-internal.cc

@@ -199,6 +199,9 @@ void SetupIsolateDelegate::AddBuiltin(Builtins* builtins, Builtin builtin,
                                       Code code) {
   DCHECK_EQ(builtin, code.builtin_id());
   builtins->set_code(builtin, code);
+  if (V8_EXTERNAL_CODE_SPACE_BOOL) {
+    builtins->set_codet(builtin, ToCodeT(code));
+  }
 }
 
 // static

src/codegen/turbo-assembler.cc

@@ -50,6 +50,12 @@ void TurboAssemblerBase::IndirectLoadConstant(Register destination,
   if (isolate()->roots_table().IsRootHandle(object, &root_index)) {
     // Roots are loaded relative to the root register.
     LoadRoot(destination, root_index);
+  } else if (V8_EXTERNAL_CODE_SPACE_BOOL &&
+             isolate()->builtins()->IsBuiltinCodeDataContainerHandle(
+                 object, &builtin)) {
+    // Similar to roots, builtins may be loaded from the builtins table.
+    LoadRootRelative(destination,
+                     RootRegisterOffsetForBuiltinCodeDataContainer(builtin));
   } else if (isolate()->builtins()->IsBuiltinHandle(object, &builtin)) {
     // Similar to roots, builtins may be loaded from the builtins table.
     LoadRootRelative(destination, RootRegisterOffsetForBuiltin(builtin));
@@ -100,6 +106,12 @@ int32_t TurboAssemblerBase::RootRegisterOffsetForBuiltin(Builtin builtin) {
   return IsolateData::BuiltinSlotOffset(builtin);
 }
 
+// static
+int32_t TurboAssemblerBase::RootRegisterOffsetForBuiltinCodeDataContainer(
+    Builtin builtin) {
+  return IsolateData::BuiltinCodeDataContainerSlotOffset(builtin);
+}
+
 // static
 intptr_t TurboAssemblerBase::RootRegisterOffsetForExternalReference(
     Isolate* isolate, const ExternalReference& reference) {

src/codegen/turbo-assembler.h

@@ -80,6 +80,7 @@ class V8_EXPORT_PRIVATE TurboAssemblerBase : public Assembler {
 
   static int32_t RootRegisterOffsetForRootIndex(RootIndex root_index);
   static int32_t RootRegisterOffsetForBuiltin(Builtin builtin);
+  static int32_t RootRegisterOffsetForBuiltinCodeDataContainer(Builtin builtin);
 
   // Returns the root-relative offset to reference.address().
   static intptr_t RootRegisterOffsetForExternalReference(

src/execution/isolate-data.h

@@ -48,9 +48,19 @@ class Isolate;
     builtin_entry_table)                                                      \
   V(kBuiltinTableOffset, Builtins::kBuiltinCount* kSystemPointerSize,         \
     builtin_table)                                                            \
+  ISOLATE_DATA_FIELDS_EXTERNAL_CODE_SPACE(V)                                  \
   ISOLATE_DATA_FIELDS_HEAP_SANDBOX(V)                                         \
   V(kStackIsIterableOffset, kUInt8Size, stack_is_iterable)
 
+#ifdef V8_EXTERNAL_CODE_SPACE
+#define ISOLATE_DATA_FIELDS_EXTERNAL_CODE_SPACE(V) \
+  V(kBuiltinCodeDataContainerTableOffset,          \
+    Builtins::kBuiltinCount* kSystemPointerSize,   \
+    builtin_code_data_container_table)
+#else
+#define ISOLATE_DATA_FIELDS_EXTERNAL_CODE_SPACE(V)
+#endif  // V8_EXTERNAL_CODE_SPACE
+
 #ifdef V8_HEAP_SANDBOX
 #define ISOLATE_DATA_FIELDS_HEAP_SANDBOX(V) \
   V(kExternalPointerTableOffset, kSystemPointerSize * 3, external_pointer_table)
@@ -104,6 +114,17 @@ class IsolateData final {
            Builtins::ToInt(id) * kSystemPointerSize;
   }
 
+  static int BuiltinCodeDataContainerSlotOffset(Builtin id) {
+#ifdef V8_EXTERNAL_CODE_SPACE
+    // TODO(v8:11880): implement table tiering once the builtin table containing
+    // Code objects is no longer used.
+    return builtin_code_data_container_table_offset() +
+           Builtins::ToInt(id) * kSystemPointerSize;
+#else
+    UNREACHABLE();
+#endif  // V8_EXTERNAL_CODE_SPACE
+  }
+
 #define V(Offset, Size, Name) \
   Address Name##_address() { return reinterpret_cast<Address>(&Name##_); }
   ISOLATE_DATA_FIELDS(V)
@@ -126,6 +147,13 @@ class IsolateData final {
   ThreadLocalTop const& thread_local_top() const { return thread_local_top_; }
   Address* builtin_entry_table() { return builtin_entry_table_; }
   Address* builtin_table() { return builtin_table_; }
+  Address* builtin_code_data_container_table() {
+#ifdef V8_EXTERNAL_CODE_SPACE
+    return builtin_code_data_container_table_;
+#else
+    UNREACHABLE();
+#endif
+  }
   uint8_t stack_is_iterable() const { return stack_is_iterable_; }
 
   // Returns true if this address points to data stored in this instance. If
@@ -201,6 +229,10 @@ class IsolateData final {
   // The entries in this array are tagged pointers to Code objects.
   Address builtin_table_[Builtins::kBuiltinCount] = {};
 
+#ifdef V8_EXTERNAL_CODE_SPACE
+  Address builtin_code_data_container_table_[Builtins::kBuiltinCount] = {};
+#endif
+
   // Table containing pointers to external objects.
 #ifdef V8_HEAP_SANDBOX
   ExternalPointerTable external_pointer_table_;

src/execution/isolate.cc

@@ -3426,6 +3426,9 @@ void CreateOffHeapTrampolines(Isolate* isolate) {
     // From this point onwards, the old builtin code object is unreachable and
     // will be collected by the next GC.
     builtins->set_code(builtin, *trampoline);
+    if (V8_EXTERNAL_CODE_SPACE_BOOL) {
+      builtins->set_codet(builtin, ToCodeT(*trampoline));
+    }
   }
 }
 

src/execution/isolate.h

@@ -1146,6 +1146,9 @@ class V8_EXPORT_PRIVATE Isolate final : private HiddenFactory {
   V8_INLINE Address* builtin_tier0_table() {
     return isolate_data_.builtin_tier0_table();
   }
+  V8_INLINE Address* builtin_code_data_container_table() {
+    return isolate_data_.builtin_code_data_container_table();
+  }
 
   bool IsBuiltinTableHandleLocation(Address* handle_location);
 

src/heap/heap.cc

@@ -4886,8 +4886,12 @@ void Heap::IterateBuiltins(RootVisitor* v) {
   Builtins* builtins = isolate()->builtins();
   for (Builtin builtin = Builtins::kFirst; builtin <= Builtins::kLast;
        ++builtin) {
-    v->VisitRootPointer(Root::kBuiltins, Builtins::name(builtin),
-                        builtins->builtin_slot(builtin));
+    const char* name = Builtins::name(builtin);
+    v->VisitRootPointer(Root::kBuiltins, name, builtins->builtin_slot(builtin));
+    if (V8_EXTERNAL_CODE_SPACE_BOOL) {
+      v->VisitRootPointer(Root::kBuiltins, name,
+                          builtins->builtin_code_data_container_slot(builtin));
+    }
   }
 
   for (Builtin builtin = Builtins::kFirst; builtin <= Builtins::kLastTier0;

src/profiler/heap-snapshot-generator.cc

@@ -1216,7 +1216,8 @@ void V8HeapExplorer::ExtractAccessorPairReferences(HeapEntry* entry,
                        AccessorPair::kSetterOffset);
 }
 
-void V8HeapExplorer::TagBuiltinCodeObject(Code code, const char* name) {
+void V8HeapExplorer::TagBuiltinCodeObject(Object code, const char* name) {
+  DCHECK(code.IsCode() || (V8_EXTERNAL_CODE_SPACE_BOOL && code.IsCodeT()));
   TagObject(code, names_->GetFormatted("(%s builtin)", name));
 }
 
@@ -1577,7 +1578,7 @@ class RootsReferencesExtractor : public RootVisitor {
   void VisitRootPointer(Root root, const char* description,
                         FullObjectSlot object) override {
     if (root == Root::kBuiltins) {
-      explorer_->TagBuiltinCodeObject(Code::cast(*object), description);
+      explorer_->TagBuiltinCodeObject(*object, description);
     }
     explorer_->SetGcSubrootReference(root, description, visiting_weak_roots_,
                                      *object);

src/profiler/heap-snapshot-generator.h

@@ -358,7 +358,7 @@ class V8_EXPORT_PRIVATE V8HeapExplorer : public HeapEntriesAllocator {
   bool IterateAndExtractReferences(HeapSnapshotGenerator* generator);
   void CollectGlobalObjectsTags();
   void MakeGlobalObjectTagMap(const SafepointScope& safepoint_scope);
-  void TagBuiltinCodeObject(Code code, const char* name);
+  void TagBuiltinCodeObject(Object code, const char* name);
   HeapEntry* AddEntry(Address address,
                       HeapEntry::Type type,
                       const char* name,