diff --git a/src/builtins/builtins.cc b/src/builtins/builtins.cc index af1e7490b0..cf839fb1c3 100644 --- a/src/builtins/builtins.cc +++ b/src/builtins/builtins.cc @@ -194,6 +194,37 @@ Handle Builtins::code_handle(Builtin builtin) { return Handle(location); } +FullObjectSlot Builtins::builtin_code_data_container_slot(Builtin builtin) { + CHECK(V8_EXTERNAL_CODE_SPACE_BOOL); + Address* location = + &isolate_->builtin_code_data_container_table()[Builtins::ToInt(builtin)]; + return FullObjectSlot(location); +} + +void Builtins::set_codet(Builtin builtin, CodeT code) { + CHECK(V8_EXTERNAL_CODE_SPACE_BOOL); + // TODO(v8:11880): add DCHECK_EQ(builtin, code.builtin_id()); once CodeT + // has respective field. + DCHECK(Internals::HasHeapObjectTag(code.ptr())); + // The given builtin may be uninitialized thus we cannot check its type here. + isolate_->builtin_code_data_container_table()[Builtins::ToInt(builtin)] = + code.ptr(); +} + +CodeT Builtins::codet(Builtin builtin) { + CHECK(V8_EXTERNAL_CODE_SPACE_BOOL); + Address ptr = + isolate_->builtin_code_data_container_table()[Builtins::ToInt(builtin)]; + return CodeT::cast(Object(ptr)); +} + +Handle Builtins::codet_handle(Builtin builtin) { + CHECK(V8_EXTERNAL_CODE_SPACE_BOOL); + Address* location = + &isolate_->builtin_code_data_container_table()[Builtins::ToInt(builtin)]; + return Handle(location); +} + // static int Builtins::GetStackParameterCount(Builtin builtin) { DCHECK(Builtins::KindOf(builtin) == TFJ); @@ -296,6 +327,17 @@ bool Builtins::IsBuiltinHandle(Handle maybe_code, return true; } +bool Builtins::IsBuiltinCodeDataContainerHandle(Handle maybe_code, + Builtin* builtin) const { + Address* handle_location = maybe_code.location(); + Address* builtins_table = isolate_->builtin_code_data_container_table(); + if (handle_location < builtins_table) return false; + Address* builtins_table_end = &builtins_table[Builtins::kBuiltinCount]; + if (handle_location >= builtins_table_end) return false; + *builtin = FromInt(static_cast(handle_location - builtins_table)); + return true; +} + // static bool Builtins::IsIsolateIndependentBuiltin(const Code code) { const Builtin builtin = code.builtin_id(); diff --git a/src/builtins/builtins.h b/src/builtins/builtins.h index 79e4da840c..99666a63e3 100644 --- a/src/builtins/builtins.h +++ b/src/builtins/builtins.h @@ -36,6 +36,13 @@ static constexpr T FirstFromVarArgs(T x, ...) noexcept { #define BUILTIN_CODE(isolate, name) \ (isolate)->builtins()->code_handle(i::Builtin::k##name) +#if V8_EXTERNAL_CODE_SPACE +#define BUILTIN_CODET(isolate, name) \ + (isolate)->builtins()->codet_handle(i::Builtin::k##name) +#else +#define BUILTIN_CODET(isolate, name) BUILTIN_CODE(isolate, name) +#endif // V8_EXTERNAL_CODE_SPACE + enum class Builtin : int32_t { kNoBuiltinId = -1, #define DEF_ENUM(Name, ...) k##Name, @@ -158,10 +165,14 @@ class Builtins { // Used by CreateOffHeapTrampolines in isolate.cc. void set_code(Builtin builtin, Code code); + void set_codet(Builtin builtin, CodeT code); V8_EXPORT_PRIVATE Code code(Builtin builtin); V8_EXPORT_PRIVATE Handle code_handle(Builtin builtin); + V8_EXPORT_PRIVATE CodeT codet(Builtin builtin); + V8_EXPORT_PRIVATE Handle codet_handle(Builtin builtin); + static CallInterfaceDescriptor CallInterfaceDescriptorFor(Builtin builtin); V8_EXPORT_PRIVATE static Callable CallableFor(Isolate* isolate, Builtin builtin); @@ -192,6 +203,11 @@ class Builtins { // by handle location. Similar to Heap::IsRootHandle. bool IsBuiltinHandle(Handle maybe_code, Builtin* index) const; + // Similar to IsBuiltinHandle but for respective CodeDataContainer handle. + // Can be used only when external code space is enabled. + bool IsBuiltinCodeDataContainerHandle(Handle maybe_code, + Builtin* index) const; + // True, iff the given code object is a builtin with off-heap embedded code. static bool IsIsolateIndependentBuiltin(const Code code); @@ -280,6 +296,8 @@ class Builtins { FullObjectSlot builtin_slot(Builtin builtin); // Returns given builtin's slot in the tier0 builtin table. FullObjectSlot builtin_tier0_slot(Builtin builtin); + // Returns given builtin's slot in the builtin code data container table. + FullObjectSlot builtin_code_data_container_slot(Builtin builtin); private: static void Generate_CallFunction(MacroAssembler* masm, diff --git a/src/builtins/setup-builtins-internal.cc b/src/builtins/setup-builtins-internal.cc index 9dcecdab33..51d7fb48d5 100644 --- a/src/builtins/setup-builtins-internal.cc +++ b/src/builtins/setup-builtins-internal.cc @@ -199,6 +199,9 @@ void SetupIsolateDelegate::AddBuiltin(Builtins* builtins, Builtin builtin, Code code) { DCHECK_EQ(builtin, code.builtin_id()); builtins->set_code(builtin, code); + if (V8_EXTERNAL_CODE_SPACE_BOOL) { + builtins->set_codet(builtin, ToCodeT(code)); + } } // static diff --git a/src/codegen/turbo-assembler.cc b/src/codegen/turbo-assembler.cc index 09c4559813..e1546f71ca 100644 --- a/src/codegen/turbo-assembler.cc +++ b/src/codegen/turbo-assembler.cc @@ -50,6 +50,12 @@ void TurboAssemblerBase::IndirectLoadConstant(Register destination, if (isolate()->roots_table().IsRootHandle(object, &root_index)) { // Roots are loaded relative to the root register. LoadRoot(destination, root_index); + } else if (V8_EXTERNAL_CODE_SPACE_BOOL && + isolate()->builtins()->IsBuiltinCodeDataContainerHandle( + object, &builtin)) { + // Similar to roots, builtins may be loaded from the builtins table. + LoadRootRelative(destination, + RootRegisterOffsetForBuiltinCodeDataContainer(builtin)); } else if (isolate()->builtins()->IsBuiltinHandle(object, &builtin)) { // Similar to roots, builtins may be loaded from the builtins table. LoadRootRelative(destination, RootRegisterOffsetForBuiltin(builtin)); @@ -100,6 +106,12 @@ int32_t TurboAssemblerBase::RootRegisterOffsetForBuiltin(Builtin builtin) { return IsolateData::BuiltinSlotOffset(builtin); } +// static +int32_t TurboAssemblerBase::RootRegisterOffsetForBuiltinCodeDataContainer( + Builtin builtin) { + return IsolateData::BuiltinCodeDataContainerSlotOffset(builtin); +} + // static intptr_t TurboAssemblerBase::RootRegisterOffsetForExternalReference( Isolate* isolate, const ExternalReference& reference) { diff --git a/src/codegen/turbo-assembler.h b/src/codegen/turbo-assembler.h index 2f2deadaac..7403aa1bfd 100644 --- a/src/codegen/turbo-assembler.h +++ b/src/codegen/turbo-assembler.h @@ -80,6 +80,7 @@ class V8_EXPORT_PRIVATE TurboAssemblerBase : public Assembler { static int32_t RootRegisterOffsetForRootIndex(RootIndex root_index); static int32_t RootRegisterOffsetForBuiltin(Builtin builtin); + static int32_t RootRegisterOffsetForBuiltinCodeDataContainer(Builtin builtin); // Returns the root-relative offset to reference.address(). static intptr_t RootRegisterOffsetForExternalReference( diff --git a/src/execution/isolate-data.h b/src/execution/isolate-data.h index d0e5aa87d5..83e00a3c04 100644 --- a/src/execution/isolate-data.h +++ b/src/execution/isolate-data.h @@ -48,9 +48,19 @@ class Isolate; builtin_entry_table) \ V(kBuiltinTableOffset, Builtins::kBuiltinCount* kSystemPointerSize, \ builtin_table) \ + ISOLATE_DATA_FIELDS_EXTERNAL_CODE_SPACE(V) \ ISOLATE_DATA_FIELDS_HEAP_SANDBOX(V) \ V(kStackIsIterableOffset, kUInt8Size, stack_is_iterable) +#ifdef V8_EXTERNAL_CODE_SPACE +#define ISOLATE_DATA_FIELDS_EXTERNAL_CODE_SPACE(V) \ + V(kBuiltinCodeDataContainerTableOffset, \ + Builtins::kBuiltinCount* kSystemPointerSize, \ + builtin_code_data_container_table) +#else +#define ISOLATE_DATA_FIELDS_EXTERNAL_CODE_SPACE(V) +#endif // V8_EXTERNAL_CODE_SPACE + #ifdef V8_HEAP_SANDBOX #define ISOLATE_DATA_FIELDS_HEAP_SANDBOX(V) \ V(kExternalPointerTableOffset, kSystemPointerSize * 3, external_pointer_table) @@ -104,6 +114,17 @@ class IsolateData final { Builtins::ToInt(id) * kSystemPointerSize; } + static int BuiltinCodeDataContainerSlotOffset(Builtin id) { +#ifdef V8_EXTERNAL_CODE_SPACE + // TODO(v8:11880): implement table tiering once the builtin table containing + // Code objects is no longer used. + return builtin_code_data_container_table_offset() + + Builtins::ToInt(id) * kSystemPointerSize; +#else + UNREACHABLE(); +#endif // V8_EXTERNAL_CODE_SPACE + } + #define V(Offset, Size, Name) \ Address Name##_address() { return reinterpret_cast
(&Name##_); } ISOLATE_DATA_FIELDS(V) @@ -126,6 +147,13 @@ class IsolateData final { ThreadLocalTop const& thread_local_top() const { return thread_local_top_; } Address* builtin_entry_table() { return builtin_entry_table_; } Address* builtin_table() { return builtin_table_; } + Address* builtin_code_data_container_table() { +#ifdef V8_EXTERNAL_CODE_SPACE + return builtin_code_data_container_table_; +#else + UNREACHABLE(); +#endif + } uint8_t stack_is_iterable() const { return stack_is_iterable_; } // Returns true if this address points to data stored in this instance. If @@ -201,6 +229,10 @@ class IsolateData final { // The entries in this array are tagged pointers to Code objects. Address builtin_table_[Builtins::kBuiltinCount] = {}; +#ifdef V8_EXTERNAL_CODE_SPACE + Address builtin_code_data_container_table_[Builtins::kBuiltinCount] = {}; +#endif + // Table containing pointers to external objects. #ifdef V8_HEAP_SANDBOX ExternalPointerTable external_pointer_table_; diff --git a/src/execution/isolate.cc b/src/execution/isolate.cc index de396e8732..581f21fdf4 100644 --- a/src/execution/isolate.cc +++ b/src/execution/isolate.cc @@ -3426,6 +3426,9 @@ void CreateOffHeapTrampolines(Isolate* isolate) { // From this point onwards, the old builtin code object is unreachable and // will be collected by the next GC. builtins->set_code(builtin, *trampoline); + if (V8_EXTERNAL_CODE_SPACE_BOOL) { + builtins->set_codet(builtin, ToCodeT(*trampoline)); + } } } diff --git a/src/execution/isolate.h b/src/execution/isolate.h index 8dabf059d6..1ef396e286 100644 --- a/src/execution/isolate.h +++ b/src/execution/isolate.h @@ -1146,6 +1146,9 @@ class V8_EXPORT_PRIVATE Isolate final : private HiddenFactory { V8_INLINE Address* builtin_tier0_table() { return isolate_data_.builtin_tier0_table(); } + V8_INLINE Address* builtin_code_data_container_table() { + return isolate_data_.builtin_code_data_container_table(); + } bool IsBuiltinTableHandleLocation(Address* handle_location); diff --git a/src/heap/heap.cc b/src/heap/heap.cc index c3e549c29a..2ccbb05ac5 100644 --- a/src/heap/heap.cc +++ b/src/heap/heap.cc @@ -4886,8 +4886,12 @@ void Heap::IterateBuiltins(RootVisitor* v) { Builtins* builtins = isolate()->builtins(); for (Builtin builtin = Builtins::kFirst; builtin <= Builtins::kLast; ++builtin) { - v->VisitRootPointer(Root::kBuiltins, Builtins::name(builtin), - builtins->builtin_slot(builtin)); + const char* name = Builtins::name(builtin); + v->VisitRootPointer(Root::kBuiltins, name, builtins->builtin_slot(builtin)); + if (V8_EXTERNAL_CODE_SPACE_BOOL) { + v->VisitRootPointer(Root::kBuiltins, name, + builtins->builtin_code_data_container_slot(builtin)); + } } for (Builtin builtin = Builtins::kFirst; builtin <= Builtins::kLastTier0; diff --git a/src/profiler/heap-snapshot-generator.cc b/src/profiler/heap-snapshot-generator.cc index 75f0daa13b..1f01e86e26 100644 --- a/src/profiler/heap-snapshot-generator.cc +++ b/src/profiler/heap-snapshot-generator.cc @@ -1216,7 +1216,8 @@ void V8HeapExplorer::ExtractAccessorPairReferences(HeapEntry* entry, AccessorPair::kSetterOffset); } -void V8HeapExplorer::TagBuiltinCodeObject(Code code, const char* name) { +void V8HeapExplorer::TagBuiltinCodeObject(Object code, const char* name) { + DCHECK(code.IsCode() || (V8_EXTERNAL_CODE_SPACE_BOOL && code.IsCodeT())); TagObject(code, names_->GetFormatted("(%s builtin)", name)); } @@ -1577,7 +1578,7 @@ class RootsReferencesExtractor : public RootVisitor { void VisitRootPointer(Root root, const char* description, FullObjectSlot object) override { if (root == Root::kBuiltins) { - explorer_->TagBuiltinCodeObject(Code::cast(*object), description); + explorer_->TagBuiltinCodeObject(*object, description); } explorer_->SetGcSubrootReference(root, description, visiting_weak_roots_, *object); diff --git a/src/profiler/heap-snapshot-generator.h b/src/profiler/heap-snapshot-generator.h index 70882153cf..c796690edd 100644 --- a/src/profiler/heap-snapshot-generator.h +++ b/src/profiler/heap-snapshot-generator.h @@ -358,7 +358,7 @@ class V8_EXPORT_PRIVATE V8HeapExplorer : public HeapEntriesAllocator { bool IterateAndExtractReferences(HeapSnapshotGenerator* generator); void CollectGlobalObjectsTags(); void MakeGlobalObjectTagMap(const SafepointScope& safepoint_scope); - void TagBuiltinCodeObject(Code code, const char* name); + void TagBuiltinCodeObject(Object code, const char* name); HeapEntry* AddEntry(Address address, HeapEntry::Type type, const char* name,