From 6f9b2bd48a14dcde35ef0c46aa7d45bf4b307899 Mon Sep 17 00:00:00 2001 From: Igor Sheludko Date: Wed, 18 Sep 2019 18:04:17 +0200 Subject: [PATCH] [ptr-compr] Make on-heap JSTypedArrays smi-corrupting friendly On-heap typed arrays contain HeapObject value in |base_pointer| field and an offset in |external_pointer| field. When pointer compression is enabled we want to combine decompression with the offset addition. In order to do that we add an isolate root to the external_pointer value and therefore the data pointer computation can is a simple addition of a (potentially sign-extended) |base_pointer| loaded as Tagged_t value and an |external_pointer| value. Bug: v8:9706 Change-Id: Id5c546c353c81fb25e3598921bc78165d10a9c44 Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1807369 Reviewed-by: Ulan Degenbaev Reviewed-by: Jakob Gruber Reviewed-by: Toon Verwaest Reviewed-by: Georg Neis Commit-Queue: Igor Sheludko Cr-Commit-Position: refs/heads/master@{#63874} --- src/builtins/base.tq | 9 +-- src/builtins/builtins-array-gen.cc | 4 +- src/builtins/builtins-typed-array-gen.cc | 18 ++--- src/builtins/typed-array-createtypedarray.tq | 24 +++--- src/codegen/code-stub-assembler.cc | 80 ++++++++++++++----- src/codegen/code-stub-assembler.h | 8 +- src/codegen/machine-type.h | 12 +++ src/compiler/access-builder.cc | 9 +-- src/compiler/effect-control-linearizer.cc | 39 +++++---- src/compiler/heap-refs.h | 2 +- src/compiler/js-heap-broker.cc | 12 +-- .../js-native-context-specialization.cc | 12 +-- src/compiler/representation-change.h | 3 + src/compiler/simplified-lowering.cc | 16 ++-- src/diagnostics/objects-printer.cc | 6 ++ src/heap/factory.cc | 4 +- src/ic/accessor-assembler.cc | 27 +++---- src/objects/js-array-buffer-inl.h | 56 ++++++++++--- src/objects/js-array-buffer.cc | 3 +- src/objects/js-array-buffer.h | 32 ++++++-- src/objects/objects.h | 6 +- src/snapshot/deserializer.cc | 21 +++-- src/snapshot/serializer.cc | 38 +++++---- src/snapshot/serializer.h | 2 +- 24 files changed, 286 insertions(+), 157 deletions(-) diff --git a/src/builtins/base.tq b/src/builtins/base.tq index 91b9e55950..5a871ac09d 100644 --- a/src/builtins/base.tq +++ b/src/builtins/base.tq @@ -1329,9 +1329,6 @@ const kStrictReadOnlyProperty: constexpr MessageTemplate const kString: constexpr PrimitiveType generates 'PrimitiveType::kString'; -const kExternalPointerForOnHeapArray: constexpr RawPtr - generates 'JSTypedArray::ExternalPointerForOnHeapArray()'; - const kNameDictionaryInitialCapacity: constexpr int32 generates 'NameDictionary::kInitialCapacity'; @@ -2870,8 +2867,10 @@ extern macro IsMockArrayBufferAllocatorFlag(): bool; extern macro IsPrototypeTypedArrayPrototype(implicit context: Context)(Map): bool; -extern operator '.data_ptr' macro LoadJSTypedArrayBackingStore(JSTypedArray): - RawPtr; +extern operator '.data_ptr' macro LoadJSTypedArrayDataPtr(JSTypedArray): RawPtr; +extern macro SetJSTypedArrayOnHeapDataPtr( + JSTypedArray, ByteArray, uintptr): void; +extern macro SetJSTypedArrayOffHeapDataPtr(JSTypedArray, RawPtr, uintptr): void; extern operator '.elements_kind' macro LoadMapElementsKind(Map): ElementsKind; extern operator '.elements_kind' macro LoadElementsKind(JSTypedArray): diff --git a/src/builtins/builtins-array-gen.cc b/src/builtins/builtins-array-gen.cc index dd78f77647..ca271eecf7 100644 --- a/src/builtins/builtins-array-gen.cc +++ b/src/builtins/builtins-array-gen.cc @@ -212,7 +212,7 @@ void ArrayBuiltinsAssembler::VisitAllTypedArrayElements( list, start, end, [&](TNode index) { GotoIf(IsDetachedBuffer(array_buffer), detached); - TNode data_ptr = LoadJSTypedArrayBackingStore(typed_array); + TNode data_ptr = LoadJSTypedArrayDataPtr(typed_array); TNode value = LoadFixedTypedArrayElementAsTagged( data_ptr, index, source_elements_kind_, SMI_PARAMETERS); k_ = index; @@ -1579,7 +1579,7 @@ TF_BUILTIN(ArrayIteratorPrototypeNext, CodeStubAssembler) { &allocate_iterator_result); TNode elements_kind = LoadMapElementsKind(array_map); - TNode data_ptr = LoadJSTypedArrayBackingStore(CAST(array)); + TNode data_ptr = LoadJSTypedArrayDataPtr(CAST(array)); var_value = LoadFixedTypedArrayElementAsTagged(data_ptr, CAST(index), elements_kind); Goto(&allocate_entry_if_needed); diff --git a/src/builtins/builtins-typed-array-gen.cc b/src/builtins/builtins-typed-array-gen.cc index 94250c145f..79ef28179c 100644 --- a/src/builtins/builtins-typed-array-gen.cc +++ b/src/builtins/builtins-typed-array-gen.cc @@ -316,8 +316,8 @@ void TypedArrayBuiltinsAssembler::SetTypedArraySource( // Grab pointers and byte lengths we need later on. - TNode target_data_ptr = LoadJSTypedArrayBackingStore(target); - TNode source_data_ptr = LoadJSTypedArrayBackingStore(source); + TNode target_data_ptr = LoadJSTypedArrayDataPtr(target); + TNode source_data_ptr = LoadJSTypedArrayDataPtr(source); TNode source_el_kind = LoadElementsKind(source); TNode target_el_kind = LoadElementsKind(target); @@ -749,10 +749,9 @@ TF_BUILTIN(TypedArrayOf, TypedArrayBuiltinsAssembler) { // GC may move backing store in ToNumber, thus load backing // store everytime in this loop. - TNode backing_store = - LoadJSTypedArrayBackingStore(new_typed_array); - StoreElement(backing_store, kind, index, value, - INTPTR_PARAMETERS); + TNode data_ptr = + LoadJSTypedArrayDataPtr(new_typed_array); + StoreElement(data_ptr, kind, index, value, INTPTR_PARAMETERS); }, 1, IndexAdvanceMode::kPost); }); @@ -971,10 +970,9 @@ TF_BUILTIN(TypedArrayFrom, TypedArrayBuiltinsAssembler) { // GC may move backing store in map_fn, thus load backing // store in each iteration of this loop. - TNode backing_store = - LoadJSTypedArrayBackingStore(target_obj.value()); - StoreElement(backing_store, kind, index, final_value, - SMI_PARAMETERS); + TNode data_ptr = + LoadJSTypedArrayDataPtr(target_obj.value()); + StoreElement(data_ptr, kind, index, final_value, SMI_PARAMETERS); }); }, 1, IndexAdvanceMode::kPost); diff --git a/src/builtins/typed-array-createtypedarray.tq b/src/builtins/typed-array-createtypedarray.tq index a476739861..38e6ccaf55 100644 --- a/src/builtins/typed-array-createtypedarray.tq +++ b/src/builtins/typed-array-createtypedarray.tq @@ -27,21 +27,16 @@ namespace typed_array_createtypedarray { isOnHeap: constexpr bool, map: Map, buffer: JSArrayBuffer, byteOffset: uintptr, byteLength: uintptr, length: uintptr): JSTypedArray { let elements: ByteArray; - let externalPointer: RawPtr; - let basePointer: ByteArray | Smi; if constexpr (isOnHeap) { elements = AllocateByteArray(byteLength); - basePointer = elements; - externalPointer = PointerConstant(kExternalPointerForOnHeapArray); } else { - basePointer = Convert(0); + elements = kEmptyByteArray; // The max byteOffset is 8 * MaxSmi on the particular platform. 32 bit // platforms are self-limiting, because we can't allocate an array bigger // than our 32-bit arithmetic range anyway. 64 bit platforms could // theoretically have an offset up to 2^35 - 1. - const backingStore: RawPtr = buffer.backing_store; - externalPointer = backingStore + Convert(byteOffset); + const backingStore: uintptr = Convert(buffer.backing_store); // Assert no overflow has occurred. Only assert if the mock array buffer // allocator is NOT used. When the mock array buffer is used, impossibly @@ -49,9 +44,7 @@ namespace typed_array_createtypedarray { // and this assertion to fail. assert( IsMockArrayBufferAllocatorFlag() || - Convert(externalPointer) >= Convert(backingStore)); - - elements = kEmptyByteArray; + (backingStore + byteOffset) >= backingStore); } // We can't just build the new object with "new JSTypedArray" here because @@ -64,8 +57,15 @@ namespace typed_array_createtypedarray { typedArray.byte_offset = byteOffset; typedArray.byte_length = byteLength; typedArray.length = length; - typedArray.external_pointer = externalPointer; - typedArray.base_pointer = basePointer; + if constexpr (isOnHeap) { + SetJSTypedArrayOnHeapDataPtr(typedArray, elements, byteOffset); + } else { + SetJSTypedArrayOffHeapDataPtr( + typedArray, buffer.backing_store, byteOffset); + assert( + typedArray.data_ptr == + (buffer.backing_store + Convert(byteOffset))); + } SetupTypedArrayEmbedderFields(typedArray); return typedArray; } diff --git a/src/codegen/code-stub-assembler.cc b/src/codegen/code-stub-assembler.cc index 8ad66de442..9822dedf4f 100644 --- a/src/codegen/code-stub-assembler.cc +++ b/src/codegen/code-stub-assembler.cc @@ -2242,16 +2242,60 @@ TNode CodeStubAssembler::LoadPropertyArrayLength( return Signed(DecodeWord(value)); } -TNode CodeStubAssembler::LoadJSTypedArrayBackingStore( +TNode CodeStubAssembler::LoadJSTypedArrayDataPtr( TNode typed_array) { - // Backing store = external_pointer + base_pointer. - Node* external_pointer = - LoadObjectField(typed_array, JSTypedArray::kExternalPointerOffset, - MachineType::Pointer()); - TNode base_pointer = - LoadObjectField(typed_array, JSTypedArray::kBasePointerOffset); - return UncheckedCast( - IntPtrAdd(external_pointer, BitcastTaggedToWord(base_pointer))); + // Data pointer = external_pointer + static_cast(base_pointer). + TNode external_pointer = LoadObjectField( + typed_array, JSTypedArray::kExternalPointerOffset); + + TNode base_pointer; +#ifdef V8_COMPRESS_POINTERS + TNode compressed_base = + LoadObjectField(typed_array, JSTypedArray::kBasePointerOffset); + // Sign extend TaggedT to IntPtrT according to current compression scheme + // so that the addition with |external_pointer| (which already contains + // compensated offset value) below will decompress the tagged value. + // See JSTypedArray::ExternalPointerCompensationForOnHeapArray() for details. + base_pointer = ChangeInt32ToIntPtr(compressed_base); +#else + base_pointer = + LoadObjectField(typed_array, JSTypedArray::kBasePointerOffset); +#endif + return RawPtrAdd(external_pointer, base_pointer); +} + +void CodeStubAssembler::SetJSTypedArrayOffHeapDataPtr( + TNode holder, TNode base, TNode offset) { + StoreObjectFieldNoWriteBarrier(holder, JSTypedArray::kBasePointerOffset, + SmiConstant(0)); + + base = RawPtrAdd(base, Signed(offset)); + StoreObjectFieldNoWriteBarrier( + holder, JSTypedArray::kExternalPointerOffset, base); +} + +void CodeStubAssembler::SetJSTypedArrayOnHeapDataPtr(TNode holder, + TNode base, + TNode offset) { + offset = UintPtrAdd(UintPtrConstant(ByteArray::kHeaderSize - kHeapObjectTag), + offset); + if (COMPRESS_POINTERS_BOOL) { + TNode full_base = Signed(BitcastTaggedToWord(base)); + TNode compressed_base = TruncateIntPtrToInt32(full_base); + // TODO(v8:9706): Add a way to directly use kRootRegister value. + TNode isolate_root = + IntPtrSub(full_base, ChangeInt32ToIntPtr(compressed_base)); + // Add JSTypedArray::ExternalPointerCompensationForOnHeapArray() to offset. + DCHECK_EQ( + isolate()->isolate_root(), + JSTypedArray::ExternalPointerCompensationForOnHeapArray(isolate())); + // See JSTypedArray::SetOnHeapDataPtr() for details. + offset = Unsigned(IntPtrAdd(offset, isolate_root)); + } + + StoreObjectField(holder, JSTypedArray::kBasePointerOffset, base); + StoreObjectFieldNoWriteBarrier( + holder, JSTypedArray::kExternalPointerOffset, offset); } TNode CodeStubAssembler::LoadFixedBigInt64ArrayElementAsTagged( @@ -2537,33 +2581,33 @@ TNode CodeStubAssembler::LoadFixedTypedArrayElementAsTagged( void CodeStubAssembler::StoreJSTypedArrayElementFromTagged( TNode context, TNode typed_array, TNode index_node, TNode value, ElementsKind elements_kind) { - TNode data_pointer = LoadJSTypedArrayBackingStore(typed_array); + TNode data_ptr = LoadJSTypedArrayDataPtr(typed_array); switch (elements_kind) { case UINT8_ELEMENTS: case UINT8_CLAMPED_ELEMENTS: case INT8_ELEMENTS: case UINT16_ELEMENTS: case INT16_ELEMENTS: - StoreElement(data_pointer, elements_kind, index_node, - SmiToInt32(CAST(value)), SMI_PARAMETERS); + StoreElement(data_ptr, elements_kind, index_node, SmiToInt32(CAST(value)), + SMI_PARAMETERS); break; case UINT32_ELEMENTS: case INT32_ELEMENTS: - StoreElement(data_pointer, elements_kind, index_node, + StoreElement(data_ptr, elements_kind, index_node, TruncateTaggedToWord32(context, value), SMI_PARAMETERS); break; case FLOAT32_ELEMENTS: - StoreElement(data_pointer, elements_kind, index_node, + StoreElement(data_ptr, elements_kind, index_node, TruncateFloat64ToFloat32(LoadHeapNumberValue(CAST(value))), SMI_PARAMETERS); break; case FLOAT64_ELEMENTS: - StoreElement(data_pointer, elements_kind, index_node, + StoreElement(data_ptr, elements_kind, index_node, LoadHeapNumberValue(CAST(value)), SMI_PARAMETERS); break; case BIGUINT64_ELEMENTS: case BIGINT64_ELEMENTS: - StoreElement(data_pointer, elements_kind, index_node, + StoreElement(data_ptr, elements_kind, index_node, UncheckedCast(value), SMI_PARAMETERS); break; default: @@ -10873,8 +10917,8 @@ void CodeStubAssembler::EmitElementStore(Node* object, Node* key, Node* value, GotoIfNot(UintPtrLessThan(intptr_key, length), &update_value_and_bailout); } - TNode backing_store = LoadJSTypedArrayBackingStore(CAST(object)); - StoreElement(backing_store, elements_kind, intptr_key, converted_value, + TNode data_ptr = LoadJSTypedArrayDataPtr(CAST(object)); + StoreElement(data_ptr, elements_kind, intptr_key, converted_value, parameter_mode); Goto(&done); diff --git a/src/codegen/code-stub-assembler.h b/src/codegen/code-stub-assembler.h index f5a05caf04..bbfdccf163 100644 --- a/src/codegen/code-stub-assembler.h +++ b/src/codegen/code-stub-assembler.h @@ -3536,7 +3536,13 @@ class V8_EXPORT_PRIVATE CodeStubAssembler // JSTypedArray helpers TNode LoadJSTypedArrayLength(TNode typed_array); - TNode LoadJSTypedArrayBackingStore(TNode typed_array); + TNode LoadJSTypedArrayDataPtr(TNode typed_array); + void SetJSTypedArrayOnHeapDataPtr(TNode holder, + TNode base, + TNode offset); + void SetJSTypedArrayOffHeapDataPtr(TNode holder, + TNode base, + TNode offset); template TNode ElementOffsetFromIndex(TNode index, ElementsKind kind, diff --git a/src/codegen/machine-type.h b/src/codegen/machine-type.h index 15e3df65c5..be850999de 100644 --- a/src/codegen/machine-type.h +++ b/src/codegen/machine-type.h @@ -114,6 +114,10 @@ class MachineType { constexpr bool IsCompressedPointer() const { return representation() == MachineRepresentation::kCompressedPointer; } + constexpr static MachineRepresentation TaggedRepresentation() { + return (kTaggedSize == 4) ? MachineRepresentation::kWord32 + : MachineRepresentation::kWord64; + } constexpr static MachineRepresentation PointerRepresentation() { return (kSystemPointerSize == 4) ? MachineRepresentation::kWord32 : MachineRepresentation::kWord64; @@ -260,6 +264,14 @@ class MachineType { #endif } + constexpr static MachineType TypeRawTagged() { +#ifdef V8_COMPRESS_POINTERS + return MachineType::Int32(); +#else + return MachineType::Pointer(); +#endif + } + constexpr static MachineType TypeCompressedTagged() { #ifdef V8_COMPRESS_POINTERS return MachineType::AnyCompressed(); diff --git a/src/compiler/access-builder.cc b/src/compiler/access-builder.cc index 7a72be8028..ab75e72d27 100644 --- a/src/compiler/access-builder.cc +++ b/src/compiler/access-builder.cc @@ -426,11 +426,10 @@ FieldAccess AccessBuilder::ForJSTypedArrayLength() { // static FieldAccess AccessBuilder::ForJSTypedArrayBasePointer() { - FieldAccess access = { - kTaggedBase, JSTypedArray::kBasePointerOffset, - MaybeHandle(), MaybeHandle(), - Type::OtherInternal(), MachineType::TypeCompressedTagged(), - kFullWriteBarrier, LoadSensitivity::kCritical}; + FieldAccess access = {kTaggedBase, JSTypedArray::kBasePointerOffset, + MaybeHandle(), MaybeHandle(), + Type::OtherInternal(), MachineType::TypeRawTagged(), + kFullWriteBarrier, LoadSensitivity::kCritical}; return access; } diff --git a/src/compiler/effect-control-linearizer.cc b/src/compiler/effect-control-linearizer.cc index 71df72988a..d1941fbd2f 100644 --- a/src/compiler/effect-control-linearizer.cc +++ b/src/compiler/effect-control-linearizer.cc @@ -227,6 +227,8 @@ class EffectControlLinearizer { Node* LowerStringComparison(Callable const& callable, Node* node); Node* IsElementsKindGreaterThan(Node* kind, ElementsKind reference_kind); + Node* BuildTypedArrayDataPointer(Node* base, Node* external); + Node* ChangeInt32ToCompressedSmi(Node* value); Node* ChangeInt32ToSmi(Node* value); Node* ChangeInt32ToIntPtr(Node* value); @@ -5003,6 +5005,25 @@ void EffectControlLinearizer::LowerStoreDataViewElement(Node* node) { done.PhiAt(0)); } +// Compute the data pointer, handling the case where the {external} pointer +// is the effective data pointer (i.e. the {base} is Smi zero). +Node* EffectControlLinearizer::BuildTypedArrayDataPointer(Node* base, + Node* external) { + if (IntPtrMatcher(base).Is(0)) { + return external; + } else { + if (COMPRESS_POINTERS_BOOL) { + // Sign-extend Tagged_t to IntPtr according to current compression + // scheme so that the addition with |external_pointer| (which already + // contains compensated offset value) will decompress the tagged value. + // See JSTypedArray::ExternalPointerCompensationForOnHeapArray() for + // details. + base = ChangeInt32ToIntPtr(base); + } + return __ UnsafePointerAdd(base, external); + } +} + Node* EffectControlLinearizer::LowerLoadTypedElement(Node* node) { ExternalArrayType array_type = ExternalArrayTypeOf(node->op()); Node* buffer = node->InputAt(0); @@ -5014,17 +5035,12 @@ Node* EffectControlLinearizer::LowerLoadTypedElement(Node* node) { // ArrayBuffer (if there's any) as long as we are still operating on it. __ Retain(buffer); - // Compute the effective storage pointer, handling the case where the - // {external} pointer is the effective storage pointer (i.e. the {base} - // is Smi zero). - Node* storage = IntPtrMatcher(base).Is(0) - ? external - : __ UnsafePointerAdd(base, external); + Node* data_ptr = BuildTypedArrayDataPointer(base, external); // Perform the actual typed element access. return __ LoadElement(AccessBuilder::ForTypedArrayElement( array_type, true, LoadSensitivity::kCritical), - storage, index); + data_ptr, index); } void EffectControlLinearizer::LowerStoreTypedElement(Node* node) { @@ -5039,16 +5055,11 @@ void EffectControlLinearizer::LowerStoreTypedElement(Node* node) { // ArrayBuffer (if there's any) as long as we are still operating on it. __ Retain(buffer); - // Compute the effective storage pointer, handling the case where the - // {external} pointer is the effective storage pointer (i.e. the {base} - // is Smi zero). - Node* storage = IntPtrMatcher(base).Is(0) - ? external - : __ UnsafePointerAdd(base, external); + Node* data_ptr = BuildTypedArrayDataPointer(base, external); // Perform the actual typed element access. __ StoreElement(AccessBuilder::ForTypedArrayElement(array_type, true), - storage, index, value); + data_ptr, index, value); } void EffectControlLinearizer::TransitionElementsTo(Node* node, Node* array, diff --git a/src/compiler/heap-refs.h b/src/compiler/heap-refs.h index 203eeee12b..272361c5be 100644 --- a/src/compiler/heap-refs.h +++ b/src/compiler/heap-refs.h @@ -827,7 +827,7 @@ class JSTypedArrayRef : public JSObjectRef { bool is_on_heap() const; size_t length() const; - void* external_pointer() const; + void* data_ptr() const; void Serialize(); bool serialized() const; diff --git a/src/compiler/js-heap-broker.cc b/src/compiler/js-heap-broker.cc index 208ebdccd6..1cd2ecddb9 100644 --- a/src/compiler/js-heap-broker.cc +++ b/src/compiler/js-heap-broker.cc @@ -420,7 +420,7 @@ class JSTypedArrayData : public JSObjectData { bool is_on_heap() const { return is_on_heap_; } size_t length() const { return length_; } - void* external_pointer() const { return external_pointer_; } + void* data_ptr() const { return data_ptr_; } void Serialize(JSHeapBroker* broker); bool serialized() const { return serialized_; } @@ -430,7 +430,7 @@ class JSTypedArrayData : public JSObjectData { private: bool const is_on_heap_; size_t const length_; - void* const external_pointer_; + void* const data_ptr_; bool serialized_ = false; HeapObjectData* buffer_ = nullptr; @@ -441,7 +441,7 @@ JSTypedArrayData::JSTypedArrayData(JSHeapBroker* broker, ObjectData** storage, : JSObjectData(broker, storage, object), is_on_heap_(object->is_on_heap()), length_(object->length()), - external_pointer_(object->external_pointer()) {} + data_ptr_(object->DataPtr()) {} void JSTypedArrayData::Serialize(JSHeapBroker* broker) { if (serialized_) return; @@ -3383,12 +3383,12 @@ base::Optional MapRef::FindRootMap() const { return base::nullopt; } -void* JSTypedArrayRef::external_pointer() const { +void* JSTypedArrayRef::data_ptr() const { if (broker()->mode() == JSHeapBroker::kDisabled) { AllowHandleDereference allow_handle_dereference; - return object()->external_pointer(); + return object()->DataPtr(); } - return data()->AsJSTypedArray()->external_pointer(); + return data()->AsJSTypedArray()->data_ptr(); } bool MapRef::IsInobjectSlackTrackingInProgress() const { diff --git a/src/compiler/js-native-context-specialization.cc b/src/compiler/js-native-context-specialization.cc index 6a9209136c..2b1f6b300b 100644 --- a/src/compiler/js-native-context-specialization.cc +++ b/src/compiler/js-native-context-specialization.cc @@ -2591,12 +2591,14 @@ JSNativeContextSpecialization::BuildElementAccess( if (typed_array.has_value()) { length = jsgraph()->Constant(static_cast(typed_array->length())); - // Load the (known) base and external pointer for the {receiver}. The - // {external_pointer} might be invalid if the {buffer} was detached, so - // we need to make sure that any access is properly guarded. + DCHECK(!typed_array->is_on_heap()); + // Load the (known) data pointer for the {receiver} and set {base_pointer} + // and {external_pointer} to the values that will allow to generate typed + // element accesses using the known data pointer. + // The data pointer might be invalid if the {buffer} was detached, + // so we need to make sure that any access is properly guarded. base_pointer = jsgraph()->ZeroConstant(); - external_pointer = - jsgraph()->PointerConstant(typed_array->external_pointer()); + external_pointer = jsgraph()->PointerConstant(typed_array->data_ptr()); } else { // Load the {receiver}s length. length = effect = graph()->NewNode( diff --git a/src/compiler/representation-change.h b/src/compiler/representation-change.h index 43e85085ba..246e6143f2 100644 --- a/src/compiler/representation-change.h +++ b/src/compiler/representation-change.h @@ -187,6 +187,9 @@ class UseInfo { static UseInfo Word() { return UseInfo(MachineType::PointerRepresentation(), Truncation::Any()); } + static UseInfo TaggedWord() { + return UseInfo(MachineType::TaggedRepresentation(), Truncation::Any()); + } static UseInfo Bool() { return UseInfo(MachineRepresentation::kBit, Truncation::Bool()); } diff --git a/src/compiler/simplified-lowering.cc b/src/compiler/simplified-lowering.cc index 1ca7bfe707..78aef08047 100644 --- a/src/compiler/simplified-lowering.cc +++ b/src/compiler/simplified-lowering.cc @@ -3020,10 +3020,10 @@ class RepresentationSelector { case IrOpcode::kLoadTypedElement: { MachineRepresentation const rep = MachineRepresentationFromArrayType(ExternalArrayTypeOf(node->op())); - ProcessInput(node, 0, UseInfo::AnyTagged()); // buffer - ProcessInput(node, 1, UseInfo::AnyTagged()); // base pointer - ProcessInput(node, 2, UseInfo::Word()); // external pointer - ProcessInput(node, 3, UseInfo::Word()); // index + ProcessInput(node, 0, UseInfo::AnyTagged()); // buffer + ProcessInput(node, 1, UseInfo::TaggedWord()); // base pointer + ProcessInput(node, 2, UseInfo::Word()); // external pointer + ProcessInput(node, 3, UseInfo::Word()); // index ProcessRemainingInputs(node, 4); SetOutput(node, rep); return; @@ -3042,10 +3042,10 @@ class RepresentationSelector { case IrOpcode::kStoreTypedElement: { MachineRepresentation const rep = MachineRepresentationFromArrayType(ExternalArrayTypeOf(node->op())); - ProcessInput(node, 0, UseInfo::AnyTagged()); // buffer - ProcessInput(node, 1, UseInfo::AnyTagged()); // base pointer - ProcessInput(node, 2, UseInfo::Word()); // external pointer - ProcessInput(node, 3, UseInfo::Word()); // index + ProcessInput(node, 0, UseInfo::AnyTagged()); // buffer + ProcessInput(node, 1, UseInfo::TaggedWord()); // base pointer + ProcessInput(node, 2, UseInfo::Word()); // external pointer + ProcessInput(node, 3, UseInfo::Word()); // index ProcessInput(node, 4, TruncatingUseInfoFromRepresentation(rep)); // value ProcessRemainingInputs(node, 5); diff --git a/src/diagnostics/objects-printer.cc b/src/diagnostics/objects-printer.cc index b5843b3133..20c8e9f844 100644 --- a/src/diagnostics/objects-printer.cc +++ b/src/diagnostics/objects-printer.cc @@ -1388,6 +1388,12 @@ void JSTypedArray::JSTypedArrayPrint(std::ostream& os) { // NOLINT os << "\n - byte_offset: " << byte_offset(); os << "\n - byte_length: " << byte_length(); os << "\n - length: " << length(); + os << "\n - data_ptr: " << DataPtr(); + Tagged_t base_pointer = static_cast(base_pointer_raw().ptr()); + os << "\n - base_pointer: " + << reinterpret_cast(static_cast
(base_pointer)); + os << "\n - external_pointer: " + << reinterpret_cast(external_pointer_raw()); if (!buffer().IsJSArrayBuffer()) { os << "\n \n"; return; diff --git a/src/heap/factory.cc b/src/heap/factory.cc index 7d2f7e39b6..e9d27666be 100644 --- a/src/heap/factory.cc +++ b/src/heap/factory.cc @@ -3252,9 +3252,7 @@ Handle Factory::NewJSTypedArray(ExternalArrayType type, Handle::cast(NewJSArrayBufferView( map, empty_byte_array(), buffer, byte_offset, byte_length)); typed_array->set_length(length); - typed_array->set_external_pointer( - reinterpret_cast(buffer->backing_store()) + byte_offset); - typed_array->set_base_pointer(Smi::kZero); + typed_array->SetOffHeapDataPtr(buffer->backing_store(), byte_offset); return typed_array; } diff --git a/src/ic/accessor-assembler.cc b/src/ic/accessor-assembler.cc index 2f7e2d3a00..3c94b13183 100644 --- a/src/ic/accessor-assembler.cc +++ b/src/ic/accessor-assembler.cc @@ -2096,8 +2096,7 @@ void AccessorAssembler::EmitElementLoad( if (access_mode == LoadAccessMode::kHas) { exit_point->Return(TrueConstant()); } else { - TNode backing_store = - LoadJSTypedArrayBackingStore(CAST(object)); + TNode data_ptr = LoadJSTypedArrayDataPtr(CAST(object)); Label uint8_elements(this), int8_elements(this), uint16_elements(this), int16_elements(this), uint32_elements(this), int32_elements(this), @@ -2123,50 +2122,48 @@ void AccessorAssembler::EmitElementLoad( BIND(&uint8_elements); { Comment("UINT8_ELEMENTS"); // Handles UINT8_CLAMPED_ELEMENTS too. - Node* element = - Load(MachineType::Uint8(), backing_store, intptr_index); + Node* element = Load(MachineType::Uint8(), data_ptr, intptr_index); exit_point->Return(SmiFromInt32(element)); } BIND(&int8_elements); { Comment("INT8_ELEMENTS"); - Node* element = - Load(MachineType::Int8(), backing_store, intptr_index); + Node* element = Load(MachineType::Int8(), data_ptr, intptr_index); exit_point->Return(SmiFromInt32(element)); } BIND(&uint16_elements); { Comment("UINT16_ELEMENTS"); TNode index = WordShl(intptr_index, IntPtrConstant(1)); - Node* element = Load(MachineType::Uint16(), backing_store, index); + Node* element = Load(MachineType::Uint16(), data_ptr, index); exit_point->Return(SmiFromInt32(element)); } BIND(&int16_elements); { Comment("INT16_ELEMENTS"); TNode index = WordShl(intptr_index, IntPtrConstant(1)); - Node* element = Load(MachineType::Int16(), backing_store, index); + Node* element = Load(MachineType::Int16(), data_ptr, index); exit_point->Return(SmiFromInt32(element)); } BIND(&uint32_elements); { Comment("UINT32_ELEMENTS"); TNode index = WordShl(intptr_index, IntPtrConstant(2)); - Node* element = Load(MachineType::Uint32(), backing_store, index); + Node* element = Load(MachineType::Uint32(), data_ptr, index); exit_point->Return(ChangeUint32ToTagged(element)); } BIND(&int32_elements); { Comment("INT32_ELEMENTS"); TNode index = WordShl(intptr_index, IntPtrConstant(2)); - Node* element = Load(MachineType::Int32(), backing_store, index); + Node* element = Load(MachineType::Int32(), data_ptr, index); exit_point->Return(ChangeInt32ToTagged(element)); } BIND(&float32_elements); { Comment("FLOAT32_ELEMENTS"); TNode index = WordShl(intptr_index, IntPtrConstant(2)); - Node* element = Load(MachineType::Float32(), backing_store, index); + Node* element = Load(MachineType::Float32(), data_ptr, index); var_double_value->Bind(ChangeFloat32ToFloat64(element)); Goto(rebox_double); } @@ -2174,7 +2171,7 @@ void AccessorAssembler::EmitElementLoad( { Comment("FLOAT64_ELEMENTS"); TNode index = WordShl(intptr_index, IntPtrConstant(3)); - Node* element = Load(MachineType::Float64(), backing_store, index); + Node* element = Load(MachineType::Float64(), data_ptr, index); var_double_value->Bind(element); Goto(rebox_double); } @@ -2182,15 +2179,13 @@ void AccessorAssembler::EmitElementLoad( { Comment("BIGINT64_ELEMENTS"); exit_point->Return(LoadFixedTypedArrayElementAsTagged( - backing_store, intptr_index, BIGINT64_ELEMENTS, - INTPTR_PARAMETERS)); + data_ptr, intptr_index, BIGINT64_ELEMENTS, INTPTR_PARAMETERS)); } BIND(&biguint64_elements); { Comment("BIGUINT64_ELEMENTS"); exit_point->Return(LoadFixedTypedArrayElementAsTagged( - backing_store, intptr_index, BIGUINT64_ELEMENTS, - INTPTR_PARAMETERS)); + data_ptr, intptr_index, BIGUINT64_ELEMENTS, INTPTR_PARAMETERS)); } } } diff --git a/src/objects/js-array-buffer-inl.h b/src/objects/js-array-buffer-inl.h index a107ed4e2f..80aa2d987e 100644 --- a/src/objects/js-array-buffer-inl.h +++ b/src/objects/js-array-buffer-inl.h @@ -114,31 +114,63 @@ void JSTypedArray::set_length(size_t value) { WriteField(kLengthOffset, value); } -void* JSTypedArray::external_pointer() const { - return reinterpret_cast(ReadField
(kExternalPointerOffset)); +Address JSTypedArray::external_pointer_raw() const { + return ReadField
(kExternalPointerOffset); } -void JSTypedArray::set_external_pointer(void* value) { - WriteField
(kExternalPointerOffset, reinterpret_cast
(value)); +void JSTypedArray::set_external_pointer_raw(Address value) { + WriteField
(kExternalPointerOffset, value); } -ACCESSORS(JSTypedArray, base_pointer, Object, kBasePointerOffset) +Address JSTypedArray::ExternalPointerCompensationForOnHeapArray( + Isolate* isolate) { +#ifdef V8_COMPRESS_POINTERS + return GetIsolateRoot(isolate); +#else + return 0; +#endif +} + +void JSTypedArray::RemoveExternalPointerCompensationForSerialization() { + DCHECK(is_on_heap()); + Isolate* isolate = GetIsolateForPtrCompr(*this); + set_external_pointer_raw(external_pointer_raw() - + ExternalPointerCompensationForOnHeapArray(isolate)); +} + +ACCESSORS(JSTypedArray, base_pointer_raw, Object, kBasePointerOffset) void* JSTypedArray::DataPtr() { + // Sign extend Tagged_t to intptr_t according to current compression scheme + // so that the addition with |external_pointer| (which already contains + // compensated offset value) will decompress the tagged value. + // See JSTypedArray::ExternalPointerCompensationForOnHeapArray() for details. return reinterpret_cast( - base_pointer().ptr() + reinterpret_cast(external_pointer())); + external_pointer_raw() + + static_cast
(static_cast( + static_cast(base_pointer_raw().ptr())))); +} + +void JSTypedArray::SetOffHeapDataPtr(void* base, Address offset) { + set_base_pointer_raw(Smi::kZero, SKIP_WRITE_BARRIER); + Address address = reinterpret_cast
(base) + offset; + set_external_pointer_raw(address); + DCHECK_EQ(address, reinterpret_cast
(DataPtr())); +} + +void JSTypedArray::SetOnHeapDataPtr(HeapObject base, Address offset) { + set_base_pointer_raw(base); + Isolate* isolate = GetIsolateForPtrCompr(*this); + set_external_pointer_raw(offset + + ExternalPointerCompensationForOnHeapArray(isolate)); + DCHECK_EQ(base.ptr() + offset, reinterpret_cast
(DataPtr())); } bool JSTypedArray::is_on_heap() const { DisallowHeapAllocation no_gc; // Checking that buffer()->backing_store() is not nullptr is not sufficient; // it will be nullptr when byte_length is 0 as well. - return base_pointer().ptr() == elements().ptr(); -} - -// static -void* JSTypedArray::ExternalPointerForOnHeapArray() { - return reinterpret_cast(ByteArray::kHeaderSize - kHeapObjectTag); + return base_pointer_raw() == elements(); } // static diff --git a/src/objects/js-array-buffer.cc b/src/objects/js-array-buffer.cc index be1773ecfd..e26bff9210 100644 --- a/src/objects/js-array-buffer.cc +++ b/src/objects/js-array-buffer.cc @@ -123,8 +123,7 @@ Handle JSTypedArray::GetBuffer() { // Clear the elements of the typed array. self->set_elements(ReadOnlyRoots(isolate).empty_byte_array()); - self->set_external_pointer(array_buffer->backing_store()); - self->set_base_pointer(Smi::kZero); + self->SetOffHeapDataPtr(array_buffer->backing_store(), 0); DCHECK(!self->is_on_heap()); return array_buffer; diff --git a/src/objects/js-array-buffer.h b/src/objects/js-array-buffer.h index 4675d5f048..b76302961d 100644 --- a/src/objects/js-array-buffer.h +++ b/src/objects/js-array-buffer.h @@ -177,12 +177,6 @@ class JSTypedArray : public JSArrayBufferView { // [length]: length of typed array in elements. DECL_PRIMITIVE_ACCESSORS(length, size_t) - // [external_pointer]: TODO(v8:4153) - DECL_PRIMITIVE_ACCESSORS(external_pointer, void*) - - // [base_pointer]: TODO(v8:4153) - DECL_ACCESSORS(base_pointer, Object) - // ES6 9.4.5.3 V8_WARN_UNUSED_RESULT static Maybe DefineOwnProperty( Isolate* isolate, Handle o, Handle key, @@ -198,10 +192,26 @@ class JSTypedArray : public JSArrayBufferView { // Use with care: returns raw pointer into heap. inline void* DataPtr(); + inline void SetOffHeapDataPtr(void* base, Address offset); + inline void SetOnHeapDataPtr(HeapObject base, Address offset); + // Whether the buffer's backing store is on-heap or off-heap. inline bool is_on_heap() const; - static inline void* ExternalPointerForOnHeapArray(); + // Note: this is a pointer compression specific optimization. + // Normally, on-heap typed arrays contain HeapObject value in |base_pointer| + // field and an offset in |external_pointer|. + // When pointer compression is enabled we want to combine decompression with + // the offset addition. In order to do that we add an isolate root to the + // |external_pointer| value and therefore the data pointer computation can + // is a simple addition of a (potentially sign-extended) |base_pointer| loaded + // as Tagged_t value and an |external_pointer| value. + // For full-pointer mode the compensation value is zero. + static inline Address ExternalPointerCompensationForOnHeapArray( + Isolate* isolate); + + // Subtracts external pointer compensation from the external pointer value. + inline void RemoveExternalPointerCompensationForSerialization(); static inline MaybeHandle Validate(Isolate* isolate, Handle receiver, @@ -240,6 +250,14 @@ class JSTypedArray : public JSArrayBufferView { #endif private: + friend class Deserializer; + + // [base_pointer]: TODO(v8:4153) + DECL_ACCESSORS(base_pointer_raw, Object) + + // [external_pointer]: TODO(v8:4153) + DECL_PRIMITIVE_ACCESSORS(external_pointer_raw, Address) + OBJECT_CONSTRUCTORS(JSTypedArray, JSArrayBufferView); }; diff --git a/src/objects/objects.h b/src/objects/objects.h index b4e78a1937..096167e0d4 100644 --- a/src/objects/objects.h +++ b/src/objects/objects.h @@ -607,15 +607,13 @@ class Object : public TaggedImpl { // For use with std::unordered_set. struct Hasher { size_t operator()(const Object o) const { - return std::hash{}(o.ptr()); + return std::hash{}(static_cast(o.ptr())); } }; // For use with std::map. struct Comparer { - bool operator()(const Object a, const Object b) const { - return a.ptr() < b.ptr(); - } + bool operator()(const Object a, const Object b) const { return a < b; } }; template ::value, diff --git a/src/snapshot/deserializer.cc b/src/snapshot/deserializer.cc index 1b4fc655b4..f154f5da98 100644 --- a/src/snapshot/deserializer.cc +++ b/src/snapshot/deserializer.cc @@ -291,22 +291,27 @@ HeapObject Deserializer::PostProcessNewObject(HeapObject obj, data_view.byte_offset()); } else if (obj.IsJSTypedArray()) { JSTypedArray typed_array = JSTypedArray::cast(obj); - // Only fixup for the off-heap case. - if (!typed_array.is_on_heap()) { - Smi store_index( - reinterpret_cast
(typed_array.external_pointer())); - auto backing_store = backing_stores_[store_index.value()]; + // Fixup typed array pointers. + if (typed_array.is_on_heap()) { + typed_array.SetOnHeapDataPtr( + HeapObject::cast(typed_array.base_pointer_raw()), + typed_array.external_pointer_raw()); + } else { + // Serializer writes backing store ref as a DataPtr() value. + size_t store_index = reinterpret_cast(typed_array.DataPtr()); + auto backing_store = backing_stores_[store_index]; auto start = backing_store ? reinterpret_cast(backing_store->buffer_start()) : nullptr; - typed_array.set_external_pointer(start + typed_array.byte_offset()); + typed_array.SetOffHeapDataPtr(start, typed_array.byte_offset()); } } else if (obj.IsJSArrayBuffer()) { JSArrayBuffer buffer = JSArrayBuffer::cast(obj); // Only fixup for the off-heap case. if (buffer.backing_store() != nullptr) { - Smi store_index(reinterpret_cast
(buffer.backing_store())); - auto backing_store = backing_stores_[store_index.value()]; + // Serializer writes backing store ref in |backing_store| field. + size_t store_index = reinterpret_cast(buffer.backing_store()); + auto backing_store = backing_stores_[store_index]; if (backing_store) { buffer.Attach(backing_store); } else { diff --git a/src/snapshot/serializer.cc b/src/snapshot/serializer.cc index 5b68aaa87b..f009f08fc7 100644 --- a/src/snapshot/serializer.cc +++ b/src/snapshot/serializer.cc @@ -342,7 +342,7 @@ void Serializer::ObjectSerializer::SerializePrologue(SnapshotSpace space, serializer_->SerializeObject(map); } -int32_t Serializer::ObjectSerializer::SerializeBackingStore( +uint32_t Serializer::ObjectSerializer::SerializeBackingStore( void* backing_store, int32_t byte_length) { SerializerReference reference = serializer_->reference_map()->LookupReference(backing_store); @@ -358,13 +358,15 @@ int32_t Serializer::ObjectSerializer::SerializeBackingStore( serializer_->reference_map()->Add(backing_store, reference); } - return static_cast(reference.off_heap_backing_store_index()); + return reference.off_heap_backing_store_index(); } void Serializer::ObjectSerializer::SerializeJSTypedArray() { JSTypedArray typed_array = JSTypedArray::cast(object_); - if (!typed_array.WasDetached()) { - if (!typed_array.is_on_heap()) { + if (typed_array.is_on_heap()) { + typed_array.RemoveExternalPointerCompensationForSerialization(); + } else { + if (!typed_array.WasDetached()) { // Explicitly serialize the backing store now. JSArrayBuffer buffer = JSArrayBuffer::cast(typed_array.buffer()); CHECK_LE(buffer.byte_length(), Smi::kMaxValue); @@ -372,21 +374,20 @@ void Serializer::ObjectSerializer::SerializeJSTypedArray() { int32_t byte_length = static_cast(buffer.byte_length()); int32_t byte_offset = static_cast(typed_array.byte_offset()); - // We need to calculate the backing store from the external pointer + // We need to calculate the backing store from the data pointer // because the ArrayBuffer may already have been serialized. void* backing_store = reinterpret_cast( - reinterpret_cast(typed_array.external_pointer()) - - byte_offset); - int32_t ref = SerializeBackingStore(backing_store, byte_length); + reinterpret_cast
(typed_array.DataPtr()) - byte_offset); - // The external_pointer is the backing_store + typed_array->byte_offset. - // To properly share the buffer, we set the backing store ref here. On - // deserialization we re-add the byte_offset to external_pointer. - typed_array.set_external_pointer( - reinterpret_cast(Smi::FromInt(ref).ptr())); + uint32_t ref = SerializeBackingStore(backing_store, byte_length); + // To properly share the buffer, we set the backing store ref as an + // off-heap offset from nullptr. On deserialization we re-set data + // pointer to proper value. + typed_array.SetOffHeapDataPtr(nullptr, ref); + DCHECK_EQ(ref, reinterpret_cast
(typed_array.DataPtr())); + } else { + typed_array.SetOffHeapDataPtr(nullptr, 0); } - } else { - typed_array.set_external_pointer(nullptr); } SerializeObject(); } @@ -400,8 +401,11 @@ void Serializer::ObjectSerializer::SerializeJSArrayBuffer() { // The embedder-allocated backing store only exists for the off-heap case. if (backing_store != nullptr) { - int32_t ref = SerializeBackingStore(backing_store, byte_length); - buffer.set_backing_store(reinterpret_cast(Smi::FromInt(ref).ptr())); + uint32_t ref = SerializeBackingStore(backing_store, byte_length); + // To properly share the buffer, we set the backing store ref as an + // a backing store address. On deserialization we re-set data pointer + // to proper value. + buffer.set_backing_store(reinterpret_cast(static_cast(ref))); } SerializeObject(); buffer.set_backing_store(backing_store); diff --git a/src/snapshot/serializer.h b/src/snapshot/serializer.h index fad2ec8a88..73a9a1eaac 100644 --- a/src/snapshot/serializer.h +++ b/src/snapshot/serializer.h @@ -328,7 +328,7 @@ class Serializer::ObjectSerializer : public ObjectVisitor { void SerializeContent(Map map, int size); void OutputRawData(Address up_to); void OutputCode(int size); - int32_t SerializeBackingStore(void* backing_store, int32_t byte_length); + uint32_t SerializeBackingStore(void* backing_store, int32_t byte_length); void SerializeJSTypedArray(); void SerializeJSArrayBuffer(); void SerializeExternalString();