[torque] references and slices to off-heap data

This uses the old trick from TypedArrays: a Smi-like all zero
pattern plus an offset that actually contains a raw address to access
off-heap data.

Bug: v8:7793
Change-Id: Ia44448d4ff7e2dcaa02a2c5653f622fb93c3dd09
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2534817
Reviewed-by: Nico Hartmann <nicohartmann@chromium.org>
Commit-Queue: Tobias Tebbi <tebbi@chromium.org>
Cr-Commit-Position: refs/heads/master@{#71287}

src/builtins/base.tq

@@ -45,6 +45,8 @@ type PositiveSmi extends Smi;
 
 // The Smi value zero, which is often used as null for HeapObject types.
 type Zero extends PositiveSmi;
+// A tagged value represented by an all-zero bitpattern.
+type TaggedZeroPattern extends TaggedIndex;
 
 // A value with the size of Tagged which may contain arbitrary data.
 type Uninitialized extends Tagged;
@@ -151,6 +153,7 @@ type ObjectHashTable extends HashTable
 extern class NumberDictionary extends HashTable;
 
 type RawPtr generates 'TNode<RawPtrT>' constexpr 'Address';
+type RawPtr<To: type> extends RawPtr;
 type ExternalPointer
     generates 'TNode<ExternalPointerT>' constexpr 'ExternalPointer_t';
 extern class Code extends HeapObject;
@@ -440,6 +443,8 @@ const kReturnString: String = ReturnStringConstant();
 
 const kNaN: NaN = NanConstant();
 const kZero: Zero = %RawDownCast<Zero>(SmiConstant(0));
+const kZeroBitPattern: TaggedZeroPattern = %RawDownCast<TaggedZeroPattern>(
+    Convert<Tagged>(BitcastWordToTaggedSigned(Convert<intptr>(0))));
 
 const true: constexpr bool generates 'true';
 const false: constexpr bool generates 'false';

src/builtins/torque-internal.tq

@@ -13,7 +13,7 @@ struct Unsafe {}
 intrinsic %SizeOf<T: type>(): constexpr int31;
 
 struct Reference<T: type> {
-  const object: HeapObject;
+  const object: HeapObject|TaggedZeroPattern;
   const offset: intptr;
   unsafeMarker: Unsafe;
 }
@@ -21,10 +21,18 @@ type ConstReference<T: type> extends Reference<T>;
 type MutableReference<T: type> extends ConstReference<T>;
 
 namespace unsafe {
-macro NewReference<T: type>(object: HeapObject, offset: intptr):&T {
+macro NewReference<T: type>(
+    object: HeapObject|TaggedZeroPattern, offset: intptr):&T {
   return %RawDownCast<&T>(
       Reference<T>{object: object, offset: offset, unsafeMarker: Unsafe {}});
 }
+macro NewOffHeapReference<T: type>(ptr: RawPtr<T>):&T {
+  return %RawDownCast<&T>(Reference<T>{
+    object: kZeroBitPattern,
+    offset: Convert<intptr>(Convert<RawPtr>(ptr)) + kHeapObjectTag,
+    unsafeMarker: Unsafe {}
+  });
+}
 macro ReferenceCast<T: type, U: type>(ref:&U):&T {
   const ref = NewReference<T>(ref.object, ref.offset);
   UnsafeCast<T>(*ref);
@@ -83,13 +91,15 @@ struct Slice<T: type> {
     };
   }
 
-  const object: HeapObject;
+  const object: HeapObject|TaggedZeroPattern;
   const offset: intptr;
   const length: intptr;
   unsafeMarker: Unsafe;
 }
 
-macro UnsafeNewSlice<T: type>(
+namespace unsafe {
+
+macro NewSlice<T: type>(
     object: HeapObject, offset: intptr, length: intptr): Slice<T> {
   return Slice<T>{
     object: object,
@@ -99,6 +109,18 @@ macro UnsafeNewSlice<T: type>(
   };
 }
 
+macro NewOffHeapSlice<T: type>(
+    startPointer: RawPtr<T>, length: intptr): Slice<T> {
+  return Slice<T>{
+    object: kZeroBitPattern,
+    offset: Convert<intptr>(Convert<RawPtr>(startPointer)) + kHeapObjectTag,
+    length: length,
+    unsafeMarker: Unsafe {}
+  };
+}
+
+}  // namespace unsafe
+
 struct SliceIterator<T: type> {
   macro Empty(): bool {
     return this.start == this.end;
@@ -118,7 +140,7 @@ struct SliceIterator<T: type> {
     }
   }
 
-  object: HeapObject;
+  object: HeapObject|TaggedZeroPattern;
   start: intptr;
   end: intptr;
   unsafeMarker: Unsafe;
@@ -191,13 +213,15 @@ extern macro StoreDoubleHole(HeapObject, intptr);
 
 macro LoadFloat64OrHole(r:&float64_or_hole): float64_or_hole {
   return float64_or_hole{
-    is_hole: IsDoubleHole(r.object, r.offset - kHeapObjectTag),
+    is_hole: IsDoubleHole(
+        %RawDownCast<HeapObject>(r.object), r.offset - kHeapObjectTag),
     value: *unsafe::NewReference<float64>(r.object, r.offset)
   };
 }
 macro StoreFloat64OrHole(r:&float64_or_hole, value: float64_or_hole) {
   if (value.is_hole) {
-    StoreDoubleHole(r.object, r.offset - kHeapObjectTag);
+    StoreDoubleHole(
+        %RawDownCast<HeapObject>(r.object), r.offset - kHeapObjectTag);
   } else {
     *unsafe::NewReference<float64>(r.object, r.offset) = value.value;
   }

src/codegen/code-stub-assembler.h

@@ -1124,12 +1124,13 @@ class V8_EXPORT_PRIVATE CodeStubAssembler
 
   // Reference is the CSA-equivalent of a Torque reference value,
   // representing an inner pointer into a HeapObject.
+  // The object can be a HeapObject or an all-zero bitpattern.
   // TODO(gsps): Remove in favor of flattened {Load,Store}Reference interface
   struct Reference {
-    TNode<HeapObject> object;
+    TNode<Object> object;
     TNode<IntPtrT> offset;
 
-    std::tuple<TNode<HeapObject>, TNode<IntPtrT>> Flatten() const {
+    std::tuple<TNode<Object>, TNode<IntPtrT>> Flatten() const {
       return std::make_tuple(object, offset);
     }
   };
@@ -1140,6 +1141,7 @@ class V8_EXPORT_PRIVATE CodeStubAssembler
   TNode<T> LoadReference(Reference reference) {
     TNode<IntPtrT> offset =
         IntPtrSub(reference.offset, IntPtrConstant(kHeapObjectTag));
+    CSA_ASSERT(this, TaggedIsNotSmi(reference.object));
     return CAST(
         LoadFromObject(MachineTypeOf<T>::value, reference.object, offset));
   }
@@ -1168,6 +1170,7 @@ class V8_EXPORT_PRIVATE CodeStubAssembler
     }
     TNode<IntPtrT> offset =
         IntPtrSub(reference.offset, IntPtrConstant(kHeapObjectTag));
+    CSA_ASSERT(this, TaggedIsNotSmi(reference.object));
     StoreToObject(rep, reference.object, offset, value, write_barrier);
   }
   template <class T, typename std::enable_if<

src/compiler/code-assembler.cc

@@ -8,6 +8,7 @@
 
 #include "src/base/bits.h"
 #include "src/codegen/code-factory.h"
+#include "src/codegen/code-stub-assembler.h"
 #include "src/codegen/interface-descriptors.h"
 #include "src/codegen/machine-type.h"
 #include "src/codegen/macro-assembler.h"
@@ -696,7 +697,7 @@ template TNode<AtomicInt64> CodeAssembler::AtomicLoad64<AtomicInt64>(
 template TNode<AtomicUint64> CodeAssembler::AtomicLoad64<AtomicUint64>(
     TNode<RawPtrT> base, TNode<WordT> offset);
 
-Node* CodeAssembler::LoadFromObject(MachineType type, TNode<HeapObject> object,
+Node* CodeAssembler::LoadFromObject(MachineType type, TNode<Object> object,
                                     TNode<IntPtrT> offset) {
   return raw_assembler()->LoadFromObject(type, object, offset);
 }
@@ -727,8 +728,8 @@ void CodeAssembler::Store(Node* base, Node* value) {
 }
 
 void CodeAssembler::StoreToObject(MachineRepresentation rep,
-                                  TNode<HeapObject> object,
-                                  TNode<IntPtrT> offset, Node* value,
+                                  TNode<Object> object, TNode<IntPtrT> offset,
+                                  Node* value,
                                   StoreToObjectWriteBarrier write_barrier) {
   WriteBarrierKind write_barrier_kind;
   switch (write_barrier) {

src/compiler/code-assembler.h

@@ -762,7 +762,7 @@ class V8_EXPORT_PRIVATE CodeAssembler {
       Node* base, Node* offset,
       LoadSensitivity needs_poisoning = LoadSensitivity::kSafe);
 
-  Node* LoadFromObject(MachineType type, TNode<HeapObject> object,
+  Node* LoadFromObject(MachineType type, TNode<Object> object,
                        TNode<IntPtrT> offset);
 
   // Load a value from the root array.
@@ -791,7 +791,7 @@ class V8_EXPORT_PRIVATE CodeAssembler {
   TNode<HeapObject> OptimizedAllocate(TNode<IntPtrT> size,
                                       AllocationType allocation,
                                       AllowLargeObjects allow_large_objects);
-  void StoreToObject(MachineRepresentation rep, TNode<HeapObject> object,
+  void StoreToObject(MachineRepresentation rep, TNode<Object> object,
                      TNode<IntPtrT> offset, Node* value,
                      StoreToObjectWriteBarrier write_barrier);
   void OptimizedStoreField(MachineRepresentation rep, TNode<HeapObject> object,

src/torque/constants.h

@@ -31,6 +31,7 @@ static const char* const JS_FUNCTION_TYPE_STRING = "JSFunction";
 static const char* const MAP_TYPE_STRING = "Map";
 static const char* const OBJECT_TYPE_STRING = "Object";
 static const char* const HEAP_OBJECT_TYPE_STRING = "HeapObject";
+static const char* const TAGGED_ZERO_PATTERN_TYPE_STRING = "TaggedZeroPattern";
 static const char* const JSANY_TYPE_STRING = "JSAny";
 static const char* const JSOBJECT_TYPE_STRING = "JSObject";
 static const char* const SMI_TYPE_STRING = "Smi";

src/torque/instructions.cc

@@ -595,7 +595,9 @@ DefinitionLocation UnsafeCastInstruction::GetValueDefinition() const {
 void LoadReferenceInstruction::TypeInstruction(Stack<const Type*>* stack,
                                                ControlFlowGraph* cfg) const {
   ExpectType(TypeOracle::GetIntPtrType(), stack->Pop());
-  ExpectSubtype(stack->Pop(), TypeOracle::GetHeapObjectType());
+  ExpectSubtype(stack->Pop(), TypeOracle::GetUnionType(
+                                  TypeOracle::GetHeapObjectType(),
+                                  TypeOracle::GetTaggedZeroPatternType()));
   DCHECK_EQ(std::vector<const Type*>{type}, LowerType(type));
   stack->Push(type);
 }
@@ -615,7 +617,9 @@ void StoreReferenceInstruction::TypeInstruction(Stack<const Type*>* stack,
                                                 ControlFlowGraph* cfg) const {
   ExpectSubtype(stack->Pop(), type);
   ExpectType(TypeOracle::GetIntPtrType(), stack->Pop());
-  ExpectSubtype(stack->Pop(), TypeOracle::GetHeapObjectType());
+  ExpectSubtype(stack->Pop(), TypeOracle::GetUnionType(
+                                  TypeOracle::GetHeapObjectType(),
+                                  TypeOracle::GetTaggedZeroPatternType()));
 }
 
 void StoreReferenceInstruction::RecomputeDefinitionLocations(

src/torque/type-oracle.h

@@ -202,6 +202,10 @@ class TypeOracle : public ContextualClass<TypeOracle> {
     return Get().GetBuiltinType(HEAP_OBJECT_TYPE_STRING);
   }
 
+  static const Type* GetTaggedZeroPatternType() {
+    return Get().GetBuiltinType(TAGGED_ZERO_PATTERN_TYPE_STRING);
+  }
+
   static const Type* GetJSAnyType() {
     return Get().GetBuiltinType(JSANY_TYPE_STRING);
   }

test/cctest/torque/test-torque.cc

@@ -865,6 +865,22 @@ TEST(TestWord8Phi) {
   ft.Call();
 }
 
+TEST(TestOffHeapSlice) {
+  CcTest::InitializeVM();
+  Isolate* isolate(CcTest::i_isolate());
+  i::HandleScope scope(isolate);
+  CodeAssemblerTester asm_tester(isolate, 1);
+  TestTorqueAssembler m(asm_tester.state());
+  std::string data = "Hello World!";
+  {
+    m.TestOffHeapSlice(m.PointerConstant(const_cast<char*>(data.data())),
+                       m.IntPtrConstant(data.size()));
+    m.Return(m.UndefinedConstant());
+  }
+  FunctionTester ft(asm_tester.GenerateCode(), 0);
+  ft.Call();
+}
+
 }  // namespace compiler
 }  // namespace internal
 }  // namespace v8

test/torque/test-torque.tq

@@ -1340,4 +1340,17 @@ macro TestWord8Phi() {
     check(x == Convert<int8>(i));
   }
 }
+
+@export
+macro TestOffHeapSlice(ptr: RawPtr<char8>, length: intptr) {
+  const string = UnsafeCast<SeqOneByteString>(Convert<String>('Hello World!'));
+
+  check(*torque_internal::unsafe::NewOffHeapReference(ptr) == string.chars[0]);
+
+  let offHeapSlice = torque_internal::unsafe::NewOffHeapSlice(ptr, length);
+  let onHeapSlice = &string.chars;
+  for (let i: intptr = 0; i < onHeapSlice.length; ++i) {
+    check(*onHeapSlice.AtIndex(i) == *offHeapSlice.AtIndex(i));
+  }
+}
 }

test/unittests/torque/torque-unittest.cc

@@ -45,6 +45,8 @@ type Smi extends StrongTagged generates 'TNode<Smi>' constexpr 'Smi';
 type WeakHeapObject extends Tagged;
 type Weak<T : type extends HeapObject> extends WeakHeapObject;
 type Uninitialized extends Tagged;
+type TaggedIndex extends StrongTagged;
+type TaggedZeroPattern extends TaggedIndex;
 
 @abstract
 extern class HeapObject extends StrongTagged {