[turbofan] Optimize BigInt64 array store/load

This CL avoids unnecessary heap allocation for BigInt64 array
store/load by

- setting the output representation of a load to word64, and
- propagating word64 truncation to the source of a store.

This CL introduces a simplified operator SpeculativeToBigInt
which is applied to the source of a store to a BigInt64 array to
deopt on a non-bigint input.

Bug: v8:9407
Change-Id: I48ce13761bc4cf742d5b18cec4476dc9ad131414
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/4101011
Reviewed-by: Nico Hartmann <nicohartmann@chromium.org>
Commit-Queue: Qifan Pan <panq@google.com>
Cr-Commit-Position: refs/heads/main@{#84908}

src/compiler/access-builder.cc

@@ -1164,10 +1164,16 @@ ElementAccess AccessBuilder::ForTypedArrayElement(ExternalArrayType type,
                               MachineType::Float64(), kNoWriteBarrier};
       return access;
     }
-    case kExternalBigInt64Array:
-    case kExternalBigUint64Array:
-      // TODO(neis/jkummerow): Define appropriate types.
-      UNIMPLEMENTED();
+    case kExternalBigInt64Array: {
+      ElementAccess access = {taggedness, header_size, Type::SignedBigInt64(),
+                              MachineType::Int64(), kNoWriteBarrier};
+      return access;
+    }
+    case kExternalBigUint64Array: {
+      ElementAccess access = {taggedness, header_size, Type::UnsignedBigInt64(),
+                              MachineType::Uint64(), kNoWriteBarrier};
+      return access;
+    }
   }
   UNREACHABLE();
 }

src/compiler/heap-refs.cc

@@ -75,6 +75,8 @@ bool IsReadOnlyHeapObjectForCompiler(PtrComprCageBase cage_base,
          ReadOnlyHeap::Contains(object);
 }
 
+bool Is64() { return kSystemPointerSize == 8; }
+
 }  // namespace
 
 class ObjectData : public ZoneObject {
@@ -1084,8 +1086,8 @@ bool MapRef::CanInlineElementAccess() const {
   ElementsKind kind = elements_kind();
   if (IsFastElementsKind(kind)) return true;
   if (IsSharedArrayElementsKind(kind)) return true;
-  if (IsTypedArrayElementsKind(kind) && kind != BIGUINT64_ELEMENTS &&
-      kind != BIGINT64_ELEMENTS) {
+  if (IsTypedArrayElementsKind(kind) &&
+      (Is64() || (kind != BIGINT64_ELEMENTS && kind != BIGUINT64_ELEMENTS))) {
     return true;
   }
   if (v8_flags.turbo_rab_gsab && IsRabGsabTypedArrayElementsKind(kind) &&

src/compiler/js-native-context-specialization.cc

@@ -3306,12 +3306,21 @@ JSNativeContextSpecialization::BuildElementAccess(
       case AccessMode::kDefine:
         UNREACHABLE();
       case AccessMode::kStore: {
-        // Ensure that the {value} is actually a Number or an Oddball,
-        // and truncate it to a Number appropriately.
-        value = effect = graph()->NewNode(
-            simplified()->SpeculativeToNumber(
-                NumberOperationHint::kNumberOrOddball, FeedbackSource()),
-            value, effect, control);
+        if (external_array_type == kExternalBigInt64Array ||
+            external_array_type == kExternalBigUint64Array) {
+          value = effect = graph()->NewNode(
+              simplified()->SpeculativeToBigInt(BigIntOperationHint::kBigInt,
+                                                FeedbackSource()),
+              value, effect, control);
+        } else {
+          // Ensure that the {value} is actually a Number or an Oddball,
+          // and truncate it to a Number appropriately.
+          // TODO(panq): Eliminate the deopt loop introduced by the speculation.
+          value = effect = graph()->NewNode(
+              simplified()->SpeculativeToNumber(
+                  NumberOperationHint::kNumberOrOddball, FeedbackSource()),
+              value, effect, control);
+        }
 
         // Introduce the appropriate truncation for {value}. Currently we
         // only need to do this for ClamedUint8Array {receiver}s, as the

src/compiler/opcodes.h

@@ -543,7 +543,8 @@
 #define SIMPLIFIED_SPECULATIVE_BIGINT_UNOP_LIST(V) \
   V(SpeculativeBigIntAsIntN)                       \
   V(SpeculativeBigIntAsUintN)                      \
-  V(SpeculativeBigIntNegate)
+  V(SpeculativeBigIntNegate)                       \
+  V(SpeculativeToBigInt)
 
 #define SIMPLIFIED_WASM_OP_LIST(V) \
   V(AssertNotNull)                 \

src/compiler/operation-typer.cc

@@ -1200,6 +1200,10 @@ Type OperationTyper::SpeculativeBigIntNegate(Type type) {
   return Type::BigInt();
 }
 
+Type OperationTyper::SpeculativeToBigInt(Type type) {
+  return ToBigInt(Type::Intersect(type, Type::BigInt(), zone()));
+}
+
 Type OperationTyper::SpeculativeToNumber(Type type) {
   return ToNumber(Type::Intersect(type, Type::NumberOrOddball(), zone()));
 }

src/compiler/simplified-lowering.cc

@@ -96,7 +96,7 @@ MachineRepresentation MachineRepresentationFromArrayType(
       return MachineRepresentation::kFloat64;
     case kExternalBigInt64Array:
     case kExternalBigUint64Array:
-      UNIMPLEMENTED();
+      return MachineRepresentation::kWord64;
   }
   UNREACHABLE();
 }
@@ -156,7 +156,7 @@ UseInfo TruncatingUseInfoFromRepresentation(MachineRepresentation rep) {
     case MachineRepresentation::kWord32:
       return UseInfo::TruncatingWord32();
     case MachineRepresentation::kWord64:
-      return UseInfo::Word64();
+      return UseInfo::TruncatingWord64();
     case MachineRepresentation::kBit:
       return UseInfo::Bool();
     case MachineRepresentation::kCompressedPointer:
@@ -3946,6 +3946,34 @@ class RepresentationSelector {
         if (lower<T>()) DeferReplacement(node, node->InputAt(0));
         return;
       }
+      case IrOpcode::kSpeculativeToBigInt: {
+        if (truncation.IsUnused() && InputIs(node, Type::BigInt())) {
+          VisitUnused<T>(node);
+          return;
+        }
+        if (truncation.IsUsedAsWord64()) {
+          VisitUnop<T>(node,
+                       UseInfo::CheckedBigIntTruncatingWord64(FeedbackSource{}),
+                       MachineRepresentation::kWord64);
+        } else {
+          BigIntOperationParameters const& p =
+              BigIntOperationParametersOf(node->op());
+          switch (p.hint()) {
+            case BigIntOperationHint::kBigInt64: {
+              VisitUnop<T>(node, UseInfo::CheckedBigInt64AsWord64(p.feedback()),
+                           MachineRepresentation::kWord64);
+              break;
+            }
+            case BigIntOperationHint::kBigInt: {
+              VisitUnop<T>(node,
+                           UseInfo::CheckedBigIntAsTaggedPointer(p.feedback()),
+                           MachineRepresentation::kTaggedPointer);
+            }
+          }
+        }
+        if (lower<T>()) DeferReplacement(node, node->InputAt(0));
+        return;
+      }
       case IrOpcode::kObjectIsArrayBufferView: {
         // TODO(turbofan): Introduce a Type::ArrayBufferView?
         VisitUnop<T>(node, UseInfo::AnyTagged(), MachineRepresentation::kBit);

src/compiler/simplified-operator.cc

@@ -578,6 +578,26 @@ NumberOperationParameters const& NumberOperationParametersOf(
   return OpParameter<NumberOperationParameters>(op);
 }
 
+bool operator==(BigIntOperationParameters const& lhs,
+                BigIntOperationParameters const& rhs) {
+  return lhs.hint() == rhs.hint() && lhs.feedback() == rhs.feedback();
+}
+
+size_t hash_value(BigIntOperationParameters const& p) {
+  FeedbackSource::Hash feedback_hash;
+  return base::hash_combine(p.hint(), feedback_hash(p.feedback()));
+}
+
+std::ostream& operator<<(std::ostream& os, BigIntOperationParameters const& p) {
+  return os << p.hint() << ", " << p.feedback();
+}
+
+BigIntOperationParameters const& BigIntOperationParametersOf(
+    Operator const* op) {
+  DCHECK_EQ(IrOpcode::kSpeculativeToBigInt, op->opcode());
+  return OpParameter<BigIntOperationParameters>(op);
+}
+
 bool operator==(SpeculativeBigIntAsNParameters const& lhs,
                 SpeculativeBigIntAsNParameters const& rhs) {
   return lhs.bits() == rhs.bits() && lhs.feedback() == rhs.feedback();
@@ -1251,6 +1271,21 @@ struct SimplifiedOperatorGlobalCache final {
       kSpeculativeToNumberNumberOperator;
   SpeculativeToNumberOperator<NumberOperationHint::kNumberOrOddball>
       kSpeculativeToNumberNumberOrOddballOperator;
+
+  template <BigIntOperationHint kHint>
+  struct SpeculativeToBigIntOperator final
+      : public Operator1<BigIntOperationParameters> {
+    SpeculativeToBigIntOperator()
+        : Operator1<BigIntOperationParameters>(
+              IrOpcode::kSpeculativeToBigInt,
+              Operator::kFoldable | Operator::kNoThrow, "SpeculativeToBigInt",
+              1, 1, 1, 1, 1, 0,
+              BigIntOperationParameters(kHint, FeedbackSource())) {}
+  };
+  SpeculativeToBigIntOperator<BigIntOperationHint::kBigInt64>
+      kSpeculativeToBigIntBigInt64Operator;
+  SpeculativeToBigIntOperator<BigIntOperationHint::kBigInt>
+      kSpeculativeToBigIntBigIntOperator;
 };
 
 namespace {
@@ -1662,6 +1697,22 @@ const Operator* SimplifiedOperatorBuilder::SpeculativeBigIntNegate(
       1, 1, 1, 0, hint);
 }
 
+const Operator* SimplifiedOperatorBuilder::SpeculativeToBigInt(
+    BigIntOperationHint hint, const FeedbackSource& feedback) {
+  if (!feedback.IsValid()) {
+    switch (hint) {
+      case BigIntOperationHint::kBigInt64:
+        return &cache_.kSpeculativeToBigIntBigInt64Operator;
+      case BigIntOperationHint::kBigInt:
+        return &cache_.kSpeculativeToBigIntBigIntOperator;
+    }
+  }
+  return zone()->New<Operator1<BigIntOperationParameters>>(
+      IrOpcode::kSpeculativeToBigInt, Operator::kFoldable | Operator::kNoThrow,
+      "SpeculativeToBigInt", 1, 1, 1, 1, 1, 0,
+      BigIntOperationParameters(hint, feedback));
+}
+
 const Operator* SimplifiedOperatorBuilder::CheckClosure(
     const Handle<FeedbackCell>& feedback_cell) {
   return zone()->New<Operator1<Handle<FeedbackCell>>>(  // --

src/compiler/simplified-operator.h

@@ -563,6 +563,28 @@ bool operator==(NumberOperationParameters const&,
 const NumberOperationParameters& NumberOperationParametersOf(const Operator* op)
     V8_WARN_UNUSED_RESULT;
 
+class BigIntOperationParameters {
+ public:
+  BigIntOperationParameters(BigIntOperationHint hint,
+                            const FeedbackSource& feedback)
+      : hint_(hint), feedback_(feedback) {}
+
+  BigIntOperationHint hint() const { return hint_; }
+  const FeedbackSource& feedback() const { return feedback_; }
+
+ private:
+  BigIntOperationHint hint_;
+  FeedbackSource feedback_;
+};
+
+size_t hash_value(BigIntOperationParameters const&);
+V8_EXPORT_PRIVATE std::ostream& operator<<(std::ostream&,
+                                           const BigIntOperationParameters&);
+bool operator==(BigIntOperationParameters const&,
+                BigIntOperationParameters const&);
+const BigIntOperationParameters& BigIntOperationParametersOf(const Operator* op)
+    V8_WARN_UNUSED_RESULT;
+
 class SpeculativeBigIntAsNParameters {
  public:
   SpeculativeBigIntAsNParameters(int bits, const FeedbackSource& feedback)
@@ -864,6 +886,9 @@ class V8_EXPORT_PRIVATE SimplifiedOperatorBuilder final
   const Operator* SpeculativeToNumber(NumberOperationHint hint,
                                       const FeedbackSource& feedback);
 
+  const Operator* SpeculativeToBigInt(BigIntOperationHint hint,
+                                      const FeedbackSource& feedback);
+
   const Operator* StringToNumber();
   const Operator* PlainPrimitiveToNumber();
   const Operator* PlainPrimitiveToWord32();

src/compiler/type-cache.h

@@ -44,8 +44,8 @@ class V8_EXPORT_PRIVATE TypeCache final {
       std::numeric_limits<uint64_t>::min(), kMaxDoubleRepresentableUint64);
   Type const kFloat32 = Type::Number();
   Type const kFloat64 = Type::Number();
-  Type const kBigInt64 = Type::BigInt();
-  Type const kBigUint64 = Type::BigInt();
+  Type const kBigInt64 = Type::SignedBigInt64();
+  Type const kBigUint64 = Type::UnsignedBigInt64();
 
   Type const kHoleySmi = Type::Union(Type::SignedSmall(), Type::Hole(), zone());
 

src/compiler/use-info.h

@@ -189,6 +189,9 @@ class UseInfo {
   static UseInfo TruncatingWord32() {
     return UseInfo(MachineRepresentation::kWord32, Truncation::Word32());
   }
+  static UseInfo TruncatingWord64() {
+    return UseInfo(MachineRepresentation::kWord64, Truncation::Word64());
+  }
   static UseInfo CheckedBigIntTruncatingWord64(const FeedbackSource& feedback) {
     // Note that Trunction::Word64() can safely use kIdentifyZero, because
     // TypeCheckKind::kBigInt will make sure we deopt for anything other than

src/compiler/verifier.cc

@@ -1026,6 +1026,10 @@ void Verifier::Visitor::Check(Node* node, const AllNodes& all) {
       CheckValueInputIs(node, 0, Type::BigInt());
       CheckTypeIs(node, Type::BigInt());
       break;
+    case IrOpcode::kSpeculativeToBigInt:
+      CheckValueInputIs(node, 0, Type::Any());
+      CheckTypeIs(node, Type::BigInt());
+      break;
     case IrOpcode::kNumberAdd:
     case IrOpcode::kNumberSubtract:
     case IrOpcode::kNumberMultiply:

test/mjsunit/compiler/bigint64-array.js

@@ -0,0 +1,71 @@
+// Copyright 2022 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Flags: --allow-natives-syntax --turbofan --no-always-turbofan
+
+const bi = 18446744073709551615n; // 2n ** 64n - 1n
+
+function storeAndLoad(x) {
+  let buffer = new ArrayBuffer(16);
+  let biArray = new BigInt64Array(buffer);
+  biArray[0] = bi;
+  biArray[1] = x;
+  return biArray[0] + biArray[1];
+}
+
+%PrepareFunctionForOptimization(storeAndLoad);
+assertEquals(-1n, storeAndLoad(0n));
+assertEquals(41n, storeAndLoad(2n ** 64n + 42n));
+assertEquals(0n, storeAndLoad(-bi));
+assertEquals(-2n, storeAndLoad(bi));
+%OptimizeFunctionOnNextCall(storeAndLoad);
+assertEquals(-1n, storeAndLoad(0n));
+assertEquals(41n, storeAndLoad(2n ** 64n + 42n));
+assertEquals(0n, storeAndLoad(-bi));
+assertEquals(-2n, storeAndLoad(bi));
+assertOptimized(storeAndLoad);
+
+assertEquals(-1n, storeAndLoad(false));
+if (%Is64Bit()) {
+  assertUnoptimized(storeAndLoad);
+}
+
+%PrepareFunctionForOptimization(storeAndLoad);
+assertEquals(-1n, storeAndLoad(0n));
+%OptimizeFunctionOnNextCall(storeAndLoad);
+assertEquals(0n, storeAndLoad(true));
+// TODO(panq): Uncomment the assertion once the deopt loop is eliminated.
+// assertOptimized(storeAndLoad);
+
+function storeAndLoadUnsigned(x) {
+  let buffer = new ArrayBuffer(16);
+  let biArray = new BigUint64Array(buffer);
+  biArray[0] = bi;
+  biArray[1] = x;
+  return biArray[0] + biArray[1];
+}
+
+%PrepareFunctionForOptimization(storeAndLoadUnsigned);
+assertEquals(bi, storeAndLoadUnsigned(0n));
+assertEquals(bi + 42n, storeAndLoadUnsigned(2n ** 64n + 42n));
+assertEquals(bi + 1n, storeAndLoadUnsigned(-bi));
+assertEquals(bi * 2n, storeAndLoadUnsigned(bi));
+%OptimizeFunctionOnNextCall(storeAndLoadUnsigned);
+assertEquals(bi, storeAndLoadUnsigned(0n));
+assertEquals(bi + 42n, storeAndLoadUnsigned(2n ** 64n + 42n));
+assertEquals(bi + 1n, storeAndLoadUnsigned(-bi));
+assertEquals(bi * 2n, storeAndLoadUnsigned(bi));
+assertOptimized(storeAndLoadUnsigned);
+
+assertEquals(bi, storeAndLoadUnsigned(false));
+if (%Is64Bit()) {
+  assertUnoptimized(storeAndLoadUnsigned);
+}
+
+%PrepareFunctionForOptimization(storeAndLoadUnsigned);
+assertEquals(bi, storeAndLoadUnsigned(0n));
+%OptimizeFunctionOnNextCall(storeAndLoadUnsigned);
+assertEquals(bi + 1n, storeAndLoadUnsigned(true));
+// TODO(panq): Uncomment the assertion once the deopt loop is eliminated.
+// assertOptimized(storeAndLoadUnsigned);