[turbofan] Serialize JSObject boilerplates for create lowering.

Adds recursive serialization of boilerplate JSObjects. It also hooks up
serialization of FixedArrays (shallow copy of all elements).

Bug: v8:7790
Change-Id: I458133961918617ab7bdae8d14712a4e5a7d9cb5
Reviewed-on: https://chromium-review.googlesource.com/1188903
Reviewed-by: Georg Neis <neis@chromium.org>
Commit-Queue: Jaroslav Sevcik <jarin@chromium.org>
Cr-Commit-Position: refs/heads/master@{#55475}

src/compiler/js-create-lowering.cc

@@ -1648,7 +1648,7 @@ Node* JSCreateLowering::AllocateFastLiteral(Node* effect, Node* control,
         MaybeHandle<Map>(), Type::Any(),    MachineType::AnyTagged(),
         kFullWriteBarrier};
     Node* value;
-    if (boilerplate.IsUnboxedDoubleField(index)) {
+    if (boilerplate.map().IsUnboxedDoubleField(index)) {
       access.machine_type = MachineType::Float64();
       access.type = Type::Number();
       value = jsgraph()->Constant(boilerplate.RawFastDoublePropertyAt(index));
@@ -1744,16 +1744,12 @@ Node* JSCreateLowering::AllocateFastLiteralElements(Node* effect, Node* control,
   } else {
     FixedArrayRef elements = boilerplate_elements.AsFixedArray();
     for (int i = 0; i < elements_length; ++i) {
-      if (elements.is_the_hole(i)) {
-        elements_values[i] = jsgraph()->TheHoleConstant();
+      ObjectRef element_value = elements.get(i);
+      if (element_value.IsJSObject()) {
+        elements_values[i] = effect = AllocateFastLiteral(
+            effect, control, element_value.AsJSObject(), pretenure);
       } else {
-        ObjectRef element_value = elements.get(i);
-        if (element_value.IsJSObject()) {
-          elements_values[i] = effect = AllocateFastLiteral(
-              effect, control, element_value.AsJSObject(), pretenure);
-        } else {
-          elements_values[i] = jsgraph()->Constant(element_value);
-        }
+        elements_values[i] = jsgraph()->Constant(element_value);
       }
     }
   }

src/compiler/js-heap-broker.cc

@@ -4,6 +4,7 @@
 
 #include "src/compiler/js-heap-broker.h"
 
+#include "src/boxed-float.h"
 #include "src/compiler/graph-reducer.h"
 #include "src/objects-inl.h"
 #include "src/objects/js-array-inl.h"
@@ -72,11 +73,54 @@ class PropertyCellData : public HeapObjectData {
       : HeapObjectData(broker_, object_, type_) {}
 };
 
+class JSObjectField {
+ public:
+  bool IsDouble() const { return object_ == nullptr; }
+  double AsDouble() const {
+    CHECK(IsDouble());
+    return number_;
+  }
+
+  bool IsObject() const { return object_ != nullptr; }
+  ObjectData* AsObject() const {
+    CHECK(IsObject());
+    return object_;
+  }
+
+  explicit JSObjectField(double value) : number_(value) {}
+  explicit JSObjectField(ObjectData* value) : object_(value) {}
+
+ private:
+  ObjectData* object_ = nullptr;
+  double number_ = 0;
+};
+
 class JSObjectData : public HeapObjectData {
  public:
   JSObjectData(JSHeapBroker* broker_, Handle<JSObject> object_,
-               HeapObjectType type_)
-      : HeapObjectData(broker_, object_, type_) {}
+               HeapObjectType type_);
+
+  // Recursively serializes all reachable JSObjects.
+  void SerializeAsBoilerplate();
+  // Shallow serialization of {elements}.
+  void SerializeElements();
+
+  const JSObjectField& GetInobjectField(int property_index) const;
+  FixedArrayBaseData* elements() const;
+
+  // This method is only used to assert our invariants.
+  bool cow_or_empty_elements_tenured() const;
+
+ private:
+  void SerializeRecursive(int max_depths);
+
+  FixedArrayBaseData* elements_ = nullptr;
+  bool cow_or_empty_elements_tenured_ = false;
+  // The {is_serialized_as_boilerplate} flag is set when all recursively
+  // reachable JSObjects are serialized.
+  bool is_serialized_as_boilerplate_ = false;
+
+  ZoneVector<JSObjectField> inobject_fields_;
 };
 
 class JSFunctionData : public JSObjectData {
@@ -272,22 +316,37 @@ class AllocationSiteData : public HeapObjectData {
                      HeapObjectType type_)
       : HeapObjectData(broker, object_, type_),
         PointsToLiteral(object_->PointsToLiteral()),
-        GetPretenureMode(object_->GetPretenureMode()),
-        nested_site(GET_OR_CREATE(nested_site)) {
+        GetPretenureMode(object_->GetPretenureMode()) {
     if (PointsToLiteral) {
-      if (IsInlinableFastLiteral(
-              handle(object_->boilerplate(), broker->isolate()))) {
-        boilerplate = GET_OR_CREATE(boilerplate)->AsJSObject();
-      }
+      IsFastLiteral = IsInlinableFastLiteral(
+          handle(object_->boilerplate(), broker->isolate()));
     } else {
       GetElementsKind = object_->GetElementsKind();
       CanInlineCall = object_->CanInlineCall();
     }
   }
 
+  void SerializeBoilerplate() {
+    if (boilerplate != nullptr || !IsFastLiteral) return;
+
+    Handle<AllocationSite> site = Handle<AllocationSite>::cast(object);
+    Handle<JSObject> boilerplate_object(site->boilerplate(), broker->isolate());
+    boilerplate = broker->GetOrCreateData(boilerplate_object)->AsJSObject();
+    boilerplate->SerializeAsBoilerplate();
+
+    DCHECK_NULL(nested_site);
+    Handle<Object> nested_site_object =
+        handle(site->nested_site(), broker->isolate());
+    nested_site = broker->GetOrCreateData(nested_site_object);
+    if (nested_site->IsAllocationSite()) {
+      nested_site->AsAllocationSite()->SerializeBoilerplate();
+    }
+  }
+
   bool const PointsToLiteral;
   PretenureFlag const GetPretenureMode;
-  ObjectData* const nested_site;
+  ObjectData* nested_site = nullptr;
+  bool IsFastLiteral = false;
   JSObjectData* boilerplate = nullptr;
 
   // These are only valid if PointsToLiteral is false.
@@ -373,48 +432,131 @@ class FeedbackVectorData : public HeapObjectData {
   FeedbackVectorData(JSHeapBroker* broker_, Handle<FeedbackVector> object_,
                      HeapObjectType type_);
 
+  void SerializeSlots();
+
  private:
+  bool is_serialized_ = false;
   ZoneVector<ObjectData*> feedback_;
 };
 
 FeedbackVectorData::FeedbackVectorData(JSHeapBroker* broker_,
                                        Handle<FeedbackVector> object_,
                                        HeapObjectType type_)
-    : HeapObjectData(broker_, object_, type_), feedback_(broker_->zone()) {
-  feedback_.reserve(object_->length());
-  for (int i = 0; i < object_->length(); ++i) {
-    MaybeObject* value = object_->get(i);
-    feedback_.push_back(value->IsObject()
-                            ? broker->GetOrCreateData(
-                                  handle(value->ToObject(), broker->isolate()))
-                            : nullptr);
+    : HeapObjectData(broker_, object_, type_), feedback_(broker_->zone()) {}
+
+void FeedbackVectorData::SerializeSlots() {
+  if (is_serialized_) return;
+  is_serialized_ = true;
+
+  DCHECK(feedback_.empty());
+
+  Handle<FeedbackVector> feedback_object = Handle<FeedbackVector>::cast(object);
+  feedback_.reserve(feedback_object->length());
+  for (int i = 0; i < feedback_object->length(); ++i) {
+    MaybeObject* value = feedback_object->get(i);
+    ObjectData* slot_value = value->IsObject()
+                                 ? broker->GetOrCreateData(handle(
+                                       value->ToObject(), broker->isolate()))
+                                 : nullptr;
+    feedback_.push_back(slot_value);
+    if (slot_value == nullptr) continue;
+
+    if (slot_value->IsAllocationSite()) {
+      slot_value->AsAllocationSite()->SerializeBoilerplate();
+    } else if (slot_value->IsJSRegExp()) {
+      slot_value->AsJSRegExp()->SerializeElements();
+    }
   }
-  DCHECK_EQ(object_->length(), feedback_.size());
+  DCHECK_EQ(feedback_object->length(), feedback_.size());
 }
 
 class FixedArrayBaseData : public HeapObjectData {
  public:
-  int const length;
-
   FixedArrayBaseData(JSHeapBroker* broker_, Handle<FixedArrayBase> object_,
                      HeapObjectType type_)
-      : HeapObjectData(broker_, object_, type_), length(object_->length()) {}
+      : HeapObjectData(broker_, object_, type_), length_(object_->length()) {}
+
+  int length() const { return length_; }
+
+ private:
+  int const length_;
 };
 
+JSObjectData::JSObjectData(JSHeapBroker* broker_, Handle<JSObject> object_,
+                           HeapObjectType type_)
+    : HeapObjectData(broker_, object_, type_),
+      inobject_fields_(broker_->zone()) {}
+
 class FixedArrayData : public FixedArrayBaseData {
  public:
   FixedArrayData(JSHeapBroker* broker_, Handle<FixedArray> object_,
-                 HeapObjectType type_)
-      : FixedArrayBaseData(broker_, object_, type_) {}
+                 HeapObjectType type_);
+
+  // Creates all elements of the fixed array.
+  void SerializeContents();
+
+  ObjectData* Get(int i) const;
+
+ private:
+  ZoneVector<ObjectData*> contents_;
 };
 
+void FixedArrayData::SerializeContents() {
+  Handle<FixedArray> fixed_array = Handle<FixedArray>::cast(this->object);
+  CHECK_EQ(fixed_array->length(), length());
+
+  // If we are serialized, there is nothing to do.
+  if (static_cast<size_t>(length()) == contents_.size()) return;
+  CHECK(contents_.empty());
+
+  contents_.reserve(static_cast<size_t>(length()));
+
+  for (int i = 0; i < length(); i++) {
+    Handle<Object> value = handle(fixed_array->get(i), broker->isolate());
+    contents_.push_back(broker->GetOrCreateData(value));
+  }
+}
+
+FixedArrayData::FixedArrayData(JSHeapBroker* broker_,
+                               Handle<FixedArray> object_, HeapObjectType type_)
+    : FixedArrayBaseData(broker_, object_, type_), contents_(broker->zone()) {}
+
 class FixedDoubleArrayData : public FixedArrayBaseData {
  public:
   FixedDoubleArrayData(JSHeapBroker* broker_, Handle<FixedDoubleArray> object_,
-                       HeapObjectType type_)
-      : FixedArrayBaseData(broker_, object_, type_) {}
+                       HeapObjectType type_);
+
+  // Serializes all elements of the fixed array.
+  void SerializeContents();
+
+  Float64 Get(int i) const;
+
+ private:
+  ZoneVector<Float64> contents_;
 };
 
+FixedDoubleArrayData::FixedDoubleArrayData(JSHeapBroker* broker_,
+                                           Handle<FixedDoubleArray> object_,
+                                           HeapObjectType type_)
+    : FixedArrayBaseData(broker_, object_, type_), contents_(broker->zone()) {}
+
+void FixedDoubleArrayData::SerializeContents() {
+  Handle<FixedDoubleArray> fixed_double_array =
+      Handle<FixedDoubleArray>::cast(this->object);
+  CHECK_EQ(fixed_double_array->length(), length());
+
+  // If we are serialized, there is nothing to do.
+  if (static_cast<size_t>(length()) == contents_.size()) return;
+  CHECK(contents_.empty());
+
+  contents_.reserve(static_cast<size_t>(length()));
+
+  for (int i = 0; i < length(); i++) {
+    contents_.push_back(
+        Float64::FromBits(fixed_double_array->get_representation(i)));
+  }
+}
+
 class BytecodeArrayData : public FixedArrayBaseData {
  public:
   int const register_count;
@@ -511,6 +653,126 @@ class CodeData : public HeapObjectData {
 HEAP_BROKER_OBJECT_LIST(DEFINE_IS_AND_AS)
 #undef DEFINE_IS_AND_AS
 
+const JSObjectField& JSObjectData::GetInobjectField(int property_index) const {
+  CHECK_LT(static_cast<size_t>(property_index), inobject_fields_.size());
+  return inobject_fields_[property_index];
+}
+
+bool JSObjectData::cow_or_empty_elements_tenured() const {
+  return cow_or_empty_elements_tenured_;
+}
+
+FixedArrayBaseData* JSObjectData::elements() const { return elements_; }
+
+void JSObjectData::SerializeAsBoilerplate() {
+  SerializeRecursive(kMaxFastLiteralDepth);
+}
+
+void JSObjectData::SerializeElements() {
+  if (elements_) return;
+
+  Handle<JSObject> boilerplate = Handle<JSObject>::cast(this->object);
+  Handle<FixedArrayBase> elements_object(boilerplate->elements(),
+                                         broker->isolate());
+  elements_ = broker->GetOrCreateData(elements_object)->AsFixedArrayBase();
+}
+
+void JSObjectData::SerializeRecursive(int depth) {
+  Handle<JSObject> boilerplate = Handle<JSObject>::cast(this->object);
+  Isolate* const isolate = boilerplate->GetIsolate();
+
+  // We only serialize boilerplates that pass the IsInlinableFastLiteral
+  // check, so we only do a sanity check on the depth here.
+  CHECK_GT(depth, 0);
+
+  CHECK(!boilerplate->map()->is_deprecated());
+
+  // There is nothing to do if we already started serializing.
+  if (is_serialized_as_boilerplate_) return;
+  // Mark the serialization bit to prevent recursion problems.
+  is_serialized_as_boilerplate_ = true;
+
+  // Serialize the elements.
+  Handle<FixedArrayBase> elements_object(boilerplate->elements(), isolate);
+
+  // Boilerplates need special serialization - we need to make sure COW arrays
+  // are tenured. Boilerplate objects should only be reachable from their
+  // allocation site, so it is safe to assume that the elements have not been
+  // serialized yet.
+
+  bool const empty_or_cow =
+      elements_object->length() == 0 ||
+      elements_object->map() == ReadOnlyRoots(isolate).fixed_cow_array_map();
+  if (empty_or_cow) {
+    // We need to make sure copy-on-write elements are tenured.
+    if (Heap::InNewSpace(*elements_object)) {
+      elements_object =
+          broker->isolate()->factory()->CopyAndTenureFixedCOWArray(
+              Handle<FixedArray>::cast(elements_object));
+      boilerplate->set_elements(*elements_object);
+    }
+    cow_or_empty_elements_tenured_ = true;
+  }
+
+  CHECK_NULL(elements_);
+  elements_ = broker->GetOrCreateData(elements_object)->AsFixedArrayBase();
+
+  if (empty_or_cow) {
+    // No need to do anything here. Empty or copy-on-write elements
+    // do not need to be serialized because we only need to store the elements
+    // reference to the allocated object.
+  } else if (boilerplate->HasSmiOrObjectElements()) {
+    elements_->AsFixedArray()->SerializeContents();
+    Handle<FixedArray> fast_elements =
+        Handle<FixedArray>::cast(elements_object);
+    int length = elements_object->length();
+    for (int i = 0; i < length; i++) {
+      Handle<Object> value(fast_elements->get(i), isolate);
+      if (value->IsJSObject()) {
+        ObjectData* value_data = broker->GetOrCreateData(value);
+        value_data->AsJSObject()->SerializeRecursive(depth - 1);
+      }
+    }
+  } else {
+    CHECK(boilerplate->HasDoubleElements());
+    CHECK_LE(elements_object->Size(), kMaxRegularHeapObjectSize);
+    elements_->AsFixedDoubleArray()->SerializeContents();
+  }
+
+  // TODO(turbofan): Do we want to support out-of-object properties?
+  CHECK(boilerplate->HasFastProperties() &&
+        boilerplate->property_array()->length() == 0);
+  CHECK_EQ(inobject_fields_.size(), 0u);
+
+  // Check the in-object properties.
+  Handle<DescriptorArray> descriptors(
+      boilerplate->map()->instance_descriptors(), isolate);
+  int const limit = boilerplate->map()->NumberOfOwnDescriptors();
+  for (int i = 0; i < limit; i++) {
+    PropertyDetails details = descriptors->GetDetails(i);
+    if (details.location() != kField) continue;
+    DCHECK_EQ(kData, details.kind());
+
+    FieldIndex field_index = FieldIndex::ForDescriptor(boilerplate->map(), i);
+    // Make sure {field_index} agrees with {inobject_properties} on the index of
+    // this field.
+    DCHECK_EQ(field_index.property_index(),
+              static_cast<int>(inobject_fields_.size()));
+    if (boilerplate->IsUnboxedDoubleField(field_index)) {
+      double value = boilerplate->RawFastDoublePropertyAt(field_index);
+      inobject_fields_.push_back(JSObjectField{value});
+    } else {
+      Handle<Object> value(boilerplate->RawFastPropertyAt(field_index),
+                           isolate);
+      ObjectData* value_data = broker->GetOrCreateData(value);
+      if (value->IsJSObject()) {
+        value_data->AsJSObject()->SerializeRecursive(depth - 1);
+      }
+      inobject_fields_.push_back(JSObjectField{value_data});
+    }
+  }
+}
+
 ObjectData* ObjectData::Serialize(JSHeapBroker* broker, Handle<Object> object) {
   CHECK(broker->SerializingAllowed());
   return object->IsSmi() ? new (broker->zone()) ObjectData(broker, object, true)
@@ -799,25 +1061,34 @@ ObjectRef FeedbackVectorRef::get(FeedbackSlot slot) const {
   return ObjectRef(data()->AsFeedbackVector()->feedback().at(i));
 }
 
-bool JSObjectRef::IsUnboxedDoubleField(FieldIndex index) const {
-  AllowHandleDereference handle_dereference;
-  return object<JSObject>()->IsUnboxedDoubleField(index);
-}
-
 double JSObjectRef::RawFastDoublePropertyAt(FieldIndex index) const {
-  AllowHandleDereference handle_dereference;
-  return object<JSObject>()->RawFastDoublePropertyAt(index);
+  if (broker()->mode() == JSHeapBroker::kDisabled) {
+    AllowHandleDereference handle_dereference;
+    return object<JSObject>()->RawFastDoublePropertyAt(index);
+  } else {
+    JSObjectData* object_data = data()->AsJSObject();
+    CHECK(map().IsUnboxedDoubleField(index));
+    CHECK(index.is_inobject());
+    return object_data->GetInobjectField(index.property_index()).AsDouble();
+  }
 }
 
 ObjectRef JSObjectRef::RawFastPropertyAt(FieldIndex index) const {
-  AllowHandleAllocation handle_allocation;
-  AllowHandleDereference handle_dereference;
-  return ObjectRef(broker(),
-                   handle(object<JSObject>()->RawFastPropertyAt(index),
-                          broker()->isolate()));
+  if (broker()->mode() == JSHeapBroker::kDisabled) {
+    AllowHandleAllocation handle_allocation;
+    AllowHandleDereference handle_dereference;
+    return ObjectRef(broker(),
+                     handle(object<JSObject>()->RawFastPropertyAt(index),
+                            broker()->isolate()));
+  } else {
+    JSObjectData* object_data = data()->AsJSObject();
+    CHECK(!map().IsUnboxedDoubleField(index));
+    CHECK(index.is_inobject());
+    return ObjectRef(
+        object_data->GetInobjectField(index.property_index()).AsObject());
+  }
 }
 
-
 bool AllocationSiteRef::IsFastLiteral() const {
   if (broker()->mode() == JSHeapBroker::kDisabled) {
     AllowHeapAllocation
@@ -827,26 +1098,29 @@ bool AllocationSiteRef::IsFastLiteral() const {
     return IsInlinableFastLiteral(
         handle(object<AllocationSite>()->boilerplate(), broker()->isolate()));
   } else {
-    return data()->AsAllocationSite()->boilerplate != nullptr;
+    return data()->AsAllocationSite()->IsFastLiteral;
   }
 }
 
 void JSObjectRef::EnsureElementsTenured() {
-  // TODO(jarin) Eventually, we will pretenure the boilerplates before
-  // the compilation job starts.
-  AllowHandleAllocation allow_handle_allocation;
-  AllowHandleDereference allow_handle_dereference;
-  AllowHeapAllocation allow_heap_allocation;
+  if (broker()->mode() == JSHeapBroker::kDisabled) {
+    AllowHandleAllocation allow_handle_allocation;
+    AllowHandleDereference allow_handle_dereference;
+    AllowHeapAllocation allow_heap_allocation;
 
-  Handle<FixedArrayBase> object_elements = elements().object<FixedArrayBase>();
-  if (Heap::InNewSpace(*object_elements)) {
-    // If we would like to pretenure a fixed cow array, we must ensure that
-    // the array is already in old space, otherwise we'll create too many
-    // old-to-new-space pointers (overflowing the store buffer).
-    object_elements =
-        broker()->isolate()->factory()->CopyAndTenureFixedCOWArray(
-            Handle<FixedArray>::cast(object_elements));
-    object<JSObject>()->set_elements(*object_elements);
+    Handle<FixedArrayBase> object_elements =
+        elements().object<FixedArrayBase>();
+    if (Heap::InNewSpace(*object_elements)) {
+      // If we would like to pretenure a fixed cow array, we must ensure that
+      // the array is already in old space, otherwise we'll create too many
+      // old-to-new-space pointers (overflowing the store buffer).
+      object_elements =
+          broker()->isolate()->factory()->CopyAndTenureFixedCOWArray(
+              Handle<FixedArray>::cast(object_elements));
+      object<JSObject>()->set_elements(*object_elements);
+    }
+  } else {
+    CHECK(data()->AsJSObject()->cow_or_empty_elements_tenured());
   }
 }
 
@@ -897,6 +1171,11 @@ ObjectRef MapRef::GetFieldType(int descriptor) const {
   return ObjectRef(broker(), field_type);
 }
 
+bool MapRef::IsUnboxedDoubleField(FieldIndex index) const {
+  AllowHandleDereference allow_handle_dereference;
+  return object<Map>()->IsUnboxedDoubleField(index);
+}
+
 uint16_t StringRef::GetFirstChar() {
   if (broker()->mode() == JSHeapBroker::kDisabled) {
     AllowHandleDereference allow_handle_dereference;
@@ -920,26 +1199,34 @@ base::Optional<double> StringRef::ToNumber() {
   }
 }
 
-bool FixedArrayRef::is_the_hole(int i) const {
-  AllowHandleDereference allow_handle_dereference;
-  return object<FixedArray>()->is_the_hole(broker()->isolate(), i);
-}
-
 ObjectRef FixedArrayRef::get(int i) const {
-  AllowHandleAllocation handle_allocation;
-  AllowHandleDereference allow_handle_dereference;
-  return ObjectRef(broker(),
-                   handle(object<FixedArray>()->get(i), broker()->isolate()));
+  if (broker()->mode() == JSHeapBroker::kDisabled) {
+    AllowHandleAllocation handle_allocation;
+    AllowHandleDereference allow_handle_dereference;
+    return ObjectRef(broker(),
+                     handle(object<FixedArray>()->get(i), broker()->isolate()));
+  } else {
+    return ObjectRef(data()->AsFixedArray()->Get(i));
+  }
 }
 
 bool FixedDoubleArrayRef::is_the_hole(int i) const {
-  AllowHandleDereference allow_handle_dereference;
-  return object<FixedDoubleArray>()->is_the_hole(i);
+  if (broker()->mode() == JSHeapBroker::kDisabled) {
+    AllowHandleDereference allow_handle_dereference;
+    return object<FixedDoubleArray>()->is_the_hole(i);
+  } else {
+    return data()->AsFixedDoubleArray()->Get(i).is_hole_nan();
+  }
 }
 
 double FixedDoubleArrayRef::get_scalar(int i) const {
-  AllowHandleDereference allow_handle_dereference;
-  return object<FixedDoubleArray>()->get_scalar(i);
+  if (broker()->mode() == JSHeapBroker::kDisabled) {
+    AllowHandleDereference allow_handle_dereference;
+    return object<FixedDoubleArray>()->get_scalar(i);
+  } else {
+    CHECK(!data()->AsFixedDoubleArray()->Get(i).is_hole_nan());
+    return data()->AsFixedDoubleArray()->Get(i).get_scalar();
+  }
 }
 
 #define IF_BROKER_DISABLED_ACCESS_HANDLE_C(holder, name) \
@@ -1007,8 +1294,6 @@ BIMODAL_ACCESSOR_C(AllocationSite, PretenureFlag, GetPretenureMode)
 
 BIMODAL_ACCESSOR_C(BytecodeArray, int, register_count)
 
-BIMODAL_ACCESSOR_C(FixedArrayBase, int, length)
-
 BIMODAL_ACCESSOR(HeapObject, Map, map)
 HANDLE_ACCESSOR_C(HeapObject, bool, IsExternalString)
 HANDLE_ACCESSOR_C(HeapObject, bool, IsSeqString)
@@ -1023,8 +1308,6 @@ HANDLE_ACCESSOR_C(JSFunction, bool, IsConstructor)
 HANDLE_ACCESSOR(JSFunction, JSGlobalProxy, global_proxy)
 HANDLE_ACCESSOR(JSFunction, SharedFunctionInfo, shared)
 
-HANDLE_ACCESSOR(JSObject, FixedArrayBase, elements)
-
 HANDLE_ACCESSOR(JSRegExp, Object, data)
 HANDLE_ACCESSOR(JSRegExp, Object, flags)
 HANDLE_ACCESSOR(JSRegExp, Object, last_index)
@@ -1186,6 +1469,37 @@ ElementsKind JSObjectRef::GetElementsKind() const {
   return map().elements_kind();
 }
 
+FixedArrayBaseRef JSObjectRef::elements() const {
+  if (broker()->mode() == JSHeapBroker::kDisabled) {
+    AllowHandleAllocation handle_allocation;
+    AllowHandleDereference allow_handle_dereference;
+    return FixedArrayBaseRef(
+        broker(), handle(object<JSObject>()->elements(), broker()->isolate()));
+  } else {
+    return FixedArrayBaseRef(data()->AsJSObject()->elements());
+  }
+}
+
+int FixedArrayBaseRef::length() const {
+  IF_BROKER_DISABLED_ACCESS_HANDLE_C(FixedArrayBase, length);
+  return data()->AsFixedArrayBase()->length();
+}
+
+ObjectData* FixedArrayData::Get(int i) const {
+  CHECK_LT(i, static_cast<int>(contents_.size()));
+  CHECK_NOT_NULL(contents_[i]);
+  return contents_[i];
+}
+
+Float64 FixedDoubleArrayData::Get(int i) const {
+  CHECK_LT(i, static_cast<int>(contents_.size()));
+  return contents_[i];
+}
+
+void FeedbackVectorRef::SerializeSlots() {
+  data()->AsFeedbackVector()->SerializeSlots();
+}
+
 Handle<Object> ObjectRef::object() const { return data_->object; }
 
 JSHeapBroker* ObjectRef::broker() const { return data_->broker; }

src/compiler/js-heap-broker.h

@@ -163,7 +163,6 @@ class JSObjectRef : public HeapObjectRef {
  public:
   using HeapObjectRef::HeapObjectRef;
 
-  bool IsUnboxedDoubleField(FieldIndex index) const;
   double RawFastDoublePropertyAt(FieldIndex index) const;
   ObjectRef RawFastPropertyAt(FieldIndex index) const;
 
@@ -278,6 +277,8 @@ class FeedbackVectorRef : public HeapObjectRef {
   using HeapObjectRef::HeapObjectRef;
 
   ObjectRef get(FeedbackSlot slot) const;
+
+  void SerializeSlots();
 };
 
 class AllocationSiteRef : public HeapObjectRef {
@@ -332,6 +333,7 @@ class MapRef : public HeapObjectRef {
   NameRef GetPropertyKey(int i) const;
   FieldIndex GetFieldIndexFor(int i) const;
   ObjectRef GetFieldType(int descriptor) const;
+  bool IsUnboxedDoubleField(FieldIndex index) const;
 };
 
 class FixedArrayBaseRef : public HeapObjectRef {
@@ -346,7 +348,6 @@ class FixedArrayRef : public FixedArrayBaseRef {
   using FixedArrayBaseRef::FixedArrayBaseRef;
 
   ObjectRef get(int i) const;
-  bool is_the_hole(int i) const;
 };
 
 class FixedDoubleArrayRef : public FixedArrayBaseRef {

src/compiler/js-heap-copy-reducer.cc

@@ -46,14 +46,8 @@ Reduction JSHeapCopyReducer::Reduce(Node* node) {
       break;
     }
     case IrOpcode::kJSCreateEmptyLiteralArray: {
-      // TODO(neis, jarin) Force serialization of the entire feedback vector
-      // rather than just the one element.
       FeedbackParameter const& p = FeedbackParameterOf(node->op());
-      FeedbackVectorRef(broker(), p.feedback().vector());
-      Handle<Object> feedback(
-          p.feedback().vector()->Get(p.feedback().slot())->ToObject(),
-          broker()->isolate());
-      ObjectRef(broker(), feedback);
+      FeedbackVectorRef(broker(), p.feedback().vector()).SerializeSlots();
       break;
     }
     case IrOpcode::kJSCreateFunctionContext: {
@@ -65,7 +59,12 @@ Reduction JSHeapCopyReducer::Reduce(Node* node) {
     case IrOpcode::kJSCreateLiteralArray:
     case IrOpcode::kJSCreateLiteralObject: {
       CreateLiteralParameters const& p = CreateLiteralParametersOf(node->op());
-      ObjectRef(broker(), p.feedback().vector());
+      FeedbackVectorRef(broker(), p.feedback().vector()).SerializeSlots();
+      break;
+    }
+    case IrOpcode::kJSCreateLiteralRegExp: {
+      CreateLiteralParameters const& p = CreateLiteralParametersOf(node->op());
+      FeedbackVectorRef(broker(), p.feedback().vector()).SerializeSlots();
       break;
     }
     case IrOpcode::kJSLoadNamed: