[serialize] explicitly serialize code content upfront.

The serializer performs two passes over the code. The first pass copies out the
code content verbatim, the second pass visits references recorded in the reloc
info.

So far the first pass is implicit and happens as part of the second pass, when
we encounter a non-HeapObject reference when iterating the code object.  That
however does not work for internal references. So we hit an assertion if the
first non-HeapObject reference we see is an internal reference.

This change explicitly triggers the first pass.

R=petermarshall@chromium.org

Bug: v8:6817
Change-Id: I1ee9949e10b7d9409986da83be22ac6287785f9f
Reviewed-on: https://chromium-review.googlesource.com/663867
Reviewed-by: Peter Marshall <petermarshall@chromium.org>
Reviewed-by: Yang Guo <yangguo@chromium.org>
Commit-Queue: Yang Guo <yangguo@chromium.org>
Cr-Commit-Position: refs/heads/master@{#48010}

src/objects-body-descriptors-inl.h

@@ -373,6 +373,9 @@ class Code::BodyDescriptor final : public BodyDescriptorBase {
     v->VisitNextCodeLink(Code::cast(obj),
                          HeapObject::RawField(obj, kNextCodeLinkOffset));
 
+    // GC does not visit data/code in the header and in the body directly.
+    STATIC_ASSERT(Code::kNextCodeLinkOffset + kPointerSize == kDataStart);
+
     RelocIterator it(Code::cast(obj), mode_mask);
     Isolate* isolate = obj->GetIsolate();
     for (; !it.done(); it.next()) {

src/objects.h

@@ -4011,6 +4011,11 @@ class Code: public HeapObject {
   static const int kHeaderSize =
       (kHeaderPaddingStart + kCodeAlignmentMask) & ~kCodeAlignmentMask;
 
+  // Data or code not directly visited by GC directly starts here.
+  // The serializer needs to copy bytes starting from here verbatim.
+  // Objects embedded into code is visited via reloc info.
+  static const int kDataStart = kInstructionSizeOffset;
+
   inline int GetUnwindingInfoSizeOffset() const;
 
   class BodyDescriptor;

src/snapshot/deserializer.cc

@@ -653,6 +653,17 @@ bool Deserializer::ReadData(Object** current, Object** limit, int source_space,
         int size_in_bytes = source_.GetInt();
         byte* raw_data_out = reinterpret_cast<byte*>(current);
         source_.CopyRaw(raw_data_out, size_in_bytes);
+        current = reinterpret_cast<Object**>(
+            reinterpret_cast<intptr_t>(current) + size_in_bytes);
+        break;
+      }
+
+      // Deserialize raw code directly into the body of the code object.
+      // Do not move current.
+      case kVariableRawCode: {
+        int size_in_bytes = source_.GetInt();
+        source_.CopyRaw(current_object_address + Code::kDataStart,
+                        size_in_bytes);
         break;
       }
 

src/snapshot/serializer-common.h

@@ -186,21 +186,24 @@ class SerializerDeserializer : public RootVisitor {
   // Repeats of variable length.
   static const int kVariableRepeat = 0x19;
   // Raw data of variable length.
-  static const int kVariableRawData = 0x1a;
-  // Internal reference encoded as offsets of pc and target from code entry.
-  static const int kInternalReference = 0x1b;
-  static const int kInternalReferenceEncoded = 0x1c;
+  static const int kVariableRawCode = 0x1a;
+  static const int kVariableRawData = 0x1b;
+
+  // Used for embedder-allocated backing stores for TypedArrays.
+  static const int kOffHeapBackingStore = 0x1c;
+
   // Used to encode deoptimizer entry code.
   static const int kDeoptimizerEntryPlain = 0x1d;
   static const int kDeoptimizerEntryFromCode = 0x1e;
   // Used for embedder-provided serialization data for embedder fields.
   static const int kEmbedderFieldsData = 0x1f;
 
-  // Used for embedder-allocated backing stores for TypedArrays.
-  static const int kOffHeapBackingStore = 0x35;
+  // Internal reference encoded as offsets of pc and target from code entry.
+  static const int kInternalReference = 0x35;
+  static const int kInternalReferenceEncoded = 0x36;
 
   // Used to encode external referenced provided through the API.
-  static const int kApiReference = 0x36;
+  static const int kApiReference = 0x37;
 
   // 8 hot (recently seen or back-referenced) objects with optional skip.
   static const int kNumberOfHotObjects = 8;
@@ -211,7 +214,7 @@ class SerializerDeserializer : public RootVisitor {
   static const int kHotObjectWithSkip = 0x58;
   static const int kHotObjectMask = 0x07;
 
-  // 0x37, 0x55..0x57, 0x75..0x7f unused.
+  // 0x55..0x57, 0x75..0x7f unused.
 
   // ---------- byte code range 0x80..0xff ----------
   // First 32 root array items.

src/snapshot/serializer.cc

@@ -476,7 +476,7 @@ void Serializer::ObjectSerializer::SerializeJSArrayBuffer() {
     int32_t ref = SerializeBackingStore(backing_store, byte_length);
     buffer->set_backing_store(Smi::FromInt(ref));
   }
-  SerializeContent();
+  SerializeObject();
 }
 
 void Serializer::ObjectSerializer::SerializeFixedTypedArray() {
@@ -495,7 +495,7 @@ void Serializer::ObjectSerializer::SerializeFixedTypedArray() {
     int32_t ref = SerializeBackingStore(backing_store, byte_length);
     fta->set_external_pointer(Smi::FromInt(ref));
   }
-  SerializeContent();
+  SerializeObject();
 }
 
 void Serializer::ObjectSerializer::SerializeExternalString() {
@@ -515,7 +515,7 @@ void Serializer::ObjectSerializer::SerializeExternalString() {
             string->resource());
     // Replace the resource field with the type and index of the native source.
     string->set_resource(resource->EncodeForSerialization());
-    SerializeContent();
+    SerializeObject();
     // Restore the resource field.
     string->set_resource(resource);
   }
@@ -576,9 +576,6 @@ void Serializer::ObjectSerializer::SerializeExternalStringAsSequentialString() {
   int padding_size = allocation_size - SeqString::kHeaderSize - content_size;
   DCHECK(0 <= padding_size && padding_size < kObjectAlignment);
   for (int i = 0; i < padding_size; i++) sink_->PutSection(0, "StringPadding");
-
-  sink_->Put(kSkip, "SkipAfterString");
-  sink_->PutInt(bytes_to_output, "SkipDistance");
 }
 
 // Clear and later restore the next link in the weak cell or allocation site.
@@ -645,10 +642,10 @@ void Serializer::ObjectSerializer::Serialize() {
     Script::cast(object_)->set_line_ends(undefined);
   }
 
-  SerializeContent();
+  SerializeObject();
 }
 
-void Serializer::ObjectSerializer::SerializeContent() {
+void Serializer::ObjectSerializer::SerializeObject() {
   int size = object_->Size();
   Map* map = object_->map();
   AllocationSpace space =
@@ -669,10 +666,7 @@ void Serializer::ObjectSerializer::SerializeContent() {
     return;
   }
 
-  UnlinkWeakNextScope unlink_weak_next(object_);
-
-  object_->IterateBody(map->instance_type(), size, this);
-  OutputRawData(object_->address() + size);
+  SerializeContent(map, size);
 }
 
 void Serializer::ObjectSerializer::SerializeDeferred() {
@@ -697,10 +691,24 @@ void Serializer::ObjectSerializer::SerializeDeferred() {
   serializer_->PutBackReference(object_, back_reference);
   sink_->PutInt(size >> kPointerSizeLog2, "deferred object size");
 
-  UnlinkWeakNextScope unlink_weak_next(object_);
+  SerializeContent(map, size);
+}
 
-  object_->IterateBody(map->instance_type(), size, this);
-  OutputRawData(object_->address() + size);
+void Serializer::ObjectSerializer::SerializeContent(Map* map, int size) {
+  UnlinkWeakNextScope unlink_weak_next(object_);
+  if (object_->IsCode()) {
+    // For code objects, output raw bytes first.
+    OutputCode(size);
+    // Then iterate references via reloc info.
+    object_->IterateBody(map->instance_type(), size, this);
+    // Finally skip to the end.
+    serializer_->FlushSkip(SkipTo(object_->address() + size));
+  } else {
+    // For other objects, iterate references first.
+    object_->IterateBody(map->instance_type(), size, this);
+    // Then output data payload, if any.
+    OutputRawData(object_->address() + size);
+  }
 }
 
 void Serializer::ObjectSerializer::VisitPointers(HeapObject* host,
@@ -744,8 +752,7 @@ void Serializer::ObjectSerializer::VisitPointers(HeapObject* host,
 
 void Serializer::ObjectSerializer::VisitEmbeddedPointer(Code* host,
                                                         RelocInfo* rinfo) {
-  int skip = OutputRawData(rinfo->target_address_address(),
-                           kCanReturnSkipInsteadOfSkipping);
+  int skip = SkipTo(rinfo->target_address_address());
   HowToCode how_to_code = rinfo->IsCodedSpecially() ? kFromCode : kPlain;
   Object* object = rinfo->target_object();
   serializer_->SerializeObject(HeapObject::cast(object), how_to_code,
@@ -755,8 +762,7 @@ void Serializer::ObjectSerializer::VisitEmbeddedPointer(Code* host,
 
 void Serializer::ObjectSerializer::VisitExternalReference(Foreign* host,
                                                           Address* p) {
-  int skip = OutputRawData(reinterpret_cast<Address>(p),
-                           kCanReturnSkipInsteadOfSkipping);
+  int skip = SkipTo(reinterpret_cast<Address>(p));
   Address target = *p;
   auto encoded_reference = serializer_->EncodeExternalReference(target);
   if (encoded_reference.is_from_api()) {
@@ -771,8 +777,7 @@ void Serializer::ObjectSerializer::VisitExternalReference(Foreign* host,
 
 void Serializer::ObjectSerializer::VisitExternalReference(Code* host,
                                                           RelocInfo* rinfo) {
-  int skip = OutputRawData(rinfo->target_address_address(),
-                           kCanReturnSkipInsteadOfSkipping);
+  int skip = SkipTo(rinfo->target_address_address());
   Address target = rinfo->target_external_reference();
   auto encoded_reference = serializer_->EncodeExternalReference(target);
   if (encoded_reference.is_from_api()) {
@@ -791,8 +796,6 @@ void Serializer::ObjectSerializer::VisitExternalReference(Code* host,
 
 void Serializer::ObjectSerializer::VisitInternalReference(Code* host,
                                                           RelocInfo* rinfo) {
-  // We can only reference to internal references of code that has been output.
-  DCHECK(object_->IsCode() && code_has_been_output_);
   // We do not use skip from last patched pc to find the pc to patch, since
   // target_address_address may not return addresses in ascending order when
   // used for internal references. External references may be stored at the
@@ -816,8 +819,7 @@ void Serializer::ObjectSerializer::VisitInternalReference(Code* host,
 
 void Serializer::ObjectSerializer::VisitRuntimeEntry(Code* host,
                                                      RelocInfo* rinfo) {
-  int skip = OutputRawData(rinfo->target_address_address(),
-                           kCanReturnSkipInsteadOfSkipping);
+  int skip = SkipTo(rinfo->target_address_address());
   HowToCode how_to_code = rinfo->IsCodedSpecially() ? kFromCode : kPlain;
   Address target = rinfo->target_address();
   auto encoded_reference = serializer_->EncodeExternalReference(target);
@@ -830,14 +832,51 @@ void Serializer::ObjectSerializer::VisitRuntimeEntry(Code* host,
 
 void Serializer::ObjectSerializer::VisitCodeTarget(Code* host,
                                                    RelocInfo* rinfo) {
-  int skip = OutputRawData(rinfo->target_address_address(),
-                           kCanReturnSkipInsteadOfSkipping);
+  int skip = SkipTo(rinfo->target_address_address());
   Code* object = Code::GetCodeFromTargetAddress(rinfo->target_address());
   serializer_->SerializeObject(object, kFromCode, kInnerPointer, skip);
   bytes_processed_so_far_ += rinfo->target_address_size();
 }
 
-Address Serializer::ObjectSerializer::PrepareCode() {
+void Serializer::ObjectSerializer::OutputRawData(Address up_to) {
+  Address object_start = object_->address();
+  int base = bytes_processed_so_far_;
+  int up_to_offset = static_cast<int>(up_to - object_start);
+  int to_skip = up_to_offset - bytes_processed_so_far_;
+  int bytes_to_output = to_skip;
+  bytes_processed_so_far_ += to_skip;
+  DCHECK(to_skip >= 0);
+  if (bytes_to_output != 0) {
+    DCHECK(to_skip == bytes_to_output);
+    if (IsAligned(bytes_to_output, kPointerAlignment) &&
+        bytes_to_output <= kNumberOfFixedRawData * kPointerSize) {
+      int size_in_words = bytes_to_output >> kPointerSizeLog2;
+      sink_->PutSection(kFixedRawDataStart + size_in_words, "FixedRawData");
+    } else {
+      sink_->Put(kVariableRawData, "VariableRawData");
+      sink_->PutInt(bytes_to_output, "length");
+    }
+#ifdef MEMORY_SANITIZER
+    // Check that we do not serialize uninitialized memory.
+    __msan_check_mem_is_initialized(object_start + base, bytes_to_output);
+#endif  // MEMORY_SANITIZER
+    sink_->PutRaw(object_start + base, bytes_to_output, "Bytes");
+  }
+}
+
+int Serializer::ObjectSerializer::SkipTo(Address to) {
+  Address object_start = object_->address();
+  int up_to_offset = static_cast<int>(to - object_start);
+  int to_skip = up_to_offset - bytes_processed_so_far_;
+  bytes_processed_so_far_ += to_skip;
+  // This assert will fail if the reloc info gives us the target_address_address
+  // locations in a non-ascending order.  Luckily that doesn't happen.
+  DCHECK(to_skip >= 0);
+  return to_skip;
+}
+
+void Serializer::ObjectSerializer::OutputCode(int size) {
+  DCHECK_EQ(kPointerSize, bytes_processed_so_far_);
   Code* code = Code::cast(object_);
   if (FLAG_predictable) {
     // To make snapshots reproducible, we make a copy of the code object
@@ -857,56 +896,18 @@ Address Serializer::ObjectSerializer::PrepareCode() {
     // relocations, because some of these fields are needed for the latter.
     code->WipeOutHeader();
   }
-  return code->address();
-}
 
-int Serializer::ObjectSerializer::OutputRawData(
-    Address up_to, Serializer::ObjectSerializer::ReturnSkip return_skip) {
-  Address object_start = object_->address();
-  int base = bytes_processed_so_far_;
-  int up_to_offset = static_cast<int>(up_to - object_start);
-  int to_skip = up_to_offset - bytes_processed_so_far_;
-  int bytes_to_output = to_skip;
-  bytes_processed_so_far_ += to_skip;
-  // This assert will fail if the reloc info gives us the target_address_address
-  // locations in a non-ascending order.  Luckily that doesn't happen.
-  DCHECK(to_skip >= 0);
-  bool outputting_code = false;
-  bool is_code_object = object_->IsCode();
-  if (to_skip != 0 && is_code_object && !code_has_been_output_) {
-    // Output the code all at once and fix later.
-    bytes_to_output = object_->Size() + to_skip - bytes_processed_so_far_;
-    outputting_code = true;
-    code_has_been_output_ = true;
-  }
-  if (bytes_to_output != 0 && (!is_code_object || outputting_code)) {
-    if (!outputting_code && bytes_to_output == to_skip &&
-        IsAligned(bytes_to_output, kPointerAlignment) &&
-        bytes_to_output <= kNumberOfFixedRawData * kPointerSize) {
-      int size_in_words = bytes_to_output >> kPointerSizeLog2;
-      sink_->PutSection(kFixedRawDataStart + size_in_words, "FixedRawData");
-      to_skip = 0;  // This instruction includes skip.
-    } else {
-      // We always end up here if we are outputting the code of a code object.
-      sink_->Put(kVariableRawData, "VariableRawData");
-      sink_->PutInt(bytes_to_output, "length");
-    }
+  Address start = code->address() + Code::kDataStart;
+  int bytes_to_output = size - Code::kDataStart;
 
-    if (is_code_object) object_start = PrepareCode();
+  sink_->Put(kVariableRawCode, "VariableRawCode");
+  sink_->PutInt(bytes_to_output, "length");
 
-    const char* description = is_code_object ? "Code" : "Byte";
 #ifdef MEMORY_SANITIZER
-    // Check that we do not serialize uninitialized memory.
-    __msan_check_mem_is_initialized(object_start + base, bytes_to_output);
+  // Check that we do not serialize uninitialized memory.
+  __msan_check_mem_is_initialized(start, bytes_to_output);
 #endif  // MEMORY_SANITIZER
-    sink_->PutRaw(object_start + base, bytes_to_output, description);
-  }
-  if (to_skip != 0 && return_skip == kIgnoringReturn) {
-    sink_->Put(kSkip, "Skip");
-    sink_->PutInt(to_skip, "SkipDistance");
-    to_skip = 0;
-  }
-  return to_skip;
+  sink_->PutRaw(start, bytes_to_output, "Code");
 }
 
 }  // namespace internal

src/snapshot/serializer.h

@@ -310,8 +310,7 @@ class Serializer::ObjectSerializer : public ObjectVisitor {
         object_(obj),
         sink_(sink),
         reference_representation_(how_to_code + where_to_point),
-        bytes_processed_so_far_(0),
-        code_has_been_output_(false) {
+        bytes_processed_so_far_(0) {
 #ifdef DEBUG
     serializer_->PushStack(obj);
 #endif  // DEBUG
@@ -322,7 +321,7 @@ class Serializer::ObjectSerializer : public ObjectVisitor {
 #endif  // DEBUG
   }
   void Serialize();
-  void SerializeContent();
+  void SerializeObject();
   void SerializeDeferred();
   void VisitPointers(HeapObject* host, Object** start, Object** end) override;
   void VisitEmbeddedPointer(Code* host, RelocInfo* target) override;
@@ -335,13 +334,12 @@ class Serializer::ObjectSerializer : public ObjectVisitor {
  private:
   void SerializePrologue(AllocationSpace space, int size, Map* map);
 
-
-  enum ReturnSkip { kCanReturnSkipInsteadOfSkipping, kIgnoringReturn };
   // This function outputs or skips the raw data between the last pointer and
-  // up to the current position.  It optionally can just return the number of
-  // bytes to skip instead of performing a skip instruction, in case the skip
-  // can be merged into the next instruction.
-  int OutputRawData(Address up_to, ReturnSkip return_skip = kIgnoringReturn);
+  // up to the current position.
+  void SerializeContent(Map* map, int size);
+  void OutputRawData(Address up_to);
+  void OutputCode(int size);
+  int SkipTo(Address to);
   int32_t SerializeBackingStore(void* backing_store, int32_t byte_length);
   void FixupIfNeutered();
   void SerializeJSArrayBuffer();
@@ -349,15 +347,12 @@ class Serializer::ObjectSerializer : public ObjectVisitor {
   void SerializeExternalString();
   void SerializeExternalStringAsSequentialString();
 
-  Address PrepareCode();
-
   Serializer* serializer_;
   HeapObject* object_;
   SnapshotByteSink* sink_;
   std::map<void*, Smi*> backing_stores;
   int reference_representation_;
   int bytes_processed_so_far_;
-  bool code_has_been_output_;
 };
 
 }  // namespace internal