[heap] Integrate embedder heap tracer into incremental marking

- Respect a deadline when processing wrappers during incremental marking, i.e.,
  don't force completetion on each individual step
- Only finalize when both, the internal marking deque and wrappers, are
  processed

Some results (twitter using --enable-blink-features=TraceWrappables)

=== BEFORE ===
incremental_wrapper_tracing_longest_step
  len: 13
  min: 0.0
  ---> max: 6.6
  ---> avg: 3.83846153846
  [0,5[: 7
  [5,10[: 6
mark.wrapper_tracing
  len: 13
  min: 0.0
  max: 2.0 (non-incremental MC)
  avg: 0.153846153846
  [0,5[: 13

=== AFTER ===
incremental_wrapper_tracing_longest_step
  len: 8
  min: 0.0
  ---> max: 1.1
  ---> avg: 0.8875
  [0,5[: 8
mark.wrapper_tracing (not excluding non-incremental MC)
  len: 8
  min: 0.0
  max: 2.5 (non-incremental MC)
  avg: 0.3125
  [0,5[: 8

BUG=chromium:468240
R=ulan@chromium.org

Review-Url: https://codereview.chromium.org/2287343002
Cr-Commit-Position: refs/heads/master@{#39041}

src/heap/heap.cc

@@ -4175,12 +4175,20 @@ void Heap::ReduceNewSpaceSize() {
   }
 }
 
+bool Heap::MarkingDequesAreEmpty() {
+  return mark_compact_collector()->marking_deque()->IsEmpty() &&
+         (!UsingEmbedderHeapTracer() ||
+          (mark_compact_collector()->wrappers_to_trace() == 0 &&
+           mark_compact_collector()
+                   ->embedder_heap_tracer()
+                   ->NumberOfWrappersToTrace() == 0));
+}
 
 void Heap::FinalizeIncrementalMarkingIfComplete(const char* comment) {
   if (incremental_marking()->IsMarking() &&
       (incremental_marking()->IsReadyToOverApproximateWeakClosure() ||
        (!incremental_marking()->finalize_marking_completed() &&
-        mark_compact_collector()->marking_deque()->IsEmpty()))) {
+        MarkingDequesAreEmpty()))) {
     FinalizeIncrementalMarking(comment);
   } else if (incremental_marking()->IsComplete() ||
              (mark_compact_collector()->marking_deque()->IsEmpty())) {
@@ -4195,14 +4203,14 @@ bool Heap::TryFinalizeIdleIncrementalMarking(double idle_time_in_ms) {
       tracer()->FinalIncrementalMarkCompactSpeedInBytesPerMillisecond();
   if (incremental_marking()->IsReadyToOverApproximateWeakClosure() ||
       (!incremental_marking()->finalize_marking_completed() &&
-       mark_compact_collector()->marking_deque()->IsEmpty() &&
+       MarkingDequesAreEmpty() &&
        gc_idle_time_handler_->ShouldDoOverApproximateWeakClosure(
            idle_time_in_ms))) {
     FinalizeIncrementalMarking(
         "Idle notification: finalize incremental marking");
     return true;
   } else if (incremental_marking()->IsComplete() ||
-             (mark_compact_collector()->marking_deque()->IsEmpty() &&
+             (MarkingDequesAreEmpty() &&
               gc_idle_time_handler_->ShouldDoFinalIncrementalMarkCompact(
                   idle_time_in_ms, size_of_objects,
                   final_incremental_mark_compact_speed_in_bytes_per_ms))) {

src/heap/heap.h

@@ -763,14 +763,6 @@ class Heap {
   // Returns false if not able to reserve.
   bool ReserveSpace(Reservation* reservations, List<Address>* maps);
 
-  void SetEmbedderHeapTracer(EmbedderHeapTracer* tracer);
-
-  bool UsingEmbedderHeapTracer();
-
-  void TracePossibleWrapper(JSObject* js_object);
-
-  void RegisterExternallyReferencedObject(Object** object);
-
   //
   // Support for the API.
   //
@@ -1173,6 +1165,18 @@ class Heap {
 
   IncrementalMarking* incremental_marking() { return incremental_marking_; }
 
+  // ===========================================================================
+  // Embedder heap tracer support. =============================================
+  // ===========================================================================
+
+  void SetEmbedderHeapTracer(EmbedderHeapTracer* tracer);
+
+  bool UsingEmbedderHeapTracer();
+
+  void TracePossibleWrapper(JSObject* js_object);
+
+  void RegisterExternallyReferencedObject(Object** object);
+
   // ===========================================================================
   // External string table API. ================================================
   // ===========================================================================
@@ -1588,6 +1592,10 @@ class Heap {
     return current_gc_flags_ & kFinalizeIncrementalMarkingMask;
   }
 
+  // Checks whether both, the internal marking deque, and the embedder provided
+  // one are empty. Avoid in fast path as it potentially calls through the API.
+  bool MarkingDequesAreEmpty();
+
   void PreprocessStackTraces();
 
   // Checks whether a global GC is necessary

src/heap/incremental-marking.cc

@@ -749,7 +749,6 @@ void IncrementalMarking::RetainMaps() {
   }
 }
 
-
 void IncrementalMarking::FinalizeIncrementally() {
   TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_INCREMENTAL_FINALIZE_BODY);
   DCHECK(!finalize_marking_completed_);
@@ -782,6 +781,9 @@ void IncrementalMarking::FinalizeIncrementally() {
       abs(old_marking_deque_top -
           heap_->mark_compact_collector()->marking_deque()->top());
 
+  marking_progress +=
+      static_cast<int>(heap_->mark_compact_collector()->wrappers_to_trace());
+
   double end = heap_->MonotonicallyIncreasingTimeInMs();
   double delta = end - start;
   heap_->tracer()->AddMarkingTime(delta);
@@ -1055,7 +1057,7 @@ double IncrementalMarking::AdvanceIncrementalMarking(
              step_actions.force_marking, step_actions.force_completion);
     remaining_time_in_ms =
         deadline_in_ms - heap()->MonotonicallyIncreasingTimeInMs();
-  } while (bytes_processed > 0 &&
+  } while (!heap_->mark_compact_collector()->marking_deque()->IsEmpty() &&
            remaining_time_in_ms >=
                2.0 * GCIdleTimeHandler::kIncrementalMarkingStepTimeInMs &&
            !IsComplete() &&
@@ -1218,23 +1220,35 @@ intptr_t IncrementalMarking::Step(intptr_t allocated_bytes,
     }
 
     if (state_ == MARKING) {
-      bytes_processed = ProcessMarkingDeque(bytes_to_process);
-      if (FLAG_incremental_marking_wrappers &&
-          heap_->UsingEmbedderHeapTracer()) {
+      const bool incremental_wrapper_tracing =
+          FLAG_incremental_marking_wrappers && heap_->UsingEmbedderHeapTracer();
+      const bool process_wrappers =
+          incremental_wrapper_tracing &&
+          (heap_->mark_compact_collector()
+               ->RequiresImmediateWrapperProcessing() ||
+           heap_->mark_compact_collector()->marking_deque()->IsEmpty());
+      bool wrapper_work_left = incremental_wrapper_tracing;
+      if (!process_wrappers) {
+        bytes_processed = ProcessMarkingDeque(bytes_to_process);
+      } else {
+        const double kWrapperTracngStepMs = 1.0;
+        const double wrapper_deadline =
+            heap_->MonotonicallyIncreasingTimeInMs() + kWrapperTracngStepMs;
         TRACE_GC(heap()->tracer(),
                  GCTracer::Scope::MC_INCREMENTAL_WRAPPER_TRACING);
-        // This currently marks through all registered wrappers and does not
-        // respect bytes_to_process.
-        // TODO(hpayer): Integrate incremental marking of wrappers into
-        // bytes_to_process logic.
         heap_->mark_compact_collector()
             ->RegisterWrappersWithEmbedderHeapTracer();
-        heap_->mark_compact_collector()->embedder_heap_tracer()->AdvanceTracing(
-            0,
-            EmbedderHeapTracer::AdvanceTracingActions(
-                EmbedderHeapTracer::ForceCompletionAction::FORCE_COMPLETION));
+        wrapper_work_left =
+            heap_->mark_compact_collector()
+                ->embedder_heap_tracer()
+                ->AdvanceTracing(wrapper_deadline,
+                                 EmbedderHeapTracer::AdvanceTracingActions(
+                                     EmbedderHeapTracer::ForceCompletionAction::
+                                         DO_NOT_FORCE_COMPLETION));
       }
-      if (heap_->mark_compact_collector()->marking_deque()->IsEmpty()) {
+
+      if (heap_->mark_compact_collector()->marking_deque()->IsEmpty() &&
+          !wrapper_work_left) {
         if (completion == FORCE_COMPLETION ||
             IsIdleMarkingDelayCounterLimitReached()) {
           if (!finalize_marking_completed_) {

src/heap/mark-compact.cc

@@ -2206,6 +2206,11 @@ void MarkCompactCollector::SetEmbedderHeapTracer(EmbedderHeapTracer* tracer) {
   embedder_heap_tracer_ = tracer;
 }
 
+bool MarkCompactCollector::RequiresImmediateWrapperProcessing() {
+  const size_t kTooManyWrappers = 16000;
+  return wrappers_to_trace_.size() > kTooManyWrappers;
+}
+
 void MarkCompactCollector::RegisterWrappersWithEmbedderHeapTracer() {
   DCHECK(UsingEmbedderHeapTracer());
   if (wrappers_to_trace_.empty()) {

src/heap/mark-compact.h

@@ -490,16 +490,24 @@ class MarkCompactCollector {
 
   Sweeper& sweeper() { return sweeper_; }
 
-  void RegisterWrappersWithEmbedderHeapTracer();
+  // ===========================================================================
+  // Embedder heap tracer support. =============================================
+  // ===========================================================================
 
   void SetEmbedderHeapTracer(EmbedderHeapTracer* tracer);
-
   EmbedderHeapTracer* embedder_heap_tracer() { return embedder_heap_tracer_; }
-
   bool UsingEmbedderHeapTracer() { return embedder_heap_tracer(); }
 
+  // In order to avoid running out of memory we force tracing wrappers if there
+  // are too many of them.
+  bool RequiresImmediateWrapperProcessing();
+
+  void RegisterWrappersWithEmbedderHeapTracer();
+
   void TracePossibleWrapper(JSObject* js_object);
 
+  size_t wrappers_to_trace() { return wrappers_to_trace_.size(); }
+
  private:
   class EvacuateNewSpacePageVisitor;
   class EvacuateNewSpaceVisitor;