diff --git a/src/compiler/scheduler.cc b/src/compiler/scheduler.cc index 82cc95972a..69758824b3 100644 --- a/src/compiler/scheduler.cc +++ b/src/compiler/scheduler.cc @@ -49,7 +49,7 @@ Schedule* Scheduler::ComputeSchedule(ZonePool* zone_pool, Graph* graph) { Scheduler scheduler(zone_scope.zone(), graph, schedule); scheduler.BuildCFG(); - Scheduler::ComputeSpecialRPO(zone_pool, schedule); + scheduler.ComputeSpecialRPONumbering(); scheduler.GenerateImmediateDominatorTree(); scheduler.PrepareUses(); @@ -173,7 +173,7 @@ BasicBlock* Scheduler::GetCommonDominator(BasicBlock* b1, BasicBlock* b2) { // ----------------------------------------------------------------------------- -// Phase 1: Build control-flow graph and dominator tree. +// Phase 1: Build control-flow graph. // Internal class to build a control flow graph (i.e the basic blocks and edges @@ -395,10 +395,456 @@ void Scheduler::BuildCFG() { } +// ----------------------------------------------------------------------------- +// Phase 2: Compute special RPO and dominator tree. + + +// Compute the special reverse-post-order block ordering, which is essentially +// a RPO of the graph where loop bodies are contiguous. Properties: +// 1. If block A is a predecessor of B, then A appears before B in the order, +// unless B is a loop header and A is in the loop headed at B +// (i.e. A -> B is a backedge). +// => If block A dominates block B, then A appears before B in the order. +// => If block A is a loop header, A appears before all blocks in the loop +// headed at A. +// 2. All loops are contiguous in the order (i.e. no intervening blocks that +// do not belong to the loop.) +// Note a simple RPO traversal satisfies (1) but not (2). +class SpecialRPONumberer { + public: + SpecialRPONumberer(Zone* zone, Schedule* schedule) + : zone_(zone), schedule_(schedule) {} + + void ComputeSpecialRPO() { + // RPO should not have been computed for this schedule yet. + CHECK_EQ(kBlockUnvisited1, schedule_->start()->rpo_number()); + CHECK_EQ(0, static_cast(schedule_->rpo_order()->size())); + + // Perform an iterative RPO traversal using an explicit stack, + // recording backedges that form cycles. O(|B|). + ZoneList > backedges(1, zone_); + SpecialRPOStackFrame* stack = zone_->NewArray( + static_cast(schedule_->BasicBlockCount())); + BasicBlock* entry = schedule_->start(); + BlockList* order = NULL; + int stack_depth = Push(stack, 0, entry, kBlockUnvisited1); + int num_loops = 0; + + while (stack_depth > 0) { + int current = stack_depth - 1; + SpecialRPOStackFrame* frame = stack + current; + + if (frame->index < frame->block->SuccessorCount()) { + // Process the next successor. + BasicBlock* succ = frame->block->SuccessorAt(frame->index++); + if (succ->rpo_number() == kBlockVisited1) continue; + if (succ->rpo_number() == kBlockOnStack) { + // The successor is on the stack, so this is a backedge (cycle). + backedges.Add( + std::pair(frame->block, frame->index - 1), + zone_); + if (succ->loop_end() < 0) { + // Assign a new loop number to the header if it doesn't have one. + succ->set_loop_end(num_loops++); + } + } else { + // Push the successor onto the stack. + DCHECK(succ->rpo_number() == kBlockUnvisited1); + stack_depth = Push(stack, stack_depth, succ, kBlockUnvisited1); + } + } else { + // Finished with all successors; pop the stack and add the block. + order = order->Add(zone_, frame->block); + frame->block->set_rpo_number(kBlockVisited1); + stack_depth--; + } + } + + // If no loops were encountered, then the order we computed was correct. + LoopInfo* loops = NULL; + if (num_loops != 0) { + // Otherwise, compute the loop information from the backedges in order + // to perform a traversal that groups loop bodies together. + loops = ComputeLoopInfo(stack, num_loops, schedule_->BasicBlockCount(), + &backedges); + + // Initialize the "loop stack". Note the entry could be a loop header. + LoopInfo* loop = entry->IsLoopHeader() ? &loops[entry->loop_end()] : NULL; + order = NULL; + + // Perform an iterative post-order traversal, visiting loop bodies before + // edges that lead out of loops. Visits each block once, but linking loop + // sections together is linear in the loop size, so overall is + // O(|B| + max(loop_depth) * max(|loop|)) + stack_depth = Push(stack, 0, entry, kBlockUnvisited2); + while (stack_depth > 0) { + SpecialRPOStackFrame* frame = stack + (stack_depth - 1); + BasicBlock* block = frame->block; + BasicBlock* succ = NULL; + + if (frame->index < block->SuccessorCount()) { + // Process the next normal successor. + succ = block->SuccessorAt(frame->index++); + } else if (block->IsLoopHeader()) { + // Process additional outgoing edges from the loop header. + if (block->rpo_number() == kBlockOnStack) { + // Finish the loop body the first time the header is left on the + // stack. + DCHECK(loop != NULL && loop->header == block); + loop->start = order->Add(zone_, block); + order = loop->end; + block->set_rpo_number(kBlockVisited2); + // Pop the loop stack and continue visiting outgoing edges within + // the context of the outer loop, if any. + loop = loop->prev; + // We leave the loop header on the stack; the rest of this iteration + // and later iterations will go through its outgoing edges list. + } + + // Use the next outgoing edge if there are any. + int outgoing_index = + static_cast(frame->index - block->SuccessorCount()); + LoopInfo* info = &loops[block->loop_end()]; + DCHECK(loop != info); + if (info->outgoing != NULL && + outgoing_index < info->outgoing->length()) { + succ = info->outgoing->at(outgoing_index); + frame->index++; + } + } + + if (succ != NULL) { + // Process the next successor. + if (succ->rpo_number() == kBlockOnStack) continue; + if (succ->rpo_number() == kBlockVisited2) continue; + DCHECK(succ->rpo_number() == kBlockUnvisited2); + if (loop != NULL && !loop->members->Contains(succ->id().ToInt())) { + // The successor is not in the current loop or any nested loop. + // Add it to the outgoing edges of this loop and visit it later. + loop->AddOutgoing(zone_, succ); + } else { + // Push the successor onto the stack. + stack_depth = Push(stack, stack_depth, succ, kBlockUnvisited2); + if (succ->IsLoopHeader()) { + // Push the inner loop onto the loop stack. + DCHECK(succ->loop_end() >= 0 && succ->loop_end() < num_loops); + LoopInfo* next = &loops[succ->loop_end()]; + next->end = order; + next->prev = loop; + loop = next; + } + } + } else { + // Finished with all successors of the current block. + if (block->IsLoopHeader()) { + // If we are going to pop a loop header, then add its entire body. + LoopInfo* info = &loops[block->loop_end()]; + for (BlockList* l = info->start; true; l = l->next) { + if (l->next == info->end) { + l->next = order; + info->end = order; + break; + } + } + order = info->start; + } else { + // Pop a single node off the stack and add it to the order. + order = order->Add(zone_, block); + block->set_rpo_number(kBlockVisited2); + } + stack_depth--; + } + } + } + + // Construct the final order from the list. + BasicBlockVector* final_order = schedule_->rpo_order(); + order->Serialize(final_order); + + // Compute the correct loop header for every block and set the correct loop + // ends. + LoopInfo* current_loop = NULL; + BasicBlock* current_header = NULL; + int loop_depth = 0; + for (BasicBlockVectorIter i = final_order->begin(); i != final_order->end(); + ++i) { + BasicBlock* current = *i; + current->set_loop_header(current_header); + if (current->IsLoopHeader()) { + loop_depth++; + current_loop = &loops[current->loop_end()]; + BlockList* end = current_loop->end; + current->set_loop_end(end == NULL + ? static_cast(final_order->size()) + : end->block->rpo_number()); + current_header = current_loop->header; + Trace("B%d is a loop header, increment loop depth to %d\n", + current->id().ToInt(), loop_depth); + } else { + while (current_header != NULL && + current->rpo_number() >= current_header->loop_end()) { + DCHECK(current_header->IsLoopHeader()); + DCHECK(current_loop != NULL); + current_loop = current_loop->prev; + current_header = current_loop == NULL ? NULL : current_loop->header; + --loop_depth; + } + } + current->set_loop_depth(loop_depth); + if (current->loop_header() == NULL) { + Trace("B%d is not in a loop (depth == %d)\n", current->id().ToInt(), + current->loop_depth()); + } else { + Trace("B%d has loop header B%d, (depth == %d)\n", current->id().ToInt(), + current->loop_header()->id().ToInt(), current->loop_depth()); + } + } + + // Compute the assembly order (non-deferred code first, deferred code + // afterwards). + int32_t number = 0; + for (auto block : *final_order) { + if (block->deferred()) continue; + block->set_ao_number(number++); + } + for (auto block : *final_order) { + if (!block->deferred()) continue; + block->set_ao_number(number++); + } + +#if DEBUG + if (FLAG_trace_turbo_scheduler) PrintRPO(num_loops, loops, final_order); + VerifySpecialRPO(num_loops, loops, final_order); +#endif + } + + private: + // Numbering for BasicBlockData.rpo_number_ for this block traversal: + static const int kBlockOnStack = -2; + static const int kBlockVisited1 = -3; + static const int kBlockVisited2 = -4; + static const int kBlockUnvisited1 = -1; + static const int kBlockUnvisited2 = kBlockVisited1; + + struct SpecialRPOStackFrame { + BasicBlock* block; + size_t index; + }; + + struct BlockList { + BasicBlock* block; + BlockList* next; + + BlockList* Add(Zone* zone, BasicBlock* b) { + BlockList* list = static_cast(zone->New(sizeof(BlockList))); + list->block = b; + list->next = this; + return list; + } + + void Serialize(BasicBlockVector* final_order) { + for (BlockList* l = this; l != NULL; l = l->next) { + l->block->set_rpo_number(static_cast(final_order->size())); + final_order->push_back(l->block); + } + } + }; + + struct LoopInfo { + BasicBlock* header; + ZoneList* outgoing; + BitVector* members; + LoopInfo* prev; + BlockList* end; + BlockList* start; + + void AddOutgoing(Zone* zone, BasicBlock* block) { + if (outgoing == NULL) { + outgoing = new (zone) ZoneList(2, zone); + } + outgoing->Add(block, zone); + } + }; + + int Push(SpecialRPOStackFrame* stack, int depth, BasicBlock* child, + int unvisited) { + if (child->rpo_number() == unvisited) { + stack[depth].block = child; + stack[depth].index = 0; + child->set_rpo_number(kBlockOnStack); + return depth + 1; + } + return depth; + } + + // Computes loop membership from the backedges of the control flow graph. + LoopInfo* ComputeLoopInfo( + SpecialRPOStackFrame* queue, int num_loops, size_t num_blocks, + ZoneList >* backedges) { + LoopInfo* loops = zone_->NewArray(num_loops); + memset(loops, 0, num_loops * sizeof(LoopInfo)); + + // Compute loop membership starting from backedges. + // O(max(loop_depth) * max(|loop|) + for (int i = 0; i < backedges->length(); i++) { + BasicBlock* member = backedges->at(i).first; + BasicBlock* header = member->SuccessorAt(backedges->at(i).second); + int loop_num = header->loop_end(); + if (loops[loop_num].header == NULL) { + loops[loop_num].header = header; + loops[loop_num].members = + new (zone_) BitVector(static_cast(num_blocks), zone_); + } + + int queue_length = 0; + if (member != header) { + // As long as the header doesn't have a backedge to itself, + // Push the member onto the queue and process its predecessors. + if (!loops[loop_num].members->Contains(member->id().ToInt())) { + loops[loop_num].members->Add(member->id().ToInt()); + } + queue[queue_length++].block = member; + } + + // Propagate loop membership backwards. All predecessors of M up to the + // loop header H are members of the loop too. O(|blocks between M and H|). + while (queue_length > 0) { + BasicBlock* block = queue[--queue_length].block; + for (size_t i = 0; i < block->PredecessorCount(); i++) { + BasicBlock* pred = block->PredecessorAt(i); + if (pred != header) { + if (!loops[loop_num].members->Contains(pred->id().ToInt())) { + loops[loop_num].members->Add(pred->id().ToInt()); + queue[queue_length++].block = pred; + } + } + } + } + } + return loops; + } + +#if DEBUG + void PrintRPO(int num_loops, LoopInfo* loops, BasicBlockVector* order) { + OFStream os(stdout); + os << "-- RPO with " << num_loops << " loops "; + if (num_loops > 0) { + os << "("; + for (int i = 0; i < num_loops; i++) { + if (i > 0) os << " "; + os << "B" << loops[i].header->id(); + } + os << ") "; + } + os << "-- \n"; + + for (size_t i = 0; i < order->size(); i++) { + BasicBlock* block = (*order)[i]; + BasicBlock::Id bid = block->id(); + // TODO(jarin,svenpanne): Add formatting here once we have support for + // that in streams (we want an equivalent of PrintF("%5d:", i) here). + os << i << ":"; + for (int j = 0; j < num_loops; j++) { + bool membership = loops[j].members->Contains(bid.ToInt()); + bool range = loops[j].header->LoopContains(block); + os << (membership ? " |" : " "); + os << (range ? "x" : " "); + } + os << " B" << bid << ": "; + if (block->loop_end() >= 0) { + os << " range: [" << block->rpo_number() << ", " << block->loop_end() + << ")"; + } + os << "\n"; + } + } + + void VerifySpecialRPO(int num_loops, LoopInfo* loops, + BasicBlockVector* order) { + DCHECK(order->size() > 0); + DCHECK((*order)[0]->id().ToInt() == 0); // entry should be first. + + for (int i = 0; i < num_loops; i++) { + LoopInfo* loop = &loops[i]; + BasicBlock* header = loop->header; + + DCHECK(header != NULL); + DCHECK(header->rpo_number() >= 0); + DCHECK(header->rpo_number() < static_cast(order->size())); + DCHECK(header->loop_end() >= 0); + DCHECK(header->loop_end() <= static_cast(order->size())); + DCHECK(header->loop_end() > header->rpo_number()); + + // Verify the start ... end list relationship. + int links = 0; + BlockList* l = loop->start; + DCHECK(l != NULL && l->block == header); + bool end_found; + while (true) { + if (l == NULL || l == loop->end) { + end_found = (loop->end == l); + break; + } + // The list should be in same order as the final result. + DCHECK(l->block->rpo_number() == links + loop->header->rpo_number()); + links++; + l = l->next; + DCHECK(links < static_cast(2 * order->size())); // cycle? + } + DCHECK(links > 0); + DCHECK(links == (header->loop_end() - header->rpo_number())); + DCHECK(end_found); + + // Check the contiguousness of loops. + int count = 0; + for (int j = 0; j < static_cast(order->size()); j++) { + BasicBlock* block = order->at(j); + DCHECK(block->rpo_number() == j); + if (j < header->rpo_number() || j >= header->loop_end()) { + DCHECK(!loop->members->Contains(block->id().ToInt())); + } else { + if (block == header) { + DCHECK(!loop->members->Contains(block->id().ToInt())); + } else { + DCHECK(loop->members->Contains(block->id().ToInt())); + } + count++; + } + } + DCHECK(links == count); + } + } +#endif // DEBUG + + Zone* zone_; + Schedule* schedule_; +}; + + +BasicBlockVector* Scheduler::ComputeSpecialRPO(ZonePool* zone_pool, + Schedule* schedule) { + ZonePool::Scope zone_scope(zone_pool); + Zone* zone = zone_scope.zone(); + + SpecialRPONumberer numberer(zone, schedule); + numberer.ComputeSpecialRPO(); + return schedule->rpo_order(); +} + + +void Scheduler::ComputeSpecialRPONumbering() { + Trace("--- COMPUTING SPECIAL RPO ----------------------------------\n"); + + SpecialRPONumberer numberer(zone_, schedule_); + numberer.ComputeSpecialRPO(); +} + + void Scheduler::GenerateImmediateDominatorTree() { - // Build the dominator graph. TODO(danno): consider using Lengauer & Tarjan's - // if this becomes really slow. Trace("--- IMMEDIATE BLOCK DOMINATORS -----------------------------\n"); + + // Build the dominator graph. + // TODO(danno): consider using Lengauer & Tarjan's if this becomes too slow. for (size_t i = 0; i < schedule_->rpo_order_.size(); i++) { BasicBlock* current_rpo = schedule_->rpo_order_[i]; if (current_rpo != schedule_->start()) { @@ -428,7 +874,7 @@ void Scheduler::GenerateImmediateDominatorTree() { // ----------------------------------------------------------------------------- -// Phase 2: Prepare use counts for nodes. +// Phase 3: Prepare use counts for nodes. class PrepareUsesVisitor : public NullNodeVisitor { @@ -490,7 +936,7 @@ void Scheduler::PrepareUses() { // ----------------------------------------------------------------------------- -// Phase 3: Schedule nodes early. +// Phase 4: Schedule nodes early. class ScheduleEarlyNodeVisitor : public NullNodeVisitor { @@ -567,7 +1013,7 @@ void Scheduler::ScheduleEarly() { // ----------------------------------------------------------------------------- -// Phase 4: Schedule nodes late. +// Phase 5: Schedule nodes late. class ScheduleLateNodeVisitor { @@ -835,425 +1281,6 @@ void Scheduler::ConnectFloatingControlSubgraph(BasicBlock* block, Node* end) { block_start->op()->mnemonic()); } - -// Numbering for BasicBlockData.rpo_number_ for this block traversal: -static const int kBlockOnStack = -2; -static const int kBlockVisited1 = -3; -static const int kBlockVisited2 = -4; -static const int kBlockUnvisited1 = -1; -static const int kBlockUnvisited2 = kBlockVisited1; - -struct SpecialRPOStackFrame { - BasicBlock* block; - size_t index; -}; - -struct BlockList { - BasicBlock* block; - BlockList* next; - - BlockList* Add(Zone* zone, BasicBlock* b) { - BlockList* list = static_cast(zone->New(sizeof(BlockList))); - list->block = b; - list->next = this; - return list; - } - - void Serialize(BasicBlockVector* final_order) { - for (BlockList* l = this; l != NULL; l = l->next) { - l->block->set_rpo_number(static_cast(final_order->size())); - final_order->push_back(l->block); - } - } -}; - -struct LoopInfo { - BasicBlock* header; - ZoneList* outgoing; - BitVector* members; - LoopInfo* prev; - BlockList* end; - BlockList* start; - - void AddOutgoing(Zone* zone, BasicBlock* block) { - if (outgoing == NULL) outgoing = new (zone) ZoneList(2, zone); - outgoing->Add(block, zone); - } -}; - - -static int Push(SpecialRPOStackFrame* stack, int depth, BasicBlock* child, - int unvisited) { - if (child->rpo_number() == unvisited) { - stack[depth].block = child; - stack[depth].index = 0; - child->set_rpo_number(kBlockOnStack); - return depth + 1; - } - return depth; -} - - -// Computes loop membership from the backedges of the control flow graph. -static LoopInfo* ComputeLoopInfo( - Zone* zone, SpecialRPOStackFrame* queue, int num_loops, size_t num_blocks, - ZoneList >* backedges) { - LoopInfo* loops = zone->NewArray(num_loops); - memset(loops, 0, num_loops * sizeof(LoopInfo)); - - // Compute loop membership starting from backedges. - // O(max(loop_depth) * max(|loop|) - for (int i = 0; i < backedges->length(); i++) { - BasicBlock* member = backedges->at(i).first; - BasicBlock* header = member->SuccessorAt(backedges->at(i).second); - int loop_num = header->loop_end(); - if (loops[loop_num].header == NULL) { - loops[loop_num].header = header; - loops[loop_num].members = - new (zone) BitVector(static_cast(num_blocks), zone); - } - - int queue_length = 0; - if (member != header) { - // As long as the header doesn't have a backedge to itself, - // Push the member onto the queue and process its predecessors. - if (!loops[loop_num].members->Contains(member->id().ToInt())) { - loops[loop_num].members->Add(member->id().ToInt()); - } - queue[queue_length++].block = member; - } - - // Propagate loop membership backwards. All predecessors of M up to the - // loop header H are members of the loop too. O(|blocks between M and H|). - while (queue_length > 0) { - BasicBlock* block = queue[--queue_length].block; - for (size_t i = 0; i < block->PredecessorCount(); i++) { - BasicBlock* pred = block->PredecessorAt(i); - if (pred != header) { - if (!loops[loop_num].members->Contains(pred->id().ToInt())) { - loops[loop_num].members->Add(pred->id().ToInt()); - queue[queue_length++].block = pred; - } - } - } - } - } - return loops; -} - - -#if DEBUG -static void PrintRPO(int num_loops, LoopInfo* loops, BasicBlockVector* order) { - OFStream os(stdout); - os << "-- RPO with " << num_loops << " loops "; - if (num_loops > 0) { - os << "("; - for (int i = 0; i < num_loops; i++) { - if (i > 0) os << " "; - os << "B" << loops[i].header->id(); - } - os << ") "; - } - os << "-- \n"; - - for (size_t i = 0; i < order->size(); i++) { - BasicBlock* block = (*order)[i]; - BasicBlock::Id bid = block->id(); - // TODO(jarin,svenpanne): Add formatting here once we have support for that - // in streams (we want an equivalent of PrintF("%5d:", i) here). - os << i << ":"; - for (int j = 0; j < num_loops; j++) { - bool membership = loops[j].members->Contains(bid.ToInt()); - bool range = loops[j].header->LoopContains(block); - os << (membership ? " |" : " "); - os << (range ? "x" : " "); - } - os << " B" << bid << ": "; - if (block->loop_end() >= 0) { - os << " range: [" << block->rpo_number() << ", " << block->loop_end() - << ")"; - } - os << "\n"; - } -} - - -static void VerifySpecialRPO(int num_loops, LoopInfo* loops, - BasicBlockVector* order) { - DCHECK(order->size() > 0); - DCHECK((*order)[0]->id().ToInt() == 0); // entry should be first. - - for (int i = 0; i < num_loops; i++) { - LoopInfo* loop = &loops[i]; - BasicBlock* header = loop->header; - - DCHECK(header != NULL); - DCHECK(header->rpo_number() >= 0); - DCHECK(header->rpo_number() < static_cast(order->size())); - DCHECK(header->loop_end() >= 0); - DCHECK(header->loop_end() <= static_cast(order->size())); - DCHECK(header->loop_end() > header->rpo_number()); - - // Verify the start ... end list relationship. - int links = 0; - BlockList* l = loop->start; - DCHECK(l != NULL && l->block == header); - bool end_found; - while (true) { - if (l == NULL || l == loop->end) { - end_found = (loop->end == l); - break; - } - // The list should be in same order as the final result. - DCHECK(l->block->rpo_number() == links + loop->header->rpo_number()); - links++; - l = l->next; - DCHECK(links < static_cast(2 * order->size())); // cycle? - } - DCHECK(links > 0); - DCHECK(links == (header->loop_end() - header->rpo_number())); - DCHECK(end_found); - - // Check the contiguousness of loops. - int count = 0; - for (int j = 0; j < static_cast(order->size()); j++) { - BasicBlock* block = order->at(j); - DCHECK(block->rpo_number() == j); - if (j < header->rpo_number() || j >= header->loop_end()) { - DCHECK(!loop->members->Contains(block->id().ToInt())); - } else { - if (block == header) { - DCHECK(!loop->members->Contains(block->id().ToInt())); - } else { - DCHECK(loop->members->Contains(block->id().ToInt())); - } - count++; - } - } - DCHECK(links == count); - } -} -#endif // DEBUG - - -// Compute the special reverse-post-order block ordering, which is essentially -// a RPO of the graph where loop bodies are contiguous. Properties: -// 1. If block A is a predecessor of B, then A appears before B in the order, -// unless B is a loop header and A is in the loop headed at B -// (i.e. A -> B is a backedge). -// => If block A dominates block B, then A appears before B in the order. -// => If block A is a loop header, A appears before all blocks in the loop -// headed at A. -// 2. All loops are contiguous in the order (i.e. no intervening blocks that -// do not belong to the loop.) -// Note a simple RPO traversal satisfies (1) but not (3). -BasicBlockVector* Scheduler::ComputeSpecialRPO(ZonePool* zone_pool, - Schedule* schedule) { - ZonePool::Scope zone_scope(zone_pool); - Zone* zone = zone_scope.zone(); - Trace("--- COMPUTING SPECIAL RPO ----------------------------------\n"); - // RPO should not have been computed for this schedule yet. - CHECK_EQ(kBlockUnvisited1, schedule->start()->rpo_number()); - CHECK_EQ(0, static_cast(schedule->rpo_order_.size())); - - // Perform an iterative RPO traversal using an explicit stack, - // recording backedges that form cycles. O(|B|). - ZoneList > backedges(1, zone); - SpecialRPOStackFrame* stack = zone->NewArray( - static_cast(schedule->BasicBlockCount())); - BasicBlock* entry = schedule->start(); - BlockList* order = NULL; - int stack_depth = Push(stack, 0, entry, kBlockUnvisited1); - int num_loops = 0; - - while (stack_depth > 0) { - int current = stack_depth - 1; - SpecialRPOStackFrame* frame = stack + current; - - if (frame->index < frame->block->SuccessorCount()) { - // Process the next successor. - BasicBlock* succ = frame->block->SuccessorAt(frame->index++); - if (succ->rpo_number() == kBlockVisited1) continue; - if (succ->rpo_number() == kBlockOnStack) { - // The successor is on the stack, so this is a backedge (cycle). - backedges.Add( - std::pair(frame->block, frame->index - 1), - zone); - if (succ->loop_end() < 0) { - // Assign a new loop number to the header if it doesn't have one. - succ->set_loop_end(num_loops++); - } - } else { - // Push the successor onto the stack. - DCHECK(succ->rpo_number() == kBlockUnvisited1); - stack_depth = Push(stack, stack_depth, succ, kBlockUnvisited1); - } - } else { - // Finished with all successors; pop the stack and add the block. - order = order->Add(zone, frame->block); - frame->block->set_rpo_number(kBlockVisited1); - stack_depth--; - } - } - - // If no loops were encountered, then the order we computed was correct. - LoopInfo* loops = NULL; - if (num_loops != 0) { - // Otherwise, compute the loop information from the backedges in order - // to perform a traversal that groups loop bodies together. - loops = ComputeLoopInfo(zone, stack, num_loops, schedule->BasicBlockCount(), - &backedges); - - // Initialize the "loop stack". Note the entry could be a loop header. - LoopInfo* loop = entry->IsLoopHeader() ? &loops[entry->loop_end()] : NULL; - order = NULL; - - // Perform an iterative post-order traversal, visiting loop bodies before - // edges that lead out of loops. Visits each block once, but linking loop - // sections together is linear in the loop size, so overall is - // O(|B| + max(loop_depth) * max(|loop|)) - stack_depth = Push(stack, 0, entry, kBlockUnvisited2); - while (stack_depth > 0) { - SpecialRPOStackFrame* frame = stack + (stack_depth - 1); - BasicBlock* block = frame->block; - BasicBlock* succ = NULL; - - if (frame->index < block->SuccessorCount()) { - // Process the next normal successor. - succ = block->SuccessorAt(frame->index++); - } else if (block->IsLoopHeader()) { - // Process additional outgoing edges from the loop header. - if (block->rpo_number() == kBlockOnStack) { - // Finish the loop body the first time the header is left on the - // stack. - DCHECK(loop != NULL && loop->header == block); - loop->start = order->Add(zone, block); - order = loop->end; - block->set_rpo_number(kBlockVisited2); - // Pop the loop stack and continue visiting outgoing edges within the - // the context of the outer loop, if any. - loop = loop->prev; - // We leave the loop header on the stack; the rest of this iteration - // and later iterations will go through its outgoing edges list. - } - - // Use the next outgoing edge if there are any. - int outgoing_index = - static_cast(frame->index - block->SuccessorCount()); - LoopInfo* info = &loops[block->loop_end()]; - DCHECK(loop != info); - if (info->outgoing != NULL && - outgoing_index < info->outgoing->length()) { - succ = info->outgoing->at(outgoing_index); - frame->index++; - } - } - - if (succ != NULL) { - // Process the next successor. - if (succ->rpo_number() == kBlockOnStack) continue; - if (succ->rpo_number() == kBlockVisited2) continue; - DCHECK(succ->rpo_number() == kBlockUnvisited2); - if (loop != NULL && !loop->members->Contains(succ->id().ToInt())) { - // The successor is not in the current loop or any nested loop. - // Add it to the outgoing edges of this loop and visit it later. - loop->AddOutgoing(zone, succ); - } else { - // Push the successor onto the stack. - stack_depth = Push(stack, stack_depth, succ, kBlockUnvisited2); - if (succ->IsLoopHeader()) { - // Push the inner loop onto the loop stack. - DCHECK(succ->loop_end() >= 0 && succ->loop_end() < num_loops); - LoopInfo* next = &loops[succ->loop_end()]; - next->end = order; - next->prev = loop; - loop = next; - } - } - } else { - // Finished with all successors of the current block. - if (block->IsLoopHeader()) { - // If we are going to pop a loop header, then add its entire body. - LoopInfo* info = &loops[block->loop_end()]; - for (BlockList* l = info->start; true; l = l->next) { - if (l->next == info->end) { - l->next = order; - info->end = order; - break; - } - } - order = info->start; - } else { - // Pop a single node off the stack and add it to the order. - order = order->Add(zone, block); - block->set_rpo_number(kBlockVisited2); - } - stack_depth--; - } - } - } - - // Construct the final order from the list. - BasicBlockVector* final_order = &schedule->rpo_order_; - order->Serialize(final_order); - - // Compute the correct loop header for every block and set the correct loop - // ends. - LoopInfo* current_loop = NULL; - BasicBlock* current_header = NULL; - int loop_depth = 0; - for (BasicBlockVectorIter i = final_order->begin(); i != final_order->end(); - ++i) { - BasicBlock* current = *i; - current->set_loop_header(current_header); - if (current->IsLoopHeader()) { - loop_depth++; - current_loop = &loops[current->loop_end()]; - BlockList* end = current_loop->end; - current->set_loop_end(end == NULL ? static_cast(final_order->size()) - : end->block->rpo_number()); - current_header = current_loop->header; - Trace("B%d is a loop header, increment loop depth to %d\n", - current->id().ToInt(), loop_depth); - } else { - while (current_header != NULL && - current->rpo_number() >= current_header->loop_end()) { - DCHECK(current_header->IsLoopHeader()); - DCHECK(current_loop != NULL); - current_loop = current_loop->prev; - current_header = current_loop == NULL ? NULL : current_loop->header; - --loop_depth; - } - } - current->set_loop_depth(loop_depth); - if (current->loop_header() == NULL) { - Trace("B%d is not in a loop (depth == %d)\n", current->id().ToInt(), - current->loop_depth()); - } else { - Trace("B%d has loop header B%d, (depth == %d)\n", current->id().ToInt(), - current->loop_header()->id().ToInt(), current->loop_depth()); - } - } - - // Compute the assembly order (non-deferred code first, deferred code - // afterwards). - int32_t number = 0; - for (auto block : *final_order) { - if (block->deferred()) continue; - block->set_ao_number(number++); - } - for (auto block : *final_order) { - if (!block->deferred()) continue; - block->set_ao_number(number++); - } - -#if DEBUG - if (FLAG_trace_turbo_scheduler) PrintRPO(num_loops, loops, final_order); - VerifySpecialRPO(num_loops, loops, final_order); -#endif - return final_order; -} - } // namespace compiler } // namespace internal } // namespace v8 diff --git a/src/compiler/scheduler.h b/src/compiler/scheduler.h index afd5cf1ce0..a0addbdc2c 100644 --- a/src/compiler/scheduler.h +++ b/src/compiler/scheduler.h @@ -45,10 +45,10 @@ class Scheduler { Zone* zone_; Graph* graph_; Schedule* schedule_; - NodeVectorVector scheduled_nodes_; - NodeVector schedule_root_nodes_; - ZoneQueue schedule_queue_; - ZoneVector node_data_; + NodeVectorVector scheduled_nodes_; // Per-block list of nodes in reverse. + NodeVector schedule_root_nodes_; // Fixed root nodes seed the worklist. + ZoneQueue schedule_queue_; // Worklist of schedulable nodes. + ZoneVector node_data_; // Per-node data for all nodes. bool has_floating_control_; Scheduler(Zone* zone, Graph* graph, Schedule* schedule); @@ -64,20 +64,24 @@ class Scheduler { inline int GetRPONumber(BasicBlock* block); BasicBlock* GetCommonDominator(BasicBlock* b1, BasicBlock* b2); - // Phase 1: Build control-flow graph and dominator tree. + // Phase 1: Build control-flow graph. friend class CFGBuilder; void BuildCFG(); + + // Phase 2: Compute special RPO and dominator tree. + friend class SpecialRPONumberer; + void ComputeSpecialRPONumbering(); void GenerateImmediateDominatorTree(); - // Phase 2: Prepare use counts for nodes. + // Phase 3: Prepare use counts for nodes. friend class PrepareUsesVisitor; void PrepareUses(); - // Phase 3: Schedule nodes early. + // Phase 4: Schedule nodes early. friend class ScheduleEarlyNodeVisitor; void ScheduleEarly(); - // Phase 4: Schedule nodes late. + // Phase 5: Schedule nodes late. friend class ScheduleLateNodeVisitor; void ScheduleLate();