diff --git a/src/compiler/loop-analysis.cc b/src/compiler/loop-analysis.cc index d52c7c7742..bc3f94deaa 100644 --- a/src/compiler/loop-analysis.cc +++ b/src/compiler/loop-analysis.cc @@ -29,6 +29,7 @@ struct NodeInfo { struct LoopInfo { Node* header; NodeInfo* header_list; + NodeInfo* exit_list; NodeInfo* body_list; LoopTree::Loop* loop; }; @@ -81,9 +82,9 @@ class LoopFinderImpl { if (marked_forward && marked_backward) { PrintF("X"); } else if (marked_forward) { - PrintF("/"); + PrintF(">"); } else if (marked_backward) { - PrintF("\\"); + PrintF("<"); } else { PrintF(" "); } @@ -198,12 +199,22 @@ class LoopFinderImpl { if (merge->opcode() == IrOpcode::kLoop) { loop_num = CreateLoopInfo(merge); } + } else if (node->opcode() == IrOpcode::kLoopExit) { + // Intentionally ignore return value. Loop exit node marks + // are propagated normally. + CreateLoopInfo(node->InputAt(1)); + } else if (node->opcode() == IrOpcode::kLoopExitValue || + node->opcode() == IrOpcode::kLoopExitEffect) { + Node* loop_exit = NodeProperties::GetControlInput(node); + // Intentionally ignore return value. Loop exit node marks + // are propagated normally. + CreateLoopInfo(loop_exit->InputAt(1)); } // Propagate marks backwards from this node. for (int i = 0; i < node->InputCount(); i++) { Node* input = node->InputAt(i); - if (loop_num > 0 && i != kAssumedLoopEntryIndex) { + if (IsBackedge(node, i)) { // Only propagate the loop mark on backedges. if (SetBackwardMark(input, loop_num)) Queue(input); } else { @@ -216,6 +227,7 @@ class LoopFinderImpl { // Make a new loop if necessary for the given node. int CreateLoopInfo(Node* node) { + DCHECK_EQ(IrOpcode::kLoop, node->opcode()); int loop_num = LoopNum(node); if (loop_num > 0) return loop_num; @@ -223,21 +235,39 @@ class LoopFinderImpl { if (INDEX(loop_num) >= width_) ResizeBackwardMarks(); // Create a new loop. - loops_.push_back({node, nullptr, nullptr, nullptr}); + loops_.push_back({node, nullptr, nullptr, nullptr, nullptr}); loop_tree_->NewLoop(); + SetLoopMarkForLoopHeader(node, loop_num); + return loop_num; + } + + void SetLoopMark(Node* node, int loop_num) { + info(node); // create the NodeInfo SetBackwardMark(node, loop_num); loop_tree_->node_to_loop_num_[node->id()] = loop_num; + } - // Setup loop mark for phis attached to loop header. + void SetLoopMarkForLoopHeader(Node* node, int loop_num) { + DCHECK_EQ(IrOpcode::kLoop, node->opcode()); + SetLoopMark(node, loop_num); for (Node* use : node->uses()) { if (NodeProperties::IsPhi(use)) { - info(use); // create the NodeInfo - SetBackwardMark(use, loop_num); - loop_tree_->node_to_loop_num_[use->id()] = loop_num; + SetLoopMark(use, loop_num); + } + + // Do not keep the loop alive if it does not have any backedges. + if (node->InputCount() <= 1) continue; + + if (use->opcode() == IrOpcode::kLoopExit) { + SetLoopMark(use, loop_num); + for (Node* exit_use : use->uses()) { + if (exit_use->opcode() == IrOpcode::kLoopExitValue || + exit_use->opcode() == IrOpcode::kLoopExitEffect) { + SetLoopMark(exit_use, loop_num); + } + } } } - - return loop_num; } void ResizeBackwardMarks() { @@ -276,20 +306,33 @@ class LoopFinderImpl { queued_.Set(node, false); for (Edge edge : node->use_edges()) { Node* use = edge.from(); - if (!IsBackedge(use, edge)) { + if (!IsBackedge(use, edge.index())) { if (PropagateForwardMarks(node, use)) Queue(use); } } } } - bool IsBackedge(Node* use, Edge& edge) { + bool IsLoopHeaderNode(Node* node) { + return node->opcode() == IrOpcode::kLoop || NodeProperties::IsPhi(node); + } + + bool IsLoopExitNode(Node* node) { + return node->opcode() == IrOpcode::kLoopExit || + node->opcode() == IrOpcode::kLoopExitValue || + node->opcode() == IrOpcode::kLoopExitEffect; + } + + bool IsBackedge(Node* use, int index) { if (LoopNum(use) <= 0) return false; - if (edge.index() == kAssumedLoopEntryIndex) return false; if (NodeProperties::IsPhi(use)) { - return !NodeProperties::IsControlEdge(edge); + return index != NodeProperties::FirstControlIndex(use) && + index != kAssumedLoopEntryIndex; + } else if (use->opcode() == IrOpcode::kLoop) { + return index != kAssumedLoopEntryIndex; } - return true; + DCHECK(IsLoopExitNode(use)); + return false; } int LoopNum(Node* node) { return loop_tree_->node_to_loop_num_[node->id()]; } @@ -307,6 +350,22 @@ class LoopFinderImpl { } } + void AddNodeToLoop(NodeInfo* node_info, LoopInfo* loop, int loop_num) { + if (LoopNum(node_info->node) == loop_num) { + if (IsLoopHeaderNode(node_info->node)) { + node_info->next = loop->header_list; + loop->header_list = node_info; + } else { + DCHECK(IsLoopExitNode(node_info->node)); + node_info->next = loop->exit_list; + loop->exit_list = node_info; + } + } else { + node_info->next = loop->body_list; + loop->body_list = node_info; + } + } + void FinishLoopTree() { DCHECK(loops_found_ == static_cast(loops_.size())); DCHECK(loops_found_ == static_cast(loop_tree_->all_loops_.size())); @@ -342,13 +401,7 @@ class LoopFinderImpl { } } if (innermost == nullptr) continue; - if (LoopNum(ni.node) == innermost_index) { - ni.next = innermost->header_list; - innermost->header_list = ∋ - } else { - ni.next = innermost->body_list; - innermost->body_list = ∋ - } + AddNodeToLoop(&ni, innermost, innermost_index); count++; } @@ -368,13 +421,7 @@ class LoopFinderImpl { size_t count = 0; for (NodeInfo& ni : info_) { if (ni.node == nullptr || !IsInLoop(ni.node, 1)) continue; - if (LoopNum(ni.node) == 1) { - ni.next = li->header_list; - li->header_list = ∋ - } else { - ni.next = li->body_list; - li->body_list = ∋ - } + AddNodeToLoop(&ni, li, 1); count++; } @@ -406,7 +453,14 @@ class LoopFinderImpl { // Serialize nested loops. for (LoopTree::Loop* child : loop->children_) SerializeLoop(child); - loop->body_end_ = static_cast(loop_tree_->loop_nodes_.size()); + // Serialize the exits. + loop->exits_start_ = static_cast(loop_tree_->loop_nodes_.size()); + for (NodeInfo* ni = li.exit_list; ni != nullptr; ni = ni->next) { + loop_tree_->loop_nodes_.push_back(ni->node); + loop_tree_->node_to_loop_num_[ni->node->id()] = loop_num; + } + + loop->exits_end_ = static_cast(loop_tree_->loop_nodes_.size()); } // Connect the LoopTree loops to their parents recursively. @@ -438,9 +492,12 @@ class LoopFinderImpl { while (i < loop->body_start_) { PrintF(" H#%d", loop_tree_->loop_nodes_[i++]->id()); } - while (i < loop->body_end_) { + while (i < loop->exits_start_) { PrintF(" B#%d", loop_tree_->loop_nodes_[i++]->id()); } + while (i < loop->exits_end_) { + PrintF(" E#%d", loop_tree_->loop_nodes_[i++]->id()); + } PrintF("\n"); for (LoopTree::Loop* child : loop->children_) PrintLoop(child); } diff --git a/src/compiler/loop-analysis.h b/src/compiler/loop-analysis.h index b8bc395acb..a8c3bca7d7 100644 --- a/src/compiler/loop-analysis.h +++ b/src/compiler/loop-analysis.h @@ -38,8 +38,9 @@ class LoopTree : public ZoneObject { Loop* parent() const { return parent_; } const ZoneVector& children() const { return children_; } size_t HeaderSize() const { return body_start_ - header_start_; } - size_t BodySize() const { return body_end_ - body_start_; } - size_t TotalSize() const { return body_end_ - header_start_; } + size_t BodySize() const { return exits_start_ - body_start_; } + size_t ExitsSize() const { return exits_end_ - exits_start_; } + size_t TotalSize() const { return exits_end_ - header_start_; } size_t depth() const { return static_cast(depth_); } private: @@ -52,13 +53,15 @@ class LoopTree : public ZoneObject { children_(zone), header_start_(-1), body_start_(-1), - body_end_(-1) {} + exits_start_(-1), + exits_end_(-1) {} Loop* parent_; int depth_; ZoneVector children_; int header_start_; int body_start_; - int body_end_; + int exits_start_; + int exits_end_; }; // Return the innermost nested loop, if any, that contains {node}. @@ -97,13 +100,19 @@ class LoopTree : public ZoneObject { // Return a range which can iterate over the body nodes of {loop}. NodeRange BodyNodes(Loop* loop) { return NodeRange(&loop_nodes_[0] + loop->body_start_, - &loop_nodes_[0] + loop->body_end_); + &loop_nodes_[0] + loop->exits_start_); + } + + // Return a range which can iterate over the body nodes of {loop}. + NodeRange ExitNodes(Loop* loop) { + return NodeRange(&loop_nodes_[0] + loop->exits_start_, + &loop_nodes_[0] + loop->exits_end_); } // Return a range which can iterate over the nodes of {loop}. NodeRange LoopNodes(Loop* loop) { return NodeRange(&loop_nodes_[0] + loop->header_start_, - &loop_nodes_[0] + loop->body_end_); + &loop_nodes_[0] + loop->exits_end_); } // Return the node that represents the control, i.e. the loop node itself. diff --git a/src/compiler/loop-peeling.cc b/src/compiler/loop-peeling.cc index a09f5060e0..d48e9fbc47 100644 --- a/src/compiler/loop-peeling.cc +++ b/src/compiler/loop-peeling.cc @@ -126,8 +126,14 @@ struct Peeling { // Copy all the nodes first. for (Node* node : nodes) { inputs.clear(); - for (Node* input : node->inputs()) inputs.push_back(map(input)); - Insert(node, graph->NewNode(node->op(), node->InputCount(), &inputs[0])); + for (Node* input : node->inputs()) { + inputs.push_back(map(input)); + } + Node* copy = graph->NewNode(node->op(), node->InputCount(), &inputs[0]); + if (NodeProperties::IsTyped(node)) { + NodeProperties::SetType(copy, NodeProperties::GetType(node)); + } + Insert(node, copy); } // Fix remaining inputs of the copies. @@ -160,56 +166,54 @@ Node* PeeledIteration::map(Node* node) { return node; } - -static void FindLoopExits(LoopTree* loop_tree, LoopTree::Loop* loop, - NodeVector& exits, NodeVector& rets) { +bool LoopPeeler::CanPeel(LoopTree* loop_tree, LoopTree::Loop* loop) { // Look for returns and if projections that are outside the loop but whose // control input is inside the loop. + Node* loop_node = loop_tree->GetLoopControl(loop); for (Node* node : loop_tree->LoopNodes(loop)) { for (Node* use : node->uses()) { if (!loop_tree->Contains(loop, use)) { - if (IrOpcode::IsIfProjectionOpcode(use->opcode())) { - // This is a branch from inside the loop to outside the loop. - exits.push_back(use); - } else if (use->opcode() == IrOpcode::kReturn && - loop_tree->Contains(loop, - NodeProperties::GetControlInput(use))) { - // This is a return from inside the loop. - rets.push_back(use); + bool unmarked_exit; + switch (node->opcode()) { + case IrOpcode::kLoopExit: + unmarked_exit = (node->InputAt(1) != loop_node); + break; + case IrOpcode::kLoopExitValue: + case IrOpcode::kLoopExitEffect: + unmarked_exit = (node->InputAt(1)->InputAt(1) != loop_node); + break; + default: + unmarked_exit = (use->opcode() != IrOpcode::kTerminate); + } + if (unmarked_exit) { + if (FLAG_trace_turbo_graph) { + Node* loop_node = loop_tree->GetLoopControl(loop); + PrintF( + "Cannot peel loop %i. Loop exit without explicit mark: Node %i " + "(%s) is inside " + "loop, but its use %i (%s) is outside.\n", + loop_node->id(), node->id(), node->op()->mnemonic(), use->id(), + use->op()->mnemonic()); + } + return false; } } } } -} - - -bool LoopPeeler::CanPeel(LoopTree* loop_tree, LoopTree::Loop* loop) { - Zone zone(loop_tree->zone()->allocator()); - NodeVector exits(&zone); - NodeVector rets(&zone); - FindLoopExits(loop_tree, loop, exits, rets); - return exits.size() <= 1u; + return true; } PeeledIteration* LoopPeeler::Peel(Graph* graph, CommonOperatorBuilder* common, LoopTree* loop_tree, LoopTree::Loop* loop, Zone* tmp_zone) { - //============================================================================ - // Find the loop exit region to determine if this loop can be peeled. - //============================================================================ - NodeVector exits(tmp_zone); - NodeVector rets(tmp_zone); - FindLoopExits(loop_tree, loop, exits, rets); - - if (exits.size() != 1) return nullptr; // not peelable currently. + if (!CanPeel(loop_tree, loop)) return nullptr; //============================================================================ // Construct the peeled iteration. //============================================================================ PeeledIterationImpl* iter = new (tmp_zone) PeeledIterationImpl(tmp_zone); - size_t estimated_peeled_size = - 5 + (loop->TotalSize() + exits.size() + rets.size()) * 2; + size_t estimated_peeled_size = 5 + (loop->TotalSize()) * 2; Peeling peeling(graph, tmp_zone, estimated_peeled_size, &iter->node_pairs_); Node* dead = graph->NewNode(common->Dead()); @@ -260,77 +264,64 @@ PeeledIteration* LoopPeeler::Peel(Graph* graph, CommonOperatorBuilder* common, // Only one backedge, simply replace the input to loop with output of // peeling. for (Node* node : loop_tree->HeaderNodes(loop)) { - node->ReplaceInput(0, peeling.map(node->InputAt(0))); + node->ReplaceInput(0, peeling.map(node->InputAt(1))); } new_entry = peeling.map(loop_node->InputAt(1)); } loop_node->ReplaceInput(0, new_entry); //============================================================================ - // Duplicate the loop exit region and add a merge. + // Change the exit and exit markers to merge/phi/effect-phi. //============================================================================ - - // Currently we are limited to peeling loops with a single exit. The exit is - // the postdominator of the loop (ignoring returns). - Node* postdom = exits[0]; - for (Node* node : rets) exits.push_back(node); - for (Node* use : postdom->uses()) { - if (NodeProperties::IsPhi(use)) exits.push_back(use); - } - - NodeRange exit_range(&exits[0], &exits[0] + exits.size()); - peeling.CopyNodes(graph, tmp_zone, dead, exit_range); - - Node* merge = graph->NewNode(common->Merge(2), postdom, peeling.map(postdom)); - postdom->ReplaceUses(merge); - merge->ReplaceInput(0, postdom); // input 0 overwritten by above line. - - // Find and update all the edges into either the loop or exit region. - for (int i = 0; i < 2; i++) { - NodeRange range = i == 0 ? loop_tree->LoopNodes(loop) : exit_range; - ZoneVector value_edges(tmp_zone); - ZoneVector effect_edges(tmp_zone); - - for (Node* node : range) { - // Gather value and effect edges from outside the region. - for (Edge edge : node->use_edges()) { - if (!peeling.Marked(edge.from())) { - // Edge from outside the loop into the region. - if (NodeProperties::IsValueEdge(edge) || - NodeProperties::IsContextEdge(edge)) { - value_edges.push_back(edge); - } else if (NodeProperties::IsEffectEdge(edge)) { - effect_edges.push_back(edge); - } else { - // don't do anything for control edges. - // TODO(titzer): should update control edges to peeled? - } - } - } - - // Update all the value and effect edges at once. - if (!value_edges.empty()) { - // TODO(titzer): machine type is wrong here. - Node* phi = - graph->NewNode(common->Phi(MachineRepresentation::kTagged, 2), node, - peeling.map(node), merge); - for (Edge edge : value_edges) edge.UpdateTo(phi); - value_edges.clear(); - } - if (!effect_edges.empty()) { - Node* effect_phi = graph->NewNode(common->EffectPhi(2), node, - peeling.map(node), merge); - for (Edge edge : effect_edges) edge.UpdateTo(effect_phi); - effect_edges.clear(); - } + for (Node* exit : loop_tree->ExitNodes(loop)) { + switch (exit->opcode()) { + case IrOpcode::kLoopExit: + // Change the loop exit node to a merge node. + exit->ReplaceInput(1, peeling.map(exit->InputAt(0))); + NodeProperties::ChangeOp(exit, common->Merge(2)); + break; + case IrOpcode::kLoopExitValue: + // Change exit marker to phi. + exit->InsertInput(graph->zone(), 1, peeling.map(exit->InputAt(0))); + NodeProperties::ChangeOp( + exit, common->Phi(MachineRepresentation::kTagged, 2)); + break; + case IrOpcode::kLoopExitEffect: + // Change effect exit marker to effect phi. + exit->InsertInput(graph->zone(), 1, peeling.map(exit->InputAt(0))); + NodeProperties::ChangeOp(exit, common->EffectPhi(2)); + break; + default: + break; } } - return iter; } namespace { +void PeelInnerLoops(Graph* graph, CommonOperatorBuilder* common, + LoopTree* loop_tree, LoopTree::Loop* loop, + Zone* temp_zone) { + // If the loop has nested loops, peel inside those. + if (!loop->children().empty()) { + for (LoopTree::Loop* inner_loop : loop->children()) { + PeelInnerLoops(graph, common, loop_tree, inner_loop, temp_zone); + } + return; + } + // Only peel small-enough loops. + if (loop->TotalSize() > LoopPeeler::kMaxPeeledNodes) return; + if (FLAG_trace_turbo_graph) { + PrintF("Peeling loop with header: "); + for (Node* node : loop_tree->HeaderNodes(loop)) { + PrintF("%i ", node->id()); + } + } + + LoopPeeler::Peel(graph, common, loop_tree, loop, temp_zone); +} + void EliminateLoopExit(Node* node) { DCHECK_EQ(IrOpcode::kLoopExit, node->opcode()); // The exit markers take the loop exit as input. We iterate over uses @@ -355,6 +346,17 @@ void EliminateLoopExit(Node* node) { } // namespace +// static +void LoopPeeler::PeelInnerLoopsOfTree(Graph* graph, + CommonOperatorBuilder* common, + LoopTree* loop_tree, Zone* temp_zone) { + for (LoopTree::Loop* loop : loop_tree->outer_loops()) { + PeelInnerLoops(graph, common, loop_tree, loop, temp_zone); + } + + EliminateLoopExits(graph, temp_zone); +} + // static void LoopPeeler::EliminateLoopExits(Graph* graph, Zone* temp_zone) { ZoneQueue queue(temp_zone); diff --git a/src/compiler/loop-peeling.h b/src/compiler/loop-peeling.h index 3ee20743bd..8b38e2575c 100644 --- a/src/compiler/loop-peeling.h +++ b/src/compiler/loop-peeling.h @@ -33,7 +33,11 @@ class LoopPeeler { static PeeledIteration* Peel(Graph* graph, CommonOperatorBuilder* common, LoopTree* loop_tree, LoopTree::Loop* loop, Zone* tmp_zone); + static void PeelInnerLoopsOfTree(Graph* graph, CommonOperatorBuilder* common, + LoopTree* loop_tree, Zone* tmp_zone); + static void EliminateLoopExits(Graph* graph, Zone* temp_zone); + static const size_t kMaxPeeledNodes = 1000; }; diff --git a/src/compiler/pipeline.cc b/src/compiler/pipeline.cc index faa295f16f..9683a83d68 100644 --- a/src/compiler/pipeline.cc +++ b/src/compiler/pipeline.cc @@ -942,6 +942,22 @@ struct RepresentationSelectionPhase { } }; +struct LoopPeelingPhase { + static const char* phase_name() { return "loop peeling"; } + + void Run(PipelineData* data, Zone* temp_zone) { + GraphTrimmer trimmer(temp_zone, data->graph()); + NodeVector roots(temp_zone); + data->jsgraph()->GetCachedNodes(&roots); + trimmer.TrimGraph(roots.begin(), roots.end()); + + LoopTree* loop_tree = + LoopFinder::BuildLoopTree(data->jsgraph()->graph(), temp_zone); + LoopPeeler::PeelInnerLoopsOfTree(data->graph(), data->common(), loop_tree, + temp_zone); + } +}; + struct LoopExitEliminationPhase { static const char* phase_name() { return "loop exit elimination"; } @@ -1438,9 +1454,13 @@ bool PipelineImpl::CreateGraph() { Run(); RunPrintAndVerify("Lowered typed"); - // Eventually, loop peeling will be done here. - Run(); - RunPrintAndVerify("Loop exits eliminated", true); + if (FLAG_turbo_loop_peeling) { + Run(); + RunPrintAndVerify("Loops peeled", true); + } else { + Run(); + RunPrintAndVerify("Loop exits eliminated", true); + } if (FLAG_turbo_stress_loop_peeling) { Run(); diff --git a/src/flag-definitions.h b/src/flag-definitions.h index 07d874bd4e..2ff7ee46cc 100644 --- a/src/flag-definitions.h +++ b/src/flag-definitions.h @@ -473,6 +473,7 @@ DEFINE_BOOL(turbo_move_optimization, true, "optimize gap moves in TurboFan") DEFINE_BOOL(turbo_jt, true, "enable jump threading in TurboFan") DEFINE_BOOL(turbo_stress_loop_peeling, false, "stress loop peeling optimization") +DEFINE_BOOL(turbo_loop_peeling, false, "Turbofan loop peeling") DEFINE_BOOL(turbo_cf_optimization, true, "optimize control flow in TurboFan") DEFINE_BOOL(turbo_frame_elision, true, "elide frames in TurboFan") DEFINE_BOOL(turbo_cache_shared_code, true, "cache context-independent code") diff --git a/test/unittests/compiler/loop-peeling-unittest.cc b/test/unittests/compiler/loop-peeling-unittest.cc index 9db490560d..56691fdeef 100644 --- a/test/unittests/compiler/loop-peeling-unittest.cc +++ b/test/unittests/compiler/loop-peeling-unittest.cc @@ -28,6 +28,7 @@ struct While { Node* loop; Node* branch; Node* if_true; + Node* if_false; Node* exit; }; @@ -46,6 +47,7 @@ struct Counter { Node* inc; Node* phi; Node* add; + Node* exit_marker; }; @@ -105,12 +107,14 @@ class LoopPeelingTest : public GraphTest { While NewWhile(Node* cond, Node* control = nullptr) { if (control == nullptr) control = start(); - Node* loop = graph()->NewNode(common()->Loop(2), control, control); - Node* branch = graph()->NewNode(common()->Branch(), cond, loop); - Node* if_true = graph()->NewNode(common()->IfTrue(), branch); - Node* exit = graph()->NewNode(common()->IfFalse(), branch); - loop->ReplaceInput(1, if_true); - return {loop, branch, if_true, exit}; + While w; + w.loop = graph()->NewNode(common()->Loop(2), control, control); + w.branch = graph()->NewNode(common()->Branch(), cond, w.loop); + w.if_true = graph()->NewNode(common()->IfTrue(), w.branch); + w.if_false = graph()->NewNode(common()->IfFalse(), w.branch); + w.exit = graph()->NewNode(common()->LoopExit(), w.if_false, w.loop); + w.loop->ReplaceInput(1, w.if_true); + return w; } void Chain(While* a, Node* control) { a->loop->ReplaceInput(0, control); } @@ -124,21 +128,24 @@ class LoopPeelingTest : public GraphTest { } Branch NewBranch(Node* cond, Node* control = nullptr) { + Branch b; if (control == nullptr) control = start(); - Node* branch = graph()->NewNode(common()->Branch(), cond, control); - Node* if_true = graph()->NewNode(common()->IfTrue(), branch); - Node* if_false = graph()->NewNode(common()->IfFalse(), branch); - return {branch, if_true, if_false}; + b.branch = graph()->NewNode(common()->Branch(), cond, control); + b.if_true = graph()->NewNode(common()->IfTrue(), b.branch); + b.if_false = graph()->NewNode(common()->IfFalse(), b.branch); + return b; } Counter NewCounter(While* w, int32_t b, int32_t k) { - Node* base = Int32Constant(b); - Node* inc = Int32Constant(k); - Node* phi = graph()->NewNode( - common()->Phi(MachineRepresentation::kTagged, 2), base, base, w->loop); - Node* add = graph()->NewNode(machine()->Int32Add(), phi, inc); - phi->ReplaceInput(1, add); - return {base, inc, phi, add}; + Counter c; + c.base = Int32Constant(b); + c.inc = Int32Constant(k); + c.phi = graph()->NewNode(common()->Phi(MachineRepresentation::kTagged, 2), + c.base, c.base, w->loop); + c.add = graph()->NewNode(machine()->Int32Add(), c.phi, c.inc); + c.phi->ReplaceInput(1, c.add); + c.exit_marker = graph()->NewNode(common()->LoopExitValue(), c.phi, w->exit); + return c; } }; @@ -152,14 +159,14 @@ TEST_F(LoopPeelingTest, SimpleLoop) { Node* br1 = ExpectPeeled(w.branch, peeled); Node* if_true1 = ExpectPeeled(w.if_true, peeled); - Node* if_false1 = ExpectPeeled(w.exit, peeled); + Node* if_false1 = ExpectPeeled(w.if_false, peeled); EXPECT_THAT(br1, IsBranch(p0, start())); EXPECT_THAT(if_true1, IsIfTrue(br1)); EXPECT_THAT(if_false1, IsIfFalse(br1)); EXPECT_THAT(w.loop, IsLoop(if_true1, w.if_true)); - EXPECT_THAT(r, IsReturn(p0, start(), IsMerge(w.exit, if_false1))); + EXPECT_THAT(r, IsReturn(p0, start(), IsMerge(w.if_false, if_false1))); } @@ -167,13 +174,13 @@ TEST_F(LoopPeelingTest, SimpleLoopWithCounter) { Node* p0 = Parameter(0); While w = NewWhile(p0); Counter c = NewCounter(&w, 0, 1); - Node* r = InsertReturn(c.phi, start(), w.exit); + Node* r = InsertReturn(c.exit_marker, start(), w.exit); PeeledIteration* peeled = PeelOne(); Node* br1 = ExpectPeeled(w.branch, peeled); Node* if_true1 = ExpectPeeled(w.if_true, peeled); - Node* if_false1 = ExpectPeeled(w.exit, peeled); + Node* if_false1 = ExpectPeeled(w.if_false, peeled); EXPECT_THAT(br1, IsBranch(p0, start())); EXPECT_THAT(if_true1, IsIfTrue(br1)); @@ -182,11 +189,10 @@ TEST_F(LoopPeelingTest, SimpleLoopWithCounter) { EXPECT_THAT(peeled->map(c.add), IsInt32Add(c.base, c.inc)); - Capture merge; + EXPECT_THAT(w.exit, IsMerge(w.if_false, if_false1)); EXPECT_THAT( - r, IsReturn(IsPhi(MachineRepresentation::kTagged, c.phi, c.base, - AllOf(CaptureEq(&merge), IsMerge(w.exit, if_false1))), - start(), CaptureEq(&merge))); + r, IsReturn(IsPhi(MachineRepresentation::kTagged, c.phi, c.base, w.exit), + start(), w.exit)); } @@ -197,13 +203,13 @@ TEST_F(LoopPeelingTest, SimpleNestedLoopWithCounter_peel_outer) { Nest(&inner, &outer); Counter c = NewCounter(&outer, 0, 1); - Node* r = InsertReturn(c.phi, start(), outer.exit); + Node* r = InsertReturn(c.exit_marker, start(), outer.exit); PeeledIteration* peeled = PeelOne(); Node* bro = ExpectPeeled(outer.branch, peeled); Node* if_trueo = ExpectPeeled(outer.if_true, peeled); - Node* if_falseo = ExpectPeeled(outer.exit, peeled); + Node* if_falseo = ExpectPeeled(outer.if_false, peeled); EXPECT_THAT(bro, IsBranch(p0, start())); EXPECT_THAT(if_trueo, IsIfTrue(bro)); @@ -211,21 +217,21 @@ TEST_F(LoopPeelingTest, SimpleNestedLoopWithCounter_peel_outer) { Node* bri = ExpectPeeled(inner.branch, peeled); Node* if_truei = ExpectPeeled(inner.if_true, peeled); - Node* if_falsei = ExpectPeeled(inner.exit, peeled); + Node* if_falsei = ExpectPeeled(inner.if_false, peeled); + Node* exiti = ExpectPeeled(inner.exit, peeled); EXPECT_THAT(bri, IsBranch(p0, ExpectPeeled(inner.loop, peeled))); EXPECT_THAT(if_truei, IsIfTrue(bri)); EXPECT_THAT(if_falsei, IsIfFalse(bri)); - EXPECT_THAT(outer.loop, IsLoop(if_falsei, inner.exit)); + EXPECT_THAT(outer.loop, IsLoop(exiti, inner.exit)); EXPECT_THAT(peeled->map(c.add), IsInt32Add(c.base, c.inc)); Capture merge; - EXPECT_THAT( - r, - IsReturn(IsPhi(MachineRepresentation::kTagged, c.phi, c.base, - AllOf(CaptureEq(&merge), IsMerge(outer.exit, if_falseo))), - start(), CaptureEq(&merge))); + EXPECT_THAT(outer.exit, IsMerge(outer.if_false, if_falseo)); + EXPECT_THAT(r, IsReturn(IsPhi(MachineRepresentation::kTagged, c.phi, c.base, + outer.exit), + start(), outer.exit)); } @@ -236,7 +242,7 @@ TEST_F(LoopPeelingTest, SimpleNestedLoopWithCounter_peel_inner) { Nest(&inner, &outer); Counter c = NewCounter(&outer, 0, 1); - Node* r = InsertReturn(c.phi, start(), outer.exit); + Node* r = InsertReturn(c.exit_marker, start(), outer.exit); LoopTree* loop_tree = GetLoopTree(); LoopTree::Loop* loop = loop_tree->ContainingLoop(inner.loop); @@ -248,20 +254,22 @@ TEST_F(LoopPeelingTest, SimpleNestedLoopWithCounter_peel_inner) { ExpectNotPeeled(outer.loop, peeled); ExpectNotPeeled(outer.branch, peeled); ExpectNotPeeled(outer.if_true, peeled); + ExpectNotPeeled(outer.if_false, peeled); ExpectNotPeeled(outer.exit, peeled); Node* bri = ExpectPeeled(inner.branch, peeled); Node* if_truei = ExpectPeeled(inner.if_true, peeled); - Node* if_falsei = ExpectPeeled(inner.exit, peeled); + Node* if_falsei = ExpectPeeled(inner.if_false, peeled); EXPECT_THAT(bri, IsBranch(p0, ExpectPeeled(inner.loop, peeled))); EXPECT_THAT(if_truei, IsIfTrue(bri)); EXPECT_THAT(if_falsei, IsIfFalse(bri)); - EXPECT_THAT(outer.loop, IsLoop(start(), IsMerge(inner.exit, if_falsei))); + EXPECT_THAT(inner.exit, IsMerge(inner.if_false, if_falsei)); + EXPECT_THAT(outer.loop, IsLoop(start(), inner.exit)); ExpectNotPeeled(c.add, peeled); - EXPECT_THAT(r, IsReturn(c.phi, start(), outer.exit)); + EXPECT_THAT(r, IsReturn(c.exit_marker, start(), outer.exit)); } @@ -271,7 +279,7 @@ TEST_F(LoopPeelingTest, SimpleInnerCounter_peel_inner) { While inner = NewWhile(p0); Nest(&inner, &outer); Counter c = NewCounter(&inner, 0, 1); - Node* phi = NewPhi(&outer, Int32Constant(11), c.phi); + Node* phi = NewPhi(&outer, Int32Constant(11), c.exit_marker); Node* r = InsertReturn(phi, start(), outer.exit); @@ -285,25 +293,26 @@ TEST_F(LoopPeelingTest, SimpleInnerCounter_peel_inner) { ExpectNotPeeled(outer.loop, peeled); ExpectNotPeeled(outer.branch, peeled); ExpectNotPeeled(outer.if_true, peeled); + ExpectNotPeeled(outer.if_false, peeled); ExpectNotPeeled(outer.exit, peeled); Node* bri = ExpectPeeled(inner.branch, peeled); Node* if_truei = ExpectPeeled(inner.if_true, peeled); - Node* if_falsei = ExpectPeeled(inner.exit, peeled); + Node* if_falsei = ExpectPeeled(inner.if_false, peeled); EXPECT_THAT(bri, IsBranch(p0, ExpectPeeled(inner.loop, peeled))); EXPECT_THAT(if_truei, IsIfTrue(bri)); EXPECT_THAT(if_falsei, IsIfFalse(bri)); - EXPECT_THAT(outer.loop, IsLoop(start(), IsMerge(inner.exit, if_falsei))); + EXPECT_THAT(inner.exit, IsMerge(inner.if_false, if_falsei)); + EXPECT_THAT(outer.loop, IsLoop(start(), inner.exit)); EXPECT_THAT(peeled->map(c.add), IsInt32Add(c.base, c.inc)); - Node* back = phi->InputAt(1); - EXPECT_THAT(back, IsPhi(MachineRepresentation::kTagged, c.phi, c.base, - IsMerge(inner.exit, if_falsei))); + EXPECT_THAT(c.exit_marker, + IsPhi(MachineRepresentation::kTagged, c.phi, c.base, inner.exit)); EXPECT_THAT(phi, IsPhi(MachineRepresentation::kTagged, IsInt32Constant(11), - back, outer.loop)); + c.exit_marker, outer.loop)); EXPECT_THAT(r, IsReturn(phi, start(), outer.exit)); } @@ -318,7 +327,9 @@ TEST_F(LoopPeelingTest, TwoBackedgeLoop) { loop->ReplaceInput(1, b2.if_true); loop->ReplaceInput(2, b2.if_false); - Node* r = InsertReturn(p0, start(), b1.if_false); + Node* exit = graph()->NewNode(common()->LoopExit(), b1.if_false, loop); + + Node* r = InsertReturn(p0, start(), exit); PeeledIteration* peeled = PeelOne(); @@ -339,7 +350,8 @@ TEST_F(LoopPeelingTest, TwoBackedgeLoop) { EXPECT_THAT(b2f, IsIfFalse(b2b)); EXPECT_THAT(loop, IsLoop(IsMerge(b2t, b2f), b2.if_true, b2.if_false)); - EXPECT_THAT(r, IsReturn(p0, start(), IsMerge(b1.if_false, b1f))); + EXPECT_THAT(exit, IsMerge(b1.if_false, b1f)); + EXPECT_THAT(r, IsReturn(p0, start(), exit)); } @@ -355,7 +367,9 @@ TEST_F(LoopPeelingTest, TwoBackedgeLoopWithPhi) { loop->ReplaceInput(1, b2.if_true); loop->ReplaceInput(2, b2.if_false); - Node* r = InsertReturn(phi, start(), b1.if_false); + Node* exit = graph()->NewNode(common()->LoopExit(), b1.if_false, loop); + Node* exit_marker = graph()->NewNode(common()->LoopExitValue(), phi, exit); + Node* r = InsertReturn(exit_marker, start(), exit); PeeledIteration* peeled = PeelOne(); @@ -383,11 +397,10 @@ TEST_F(LoopPeelingTest, TwoBackedgeLoopWithPhi) { IsInt32Constant(2), IsMerge(b2t, b2f)), IsInt32Constant(1), IsInt32Constant(2), loop)); - Capture merge; - EXPECT_THAT( - r, IsReturn(IsPhi(MachineRepresentation::kTagged, phi, IsInt32Constant(0), - AllOf(CaptureEq(&merge), IsMerge(b1.if_false, b1f))), - start(), CaptureEq(&merge))); + EXPECT_THAT(exit, IsMerge(b1.if_false, b1f)); + EXPECT_THAT(exit_marker, IsPhi(MachineRepresentation::kTagged, phi, + IsInt32Constant(0), exit)); + EXPECT_THAT(r, IsReturn(exit_marker, start(), exit)); } @@ -408,7 +421,9 @@ TEST_F(LoopPeelingTest, TwoBackedgeLoopWithCounter) { loop->ReplaceInput(1, b2.if_true); loop->ReplaceInput(2, b2.if_false); - Node* r = InsertReturn(phi, start(), b1.if_false); + Node* exit = graph()->NewNode(common()->LoopExit(), b1.if_false, loop); + Node* exit_marker = graph()->NewNode(common()->LoopExitValue(), phi, exit); + Node* r = InsertReturn(exit_marker, start(), exit); PeeledIteration* peeled = PeelOne(); @@ -443,49 +458,60 @@ TEST_F(LoopPeelingTest, TwoBackedgeLoopWithCounter) { IsInt32Add(phi, IsInt32Constant(1)), IsInt32Add(phi, IsInt32Constant(2)), loop)); - Capture merge; - EXPECT_THAT( - r, IsReturn(IsPhi(MachineRepresentation::kTagged, phi, IsInt32Constant(0), - AllOf(CaptureEq(&merge), IsMerge(b1.if_false, b1f))), - start(), CaptureEq(&merge))); + EXPECT_THAT(exit, IsMerge(b1.if_false, b1f)); + EXPECT_THAT(exit_marker, IsPhi(MachineRepresentation::kTagged, phi, + IsInt32Constant(0), exit)); + EXPECT_THAT(r, IsReturn(exit_marker, start(), exit)); } - -TEST_F(LoopPeelingTest, TwoExitLoop_nope) { +TEST_F(LoopPeelingTest, TwoExitLoop) { Node* p0 = Parameter(0); Node* loop = graph()->NewNode(common()->Loop(2), start(), start()); Branch b1 = NewBranch(p0, loop); Branch b2 = NewBranch(p0, b1.if_true); loop->ReplaceInput(1, b2.if_true); - Node* merge = graph()->NewNode(common()->Merge(2), b1.if_false, b2.if_false); - InsertReturn(p0, start(), merge); - { - LoopTree* loop_tree = GetLoopTree(); - LoopTree::Loop* loop = loop_tree->outer_loops()[0]; - EXPECT_FALSE(LoopPeeler::CanPeel(loop_tree, loop)); - } + Node* exit1 = graph()->NewNode(common()->LoopExit(), b1.if_false, loop); + Node* exit2 = graph()->NewNode(common()->LoopExit(), b2.if_false, loop); + + Node* merge = graph()->NewNode(common()->Merge(2), exit1, exit2); + Node* r = InsertReturn(p0, start(), merge); + + PeeledIteration* peeled = PeelOne(); + + Node* b1p = ExpectPeeled(b1.branch, peeled); + Node* if_true1p = ExpectPeeled(b1.if_true, peeled); + Node* if_false1p = ExpectPeeled(b1.if_false, peeled); + + Node* b2p = ExpectPeeled(b2.branch, peeled); + Node* if_true2p = ExpectPeeled(b2.if_true, peeled); + Node* if_false2p = ExpectPeeled(b2.if_false, peeled); + + EXPECT_THAT(b1p, IsBranch(p0, start())); + EXPECT_THAT(if_true1p, IsIfTrue(b1p)); + EXPECT_THAT(if_false1p, IsIfFalse(b1p)); + + EXPECT_THAT(b2p, IsBranch(p0, if_true1p)); + EXPECT_THAT(if_true2p, IsIfTrue(b2p)); + EXPECT_THAT(if_false2p, IsIfFalse(b2p)); + + EXPECT_THAT(exit1, IsMerge(b1.if_false, if_false1p)); + EXPECT_THAT(exit2, IsMerge(b2.if_false, if_false2p)); + + EXPECT_THAT(loop, IsLoop(if_true2p, b2.if_true)); + + EXPECT_THAT(merge, IsMerge(exit1, exit2)); + EXPECT_THAT(r, IsReturn(p0, start(), merge)); } - -const Operator kMockCall(IrOpcode::kCall, Operator::kNoProperties, "MockCall", - 0, 0, 1, 1, 1, 2); - - -TEST_F(LoopPeelingTest, TwoExitLoopWithCall_nope) { +TEST_F(LoopPeelingTest, SimpleLoopWithUnmarkedExit) { Node* p0 = Parameter(0); Node* loop = graph()->NewNode(common()->Loop(2), start(), start()); - Branch b1 = NewBranch(p0, loop); + Branch b = NewBranch(p0, loop); + loop->ReplaceInput(1, b.if_true); - Node* call = graph()->NewNode(&kMockCall, b1.if_true); - Node* if_success = graph()->NewNode(common()->IfSuccess(), call); - Node* if_exception = graph()->NewNode( - common()->IfException(IfExceptionHint::kLocallyUncaught), call, call); - - loop->ReplaceInput(1, if_success); - Node* merge = graph()->NewNode(common()->Merge(2), b1.if_false, if_exception); - InsertReturn(p0, start(), merge); + InsertReturn(p0, start(), b.if_false); { LoopTree* loop_tree = GetLoopTree();