AuroraMiddleware/v8
1
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

[turbofan] Loop peeling with explicit loop exits.

Browse Source 
Review-Url: https://codereview.chromium.org/2143163002
Cr-Commit-Position: refs/heads/master@{#37791}
This commit is contained in:
jarin 2016-07-15 03:23:29 -07:00 committed by Commit bot
parent 9f46c1112b
commit 54a0389309
7 changed files with 331 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
src/compiler/loop-analysis.cc
View File

@ -29,6 +29,7 @@ struct NodeInfo {
struct LoopInfo {
Node* header;
NodeInfo* header_list;
NodeInfo* exit_list;
NodeInfo* body_list;
LoopTree::Loop* loop;
};
@ -81,9 +82,9 @@ class LoopFinderImpl {
if (marked_forward && marked_backward) {
PrintF("X");
} else if (marked_forward) {
PrintF("/");
PrintF(">");
} else if (marked_backward) {
PrintF("\\");
PrintF("<");
} else {
PrintF(" ");
}
@ -198,12 +199,22 @@ class LoopFinderImpl {
if (merge->opcode() == IrOpcode::kLoop) {
loop_num = CreateLoopInfo(merge);
}
} else if (node->opcode() == IrOpcode::kLoopExit) {
// Intentionally ignore return value. Loop exit node marks
// are propagated normally.
CreateLoopInfo(node->InputAt(1));
} else if (node->opcode() == IrOpcode::kLoopExitValue ||
node->opcode() == IrOpcode::kLoopExitEffect) {
Node* loop_exit = NodeProperties::GetControlInput(node);
// Intentionally ignore return value. Loop exit node marks
// are propagated normally.
CreateLoopInfo(loop_exit->InputAt(1));
}
// Propagate marks backwards from this node.
for (int i = 0; i < node->InputCount(); i++) {
Node* input = node->InputAt(i);
if (loop_num > 0 && i != kAssumedLoopEntryIndex) {
if (IsBackedge(node, i)) {
// Only propagate the loop mark on backedges.
if (SetBackwardMark(input, loop_num)) Queue(input);
} else {
@ -216,6 +227,7 @@ class LoopFinderImpl {
// Make a new loop if necessary for the given node.
int CreateLoopInfo(Node* node) {
DCHECK_EQ(IrOpcode::kLoop, node->opcode());
int loop_num = LoopNum(node);
if (loop_num > 0) return loop_num;
@ -223,21 +235,39 @@ class LoopFinderImpl {
if (INDEX(loop_num) >= width_) ResizeBackwardMarks();
// Create a new loop.
loops_.push_back({node, nullptr, nullptr, nullptr});
loops_.push_back({node, nullptr, nullptr, nullptr, nullptr});
loop_tree_->NewLoop();
SetLoopMarkForLoopHeader(node, loop_num);
return loop_num;
}
void SetLoopMark(Node* node, int loop_num) {
info(node); // create the NodeInfo
SetBackwardMark(node, loop_num);
loop_tree_->node_to_loop_num_[node->id()] = loop_num;
}
// Setup loop mark for phis attached to loop header.
void SetLoopMarkForLoopHeader(Node* node, int loop_num) {
DCHECK_EQ(IrOpcode::kLoop, node->opcode());
SetLoopMark(node, loop_num);
for (Node* use : node->uses()) {
if (NodeProperties::IsPhi(use)) {
info(use); // create the NodeInfo
SetBackwardMark(use, loop_num);
loop_tree_->node_to_loop_num_[use->id()] = loop_num;
SetLoopMark(use, loop_num);
}
// Do not keep the loop alive if it does not have any backedges.
if (node->InputCount() <= 1) continue;
if (use->opcode() == IrOpcode::kLoopExit) {
SetLoopMark(use, loop_num);
for (Node* exit_use : use->uses()) {
if (exit_use->opcode() == IrOpcode::kLoopExitValue ||
exit_use->opcode() == IrOpcode::kLoopExitEffect) {
SetLoopMark(exit_use, loop_num);
}
}
}
}
return loop_num;
}
void ResizeBackwardMarks() {
@ -276,20 +306,33 @@ class LoopFinderImpl {
queued_.Set(node, false);
for (Edge edge : node->use_edges()) {
Node* use = edge.from();
if (!IsBackedge(use, edge)) {
if (!IsBackedge(use, edge.index())) {
if (PropagateForwardMarks(node, use)) Queue(use);
}
}
}
}
bool IsBackedge(Node* use, Edge& edge) {
bool IsLoopHeaderNode(Node* node) {
return node->opcode() == IrOpcode::kLoop || NodeProperties::IsPhi(node);
}
bool IsLoopExitNode(Node* node) {
return node->opcode() == IrOpcode::kLoopExit ||
node->opcode() == IrOpcode::kLoopExitValue ||
node->opcode() == IrOpcode::kLoopExitEffect;
}
bool IsBackedge(Node* use, int index) {
if (LoopNum(use) <= 0) return false;
if (edge.index() == kAssumedLoopEntryIndex) return false;
if (NodeProperties::IsPhi(use)) {
return !NodeProperties::IsControlEdge(edge);
return index != NodeProperties::FirstControlIndex(use) &&
index != kAssumedLoopEntryIndex;
} else if (use->opcode() == IrOpcode::kLoop) {
return index != kAssumedLoopEntryIndex;
}
return true;
DCHECK(IsLoopExitNode(use));
return false;
}
int LoopNum(Node* node) { return loop_tree_->node_to_loop_num_[node->id()]; }
@ -307,6 +350,22 @@ class LoopFinderImpl {
}
}
void AddNodeToLoop(NodeInfo* node_info, LoopInfo* loop, int loop_num) {
if (LoopNum(node_info->node) == loop_num) {
if (IsLoopHeaderNode(node_info->node)) {
node_info->next = loop->header_list;
loop->header_list = node_info;
} else {
DCHECK(IsLoopExitNode(node_info->node));
node_info->next = loop->exit_list;
loop->exit_list = node_info;
}
} else {
node_info->next = loop->body_list;
loop->body_list = node_info;
}
}
void FinishLoopTree() {
DCHECK(loops_found_ == static_cast<int>(loops_.size()));
DCHECK(loops_found_ == static_cast<int>(loop_tree_->all_loops_.size()));
@ -342,13 +401,7 @@ class LoopFinderImpl {
}
}
if (innermost == nullptr) continue;
if (LoopNum(ni.node) == innermost_index) {
ni.next = innermost->header_list;
innermost->header_list = &ni;
} else {
ni.next = innermost->body_list;
innermost->body_list = &ni;
}
AddNodeToLoop(&ni, innermost, innermost_index);
count++;
}
@ -368,13 +421,7 @@ class LoopFinderImpl {
size_t count = 0;
for (NodeInfo& ni : info_) {
if (ni.node == nullptr || !IsInLoop(ni.node, 1)) continue;
if (LoopNum(ni.node) == 1) {
ni.next = li->header_list;
li->header_list = &ni;
} else {
ni.next = li->body_list;
li->body_list = &ni;
}
AddNodeToLoop(&ni, li, 1);
count++;
}
@ -406,7 +453,14 @@ class LoopFinderImpl {
// Serialize nested loops.
for (LoopTree::Loop* child : loop->children_) SerializeLoop(child);
loop->body_end_ = static_cast<int>(loop_tree_->loop_nodes_.size());
// Serialize the exits.
loop->exits_start_ = static_cast<int>(loop_tree_->loop_nodes_.size());
for (NodeInfo* ni = li.exit_list; ni != nullptr; ni = ni->next) {
loop_tree_->loop_nodes_.push_back(ni->node);
loop_tree_->node_to_loop_num_[ni->node->id()] = loop_num;
}
loop->exits_end_ = static_cast<int>(loop_tree_->loop_nodes_.size());
}
// Connect the LoopTree loops to their parents recursively.
@ -438,9 +492,12 @@ class LoopFinderImpl {
while (i < loop->body_start_) {
PrintF(" H#%d", loop_tree_->loop_nodes_[i++]->id());
}
while (i < loop->body_end_) {
while (i < loop->exits_start_) {
PrintF(" B#%d", loop_tree_->loop_nodes_[i++]->id());
}
while (i < loop->exits_end_) {
PrintF(" E#%d", loop_tree_->loop_nodes_[i++]->id());
}
PrintF("\n");
for (LoopTree::Loop* child : loop->children_) PrintLoop(child);
}

21
src/compiler/loop-analysis.h
View File

@ -38,8 +38,9 @@ class LoopTree : public ZoneObject {
Loop* parent() const { return parent_; }
const ZoneVector<Loop*>& children() const { return children_; }
size_t HeaderSize() const { return body_start_ - header_start_; }
size_t BodySize() const { return body_end_ - body_start_; }
size_t TotalSize() const { return body_end_ - header_start_; }
size_t BodySize() const { return exits_start_ - body_start_; }
size_t ExitsSize() const { return exits_end_ - exits_start_; }
size_t TotalSize() const { return exits_end_ - header_start_; }
size_t depth() const { return static_cast<size_t>(depth_); }
private:
@ -52,13 +53,15 @@ class LoopTree : public ZoneObject {
children_(zone),
header_start_(-1),
body_start_(-1),
body_end_(-1) {}
exits_start_(-1),
exits_end_(-1) {}
Loop* parent_;
int depth_;
ZoneVector<Loop*> children_;
int header_start_;
int body_start_;
int body_end_;
int exits_start_;
int exits_end_;
};
// Return the innermost nested loop, if any, that contains {node}.
@ -97,13 +100,19 @@ class LoopTree : public ZoneObject {
// Return a range which can iterate over the body nodes of {loop}.
NodeRange BodyNodes(Loop* loop) {
return NodeRange(&loop_nodes_[0] + loop->body_start_,
&loop_nodes_[0] + loop->body_end_);
&loop_nodes_[0] + loop->exits_start_);
}
// Return a range which can iterate over the body nodes of {loop}.
NodeRange ExitNodes(Loop* loop) {
return NodeRange(&loop_nodes_[0] + loop->exits_start_,
&loop_nodes_[0] + loop->exits_end_);
}
// Return a range which can iterate over the nodes of {loop}.
NodeRange LoopNodes(Loop* loop) {
return NodeRange(&loop_nodes_[0] + loop->header_start_,
&loop_nodes_[0] + loop->body_end_);
&loop_nodes_[0] + loop->exits_end_);
}
// Return the node that represents the control, i.e. the loop node itself.

180
src/compiler/loop-peeling.cc
View File

@ -126,8 +126,14 @@ struct Peeling {
// Copy all the nodes first.
for (Node* node : nodes) {
inputs.clear();
for (Node* input : node->inputs()) inputs.push_back(map(input));
Insert(node, graph->NewNode(node->op(), node->InputCount(), &inputs[0]));
for (Node* input : node->inputs()) {
inputs.push_back(map(input));
}
Node* copy = graph->NewNode(node->op(), node->InputCount(), &inputs[0]);
if (NodeProperties::IsTyped(node)) {
NodeProperties::SetType(copy, NodeProperties::GetType(node));
}
Insert(node, copy);
}
// Fix remaining inputs of the copies.
@ -160,56 +166,54 @@ Node* PeeledIteration::map(Node* node) {
return node;
}
static void FindLoopExits(LoopTree* loop_tree, LoopTree::Loop* loop,
NodeVector& exits, NodeVector& rets) {
bool LoopPeeler::CanPeel(LoopTree* loop_tree, LoopTree::Loop* loop) {
// Look for returns and if projections that are outside the loop but whose
// control input is inside the loop.
Node* loop_node = loop_tree->GetLoopControl(loop);
for (Node* node : loop_tree->LoopNodes(loop)) {
for (Node* use : node->uses()) {
if (!loop_tree->Contains(loop, use)) {
if (IrOpcode::IsIfProjectionOpcode(use->opcode())) {
// This is a branch from inside the loop to outside the loop.
exits.push_back(use);
} else if (use->opcode() == IrOpcode::kReturn &&
loop_tree->Contains(loop,
NodeProperties::GetControlInput(use))) {
// This is a return from inside the loop.
rets.push_back(use);
bool unmarked_exit;
switch (node->opcode()) {
case IrOpcode::kLoopExit:
unmarked_exit = (node->InputAt(1) != loop_node);
break;
case IrOpcode::kLoopExitValue:
case IrOpcode::kLoopExitEffect:
unmarked_exit = (node->InputAt(1)->InputAt(1) != loop_node);
break;
default:
unmarked_exit = (use->opcode() != IrOpcode::kTerminate);
}
if (unmarked_exit) {
if (FLAG_trace_turbo_graph) {
Node* loop_node = loop_tree->GetLoopControl(loop);
PrintF(
"Cannot peel loop %i. Loop exit without explicit mark: Node %i "
"(%s) is inside "
"loop, but its use %i (%s) is outside.\n",
loop_node->id(), node->id(), node->op()->mnemonic(), use->id(),
use->op()->mnemonic());
}
return false;
}
}
}
}
}
bool LoopPeeler::CanPeel(LoopTree* loop_tree, LoopTree::Loop* loop) {
Zone zone(loop_tree->zone()->allocator());
NodeVector exits(&zone);
NodeVector rets(&zone);
FindLoopExits(loop_tree, loop, exits, rets);
return exits.size() <= 1u;
return true;
}
PeeledIteration* LoopPeeler::Peel(Graph* graph, CommonOperatorBuilder* common,
LoopTree* loop_tree, LoopTree::Loop* loop,
Zone* tmp_zone) {
//============================================================================
// Find the loop exit region to determine if this loop can be peeled.
//============================================================================
NodeVector exits(tmp_zone);
NodeVector rets(tmp_zone);
FindLoopExits(loop_tree, loop, exits, rets);
if (exits.size() != 1) return nullptr; // not peelable currently.
if (!CanPeel(loop_tree, loop)) return nullptr;
//============================================================================
// Construct the peeled iteration.
//============================================================================
PeeledIterationImpl* iter = new (tmp_zone) PeeledIterationImpl(tmp_zone);
size_t estimated_peeled_size =
5 + (loop->TotalSize() + exits.size() + rets.size()) * 2;
size_t estimated_peeled_size = 5 + (loop->TotalSize()) * 2;
Peeling peeling(graph, tmp_zone, estimated_peeled_size, &iter->node_pairs_);
Node* dead = graph->NewNode(common->Dead());
@ -260,77 +264,64 @@ PeeledIteration* LoopPeeler::Peel(Graph* graph, CommonOperatorBuilder* common,
// Only one backedge, simply replace the input to loop with output of
// peeling.
for (Node* node : loop_tree->HeaderNodes(loop)) {
node->ReplaceInput(0, peeling.map(node->InputAt(0)));
node->ReplaceInput(0, peeling.map(node->InputAt(1)));
}
new_entry = peeling.map(loop_node->InputAt(1));
}
loop_node->ReplaceInput(0, new_entry);
//============================================================================
// Duplicate the loop exit region and add a merge.
// Change the exit and exit markers to merge/phi/effect-phi.
//============================================================================
// Currently we are limited to peeling loops with a single exit. The exit is
// the postdominator of the loop (ignoring returns).
Node* postdom = exits[0];
for (Node* node : rets) exits.push_back(node);
for (Node* use : postdom->uses()) {
if (NodeProperties::IsPhi(use)) exits.push_back(use);
}
NodeRange exit_range(&exits[0], &exits[0] + exits.size());
peeling.CopyNodes(graph, tmp_zone, dead, exit_range);
Node* merge = graph->NewNode(common->Merge(2), postdom, peeling.map(postdom));
postdom->ReplaceUses(merge);
merge->ReplaceInput(0, postdom); // input 0 overwritten by above line.
// Find and update all the edges into either the loop or exit region.
for (int i = 0; i < 2; i++) {
NodeRange range = i == 0 ? loop_tree->LoopNodes(loop) : exit_range;
ZoneVector<Edge> value_edges(tmp_zone);
ZoneVector<Edge> effect_edges(tmp_zone);
for (Node* node : range) {
// Gather value and effect edges from outside the region.
for (Edge edge : node->use_edges()) {
if (!peeling.Marked(edge.from())) {
// Edge from outside the loop into the region.
if (NodeProperties::IsValueEdge(edge) ||
NodeProperties::IsContextEdge(edge)) {
value_edges.push_back(edge);
} else if (NodeProperties::IsEffectEdge(edge)) {
effect_edges.push_back(edge);
} else {
// don't do anything for control edges.
// TODO(titzer): should update control edges to peeled?
}
}
}
// Update all the value and effect edges at once.
if (!value_edges.empty()) {
// TODO(titzer): machine type is wrong here.
Node* phi =
graph->NewNode(common->Phi(MachineRepresentation::kTagged, 2), node,
peeling.map(node), merge);
for (Edge edge : value_edges) edge.UpdateTo(phi);
value_edges.clear();
}
if (!effect_edges.empty()) {
Node* effect_phi = graph->NewNode(common->EffectPhi(2), node,
peeling.map(node), merge);
for (Edge edge : effect_edges) edge.UpdateTo(effect_phi);
effect_edges.clear();
}
for (Node* exit : loop_tree->ExitNodes(loop)) {
switch (exit->opcode()) {
case IrOpcode::kLoopExit:
// Change the loop exit node to a merge node.
exit->ReplaceInput(1, peeling.map(exit->InputAt(0)));
NodeProperties::ChangeOp(exit, common->Merge(2));
break;
case IrOpcode::kLoopExitValue:
// Change exit marker to phi.
exit->InsertInput(graph->zone(), 1, peeling.map(exit->InputAt(0)));
NodeProperties::ChangeOp(
exit, common->Phi(MachineRepresentation::kTagged, 2));
break;
case IrOpcode::kLoopExitEffect:
// Change effect exit marker to effect phi.
exit->InsertInput(graph->zone(), 1, peeling.map(exit->InputAt(0)));
NodeProperties::ChangeOp(exit, common->EffectPhi(2));
break;
default:
break;
}
}
return iter;
}
namespace {
void PeelInnerLoops(Graph* graph, CommonOperatorBuilder* common,
LoopTree* loop_tree, LoopTree::Loop* loop,
Zone* temp_zone) {
// If the loop has nested loops, peel inside those.
if (!loop->children().empty()) {
for (LoopTree::Loop* inner_loop : loop->children()) {
PeelInnerLoops(graph, common, loop_tree, inner_loop, temp_zone);
}
return;
}
// Only peel small-enough loops.
if (loop->TotalSize() > LoopPeeler::kMaxPeeledNodes) return;
if (FLAG_trace_turbo_graph) {
PrintF("Peeling loop with header: ");
for (Node* node : loop_tree->HeaderNodes(loop)) {
PrintF("%i ", node->id());
}
}
LoopPeeler::Peel(graph, common, loop_tree, loop, temp_zone);
}
void EliminateLoopExit(Node* node) {
DCHECK_EQ(IrOpcode::kLoopExit, node->opcode());
// The exit markers take the loop exit as input. We iterate over uses
@ -355,6 +346,17 @@ void EliminateLoopExit(Node* node) {
} // namespace
// static
void LoopPeeler::PeelInnerLoopsOfTree(Graph* graph,
CommonOperatorBuilder* common,
LoopTree* loop_tree, Zone* temp_zone) {
for (LoopTree::Loop* loop : loop_tree->outer_loops()) {
PeelInnerLoops(graph, common, loop_tree, loop, temp_zone);
}
EliminateLoopExits(graph, temp_zone);
}
// static
void LoopPeeler::EliminateLoopExits(Graph* graph, Zone* temp_zone) {
ZoneQueue<Node*> queue(temp_zone);

4
src/compiler/loop-peeling.h
View File

@ -33,7 +33,11 @@ class LoopPeeler {
static PeeledIteration* Peel(Graph* graph, CommonOperatorBuilder* common,
LoopTree* loop_tree, LoopTree::Loop* loop,
Zone* tmp_zone);
static void PeelInnerLoopsOfTree(Graph* graph, CommonOperatorBuilder* common,
LoopTree* loop_tree, Zone* tmp_zone);
static void EliminateLoopExits(Graph* graph, Zone* temp_zone);
static const size_t kMaxPeeledNodes = 1000;
};

26
src/compiler/pipeline.cc
View File

@ -942,6 +942,22 @@ struct RepresentationSelectionPhase {
}
};
struct LoopPeelingPhase {
static const char* phase_name() { return "loop peeling"; }
void Run(PipelineData* data, Zone* temp_zone) {
GraphTrimmer trimmer(temp_zone, data->graph());
NodeVector roots(temp_zone);
data->jsgraph()->GetCachedNodes(&roots);
trimmer.TrimGraph(roots.begin(), roots.end());
LoopTree* loop_tree =
LoopFinder::BuildLoopTree(data->jsgraph()->graph(), temp_zone);
LoopPeeler::PeelInnerLoopsOfTree(data->graph(), data->common(), loop_tree,
temp_zone);
}
};
struct LoopExitEliminationPhase {
static const char* phase_name() { return "loop exit elimination"; }
@ -1438,9 +1454,13 @@ bool PipelineImpl::CreateGraph() {
Run<TypedLoweringPhase>();
RunPrintAndVerify("Lowered typed");
// Eventually, loop peeling will be done here.
Run<LoopExitEliminationPhase>();
RunPrintAndVerify("Loop exits eliminated", true);
if (FLAG_turbo_loop_peeling) {
Run<LoopPeelingPhase>();
RunPrintAndVerify("Loops peeled", true);
} else {
Run<LoopExitEliminationPhase>();
RunPrintAndVerify("Loop exits eliminated", true);
}
if (FLAG_turbo_stress_loop_peeling) {
Run<StressLoopPeelingPhase>();

1
src/flag-definitions.h
View File

@ -473,6 +473,7 @@ DEFINE_BOOL(turbo_move_optimization, true, "optimize gap moves in TurboFan")
DEFINE_BOOL(turbo_jt, true, "enable jump threading in TurboFan")
DEFINE_BOOL(turbo_stress_loop_peeling, false,
"stress loop peeling optimization")
DEFINE_BOOL(turbo_loop_peeling, false, "Turbofan loop peeling")
DEFINE_BOOL(turbo_cf_optimization, true, "optimize control flow in TurboFan")
DEFINE_BOOL(turbo_frame_elision, true, "elide frames in TurboFan")
DEFINE_BOOL(turbo_cache_shared_code, true, "cache context-independent code")

192
test/unittests/compiler/loop-peeling-unittest.cc
View File

@ -28,6 +28,7 @@ struct While {
Node* loop;
Node* branch;
Node* if_true;
Node* if_false;
Node* exit;
};
@ -46,6 +47,7 @@ struct Counter {
Node* inc;
Node* phi;
Node* add;
Node* exit_marker;
};
@ -105,12 +107,14 @@ class LoopPeelingTest : public GraphTest {
While NewWhile(Node* cond, Node* control = nullptr) {
if (control == nullptr) control = start();
Node* loop = graph()->NewNode(common()->Loop(2), control, control);
Node* branch = graph()->NewNode(common()->Branch(), cond, loop);
Node* if_true = graph()->NewNode(common()->IfTrue(), branch);
Node* exit = graph()->NewNode(common()->IfFalse(), branch);
loop->ReplaceInput(1, if_true);
return {loop, branch, if_true, exit};
While w;
w.loop = graph()->NewNode(common()->Loop(2), control, control);
w.branch = graph()->NewNode(common()->Branch(), cond, w.loop);
w.if_true = graph()->NewNode(common()->IfTrue(), w.branch);
w.if_false = graph()->NewNode(common()->IfFalse(), w.branch);
w.exit = graph()->NewNode(common()->LoopExit(), w.if_false, w.loop);
w.loop->ReplaceInput(1, w.if_true);
return w;
}
void Chain(While* a, Node* control) { a->loop->ReplaceInput(0, control); }
@ -124,21 +128,24 @@ class LoopPeelingTest : public GraphTest {
}
Branch NewBranch(Node* cond, Node* control = nullptr) {
Branch b;
if (control == nullptr) control = start();
Node* branch = graph()->NewNode(common()->Branch(), cond, control);
Node* if_true = graph()->NewNode(common()->IfTrue(), branch);
Node* if_false = graph()->NewNode(common()->IfFalse(), branch);
return {branch, if_true, if_false};
b.branch = graph()->NewNode(common()->Branch(), cond, control);
b.if_true = graph()->NewNode(common()->IfTrue(), b.branch);
b.if_false = graph()->NewNode(common()->IfFalse(), b.branch);
return b;
}
Counter NewCounter(While* w, int32_t b, int32_t k) {
Node* base = Int32Constant(b);
Node* inc = Int32Constant(k);
Node* phi = graph()->NewNode(
common()->Phi(MachineRepresentation::kTagged, 2), base, base, w->loop);
Node* add = graph()->NewNode(machine()->Int32Add(), phi, inc);
phi->ReplaceInput(1, add);
return {base, inc, phi, add};
Counter c;
c.base = Int32Constant(b);
c.inc = Int32Constant(k);
c.phi = graph()->NewNode(common()->Phi(MachineRepresentation::kTagged, 2),
c.base, c.base, w->loop);
c.add = graph()->NewNode(machine()->Int32Add(), c.phi, c.inc);
c.phi->ReplaceInput(1, c.add);
c.exit_marker = graph()->NewNode(common()->LoopExitValue(), c.phi, w->exit);
return c;
}
};
@ -152,14 +159,14 @@ TEST_F(LoopPeelingTest, SimpleLoop) {
Node* br1 = ExpectPeeled(w.branch, peeled);
Node* if_true1 = ExpectPeeled(w.if_true, peeled);
Node* if_false1 = ExpectPeeled(w.exit, peeled);
Node* if_false1 = ExpectPeeled(w.if_false, peeled);
EXPECT_THAT(br1, IsBranch(p0, start()));
EXPECT_THAT(if_true1, IsIfTrue(br1));
EXPECT_THAT(if_false1, IsIfFalse(br1));
EXPECT_THAT(w.loop, IsLoop(if_true1, w.if_true));
EXPECT_THAT(r, IsReturn(p0, start(), IsMerge(w.exit, if_false1)));
EXPECT_THAT(r, IsReturn(p0, start(), IsMerge(w.if_false, if_false1)));
}
@ -167,13 +174,13 @@ TEST_F(LoopPeelingTest, SimpleLoopWithCounter) {
Node* p0 = Parameter(0);
While w = NewWhile(p0);
Counter c = NewCounter(&w, 0, 1);
Node* r = InsertReturn(c.phi, start(), w.exit);
Node* r = InsertReturn(c.exit_marker, start(), w.exit);
PeeledIteration* peeled = PeelOne();
Node* br1 = ExpectPeeled(w.branch, peeled);
Node* if_true1 = ExpectPeeled(w.if_true, peeled);
Node* if_false1 = ExpectPeeled(w.exit, peeled);
Node* if_false1 = ExpectPeeled(w.if_false, peeled);
EXPECT_THAT(br1, IsBranch(p0, start()));
EXPECT_THAT(if_true1, IsIfTrue(br1));
@ -182,11 +189,10 @@ TEST_F(LoopPeelingTest, SimpleLoopWithCounter) {
EXPECT_THAT(peeled->map(c.add), IsInt32Add(c.base, c.inc));
Capture<Node*> merge;
EXPECT_THAT(w.exit, IsMerge(w.if_false, if_false1));
EXPECT_THAT(
r, IsReturn(IsPhi(MachineRepresentation::kTagged, c.phi, c.base,
AllOf(CaptureEq(&merge), IsMerge(w.exit, if_false1))),
start(), CaptureEq(&merge)));
r, IsReturn(IsPhi(MachineRepresentation::kTagged, c.phi, c.base, w.exit),
start(), w.exit));
}
@ -197,13 +203,13 @@ TEST_F(LoopPeelingTest, SimpleNestedLoopWithCounter_peel_outer) {
Nest(&inner, &outer);
Counter c = NewCounter(&outer, 0, 1);
Node* r = InsertReturn(c.phi, start(), outer.exit);
Node* r = InsertReturn(c.exit_marker, start(), outer.exit);
PeeledIteration* peeled = PeelOne();
Node* bro = ExpectPeeled(outer.branch, peeled);
Node* if_trueo = ExpectPeeled(outer.if_true, peeled);
Node* if_falseo = ExpectPeeled(outer.exit, peeled);
Node* if_falseo = ExpectPeeled(outer.if_false, peeled);
EXPECT_THAT(bro, IsBranch(p0, start()));
EXPECT_THAT(if_trueo, IsIfTrue(bro));
@ -211,21 +217,21 @@ TEST_F(LoopPeelingTest, SimpleNestedLoopWithCounter_peel_outer) {
Node* bri = ExpectPeeled(inner.branch, peeled);
Node* if_truei = ExpectPeeled(inner.if_true, peeled);
Node* if_falsei = ExpectPeeled(inner.exit, peeled);
Node* if_falsei = ExpectPeeled(inner.if_false, peeled);
Node* exiti = ExpectPeeled(inner.exit, peeled);
EXPECT_THAT(bri, IsBranch(p0, ExpectPeeled(inner.loop, peeled)));
EXPECT_THAT(if_truei, IsIfTrue(bri));
EXPECT_THAT(if_falsei, IsIfFalse(bri));
EXPECT_THAT(outer.loop, IsLoop(if_falsei, inner.exit));
EXPECT_THAT(outer.loop, IsLoop(exiti, inner.exit));
EXPECT_THAT(peeled->map(c.add), IsInt32Add(c.base, c.inc));
Capture<Node*> merge;
EXPECT_THAT(
r,
IsReturn(IsPhi(MachineRepresentation::kTagged, c.phi, c.base,
AllOf(CaptureEq(&merge), IsMerge(outer.exit, if_falseo))),
start(), CaptureEq(&merge)));
EXPECT_THAT(outer.exit, IsMerge(outer.if_false, if_falseo));
EXPECT_THAT(r, IsReturn(IsPhi(MachineRepresentation::kTagged, c.phi, c.base,
outer.exit),
start(), outer.exit));
}
@ -236,7 +242,7 @@ TEST_F(LoopPeelingTest, SimpleNestedLoopWithCounter_peel_inner) {
Nest(&inner, &outer);
Counter c = NewCounter(&outer, 0, 1);
Node* r = InsertReturn(c.phi, start(), outer.exit);
Node* r = InsertReturn(c.exit_marker, start(), outer.exit);
LoopTree* loop_tree = GetLoopTree();
LoopTree::Loop* loop = loop_tree->ContainingLoop(inner.loop);
@ -248,20 +254,22 @@ TEST_F(LoopPeelingTest, SimpleNestedLoopWithCounter_peel_inner) {
ExpectNotPeeled(outer.loop, peeled);
ExpectNotPeeled(outer.branch, peeled);
ExpectNotPeeled(outer.if_true, peeled);
ExpectNotPeeled(outer.if_false, peeled);
ExpectNotPeeled(outer.exit, peeled);
Node* bri = ExpectPeeled(inner.branch, peeled);
Node* if_truei = ExpectPeeled(inner.if_true, peeled);
Node* if_falsei = ExpectPeeled(inner.exit, peeled);
Node* if_falsei = ExpectPeeled(inner.if_false, peeled);
EXPECT_THAT(bri, IsBranch(p0, ExpectPeeled(inner.loop, peeled)));
EXPECT_THAT(if_truei, IsIfTrue(bri));
EXPECT_THAT(if_falsei, IsIfFalse(bri));
EXPECT_THAT(outer.loop, IsLoop(start(), IsMerge(inner.exit, if_falsei)));
EXPECT_THAT(inner.exit, IsMerge(inner.if_false, if_falsei));
EXPECT_THAT(outer.loop, IsLoop(start(), inner.exit));
ExpectNotPeeled(c.add, peeled);
EXPECT_THAT(r, IsReturn(c.phi, start(), outer.exit));
EXPECT_THAT(r, IsReturn(c.exit_marker, start(), outer.exit));
}
@ -271,7 +279,7 @@ TEST_F(LoopPeelingTest, SimpleInnerCounter_peel_inner) {
While inner = NewWhile(p0);
Nest(&inner, &outer);
Counter c = NewCounter(&inner, 0, 1);
Node* phi = NewPhi(&outer, Int32Constant(11), c.phi);
Node* phi = NewPhi(&outer, Int32Constant(11), c.exit_marker);
Node* r = InsertReturn(phi, start(), outer.exit);
@ -285,25 +293,26 @@ TEST_F(LoopPeelingTest, SimpleInnerCounter_peel_inner) {
ExpectNotPeeled(outer.loop, peeled);
ExpectNotPeeled(outer.branch, peeled);
ExpectNotPeeled(outer.if_true, peeled);
ExpectNotPeeled(outer.if_false, peeled);
ExpectNotPeeled(outer.exit, peeled);
Node* bri = ExpectPeeled(inner.branch, peeled);
Node* if_truei = ExpectPeeled(inner.if_true, peeled);
Node* if_falsei = ExpectPeeled(inner.exit, peeled);
Node* if_falsei = ExpectPeeled(inner.if_false, peeled);
EXPECT_THAT(bri, IsBranch(p0, ExpectPeeled(inner.loop, peeled)));
EXPECT_THAT(if_truei, IsIfTrue(bri));
EXPECT_THAT(if_falsei, IsIfFalse(bri));
EXPECT_THAT(outer.loop, IsLoop(start(), IsMerge(inner.exit, if_falsei)));
EXPECT_THAT(inner.exit, IsMerge(inner.if_false, if_falsei));
EXPECT_THAT(outer.loop, IsLoop(start(), inner.exit));
EXPECT_THAT(peeled->map(c.add), IsInt32Add(c.base, c.inc));
Node* back = phi->InputAt(1);
EXPECT_THAT(back, IsPhi(MachineRepresentation::kTagged, c.phi, c.base,
IsMerge(inner.exit, if_falsei)));
EXPECT_THAT(c.exit_marker,
IsPhi(MachineRepresentation::kTagged, c.phi, c.base, inner.exit));
EXPECT_THAT(phi, IsPhi(MachineRepresentation::kTagged, IsInt32Constant(11),
back, outer.loop));
c.exit_marker, outer.loop));
EXPECT_THAT(r, IsReturn(phi, start(), outer.exit));
}
@ -318,7 +327,9 @@ TEST_F(LoopPeelingTest, TwoBackedgeLoop) {
loop->ReplaceInput(1, b2.if_true);
loop->ReplaceInput(2, b2.if_false);
Node* r = InsertReturn(p0, start(), b1.if_false);
Node* exit = graph()->NewNode(common()->LoopExit(), b1.if_false, loop);
Node* r = InsertReturn(p0, start(), exit);
PeeledIteration* peeled = PeelOne();
@ -339,7 +350,8 @@ TEST_F(LoopPeelingTest, TwoBackedgeLoop) {
EXPECT_THAT(b2f, IsIfFalse(b2b));
EXPECT_THAT(loop, IsLoop(IsMerge(b2t, b2f), b2.if_true, b2.if_false));
EXPECT_THAT(r, IsReturn(p0, start(), IsMerge(b1.if_false, b1f)));
EXPECT_THAT(exit, IsMerge(b1.if_false, b1f));
EXPECT_THAT(r, IsReturn(p0, start(), exit));
}
@ -355,7 +367,9 @@ TEST_F(LoopPeelingTest, TwoBackedgeLoopWithPhi) {
loop->ReplaceInput(1, b2.if_true);
loop->ReplaceInput(2, b2.if_false);
Node* r = InsertReturn(phi, start(), b1.if_false);
Node* exit = graph()->NewNode(common()->LoopExit(), b1.if_false, loop);
Node* exit_marker = graph()->NewNode(common()->LoopExitValue(), phi, exit);
Node* r = InsertReturn(exit_marker, start(), exit);
PeeledIteration* peeled = PeelOne();
@ -383,11 +397,10 @@ TEST_F(LoopPeelingTest, TwoBackedgeLoopWithPhi) {
IsInt32Constant(2), IsMerge(b2t, b2f)),
IsInt32Constant(1), IsInt32Constant(2), loop));
Capture<Node*> merge;
EXPECT_THAT(
r, IsReturn(IsPhi(MachineRepresentation::kTagged, phi, IsInt32Constant(0),
AllOf(CaptureEq(&merge), IsMerge(b1.if_false, b1f))),
start(), CaptureEq(&merge)));
EXPECT_THAT(exit, IsMerge(b1.if_false, b1f));
EXPECT_THAT(exit_marker, IsPhi(MachineRepresentation::kTagged, phi,
IsInt32Constant(0), exit));
EXPECT_THAT(r, IsReturn(exit_marker, start(), exit));
}
@ -408,7 +421,9 @@ TEST_F(LoopPeelingTest, TwoBackedgeLoopWithCounter) {
loop->ReplaceInput(1, b2.if_true);
loop->ReplaceInput(2, b2.if_false);
Node* r = InsertReturn(phi, start(), b1.if_false);
Node* exit = graph()->NewNode(common()->LoopExit(), b1.if_false, loop);
Node* exit_marker = graph()->NewNode(common()->LoopExitValue(), phi, exit);
Node* r = InsertReturn(exit_marker, start(), exit);
PeeledIteration* peeled = PeelOne();
@ -443,49 +458,60 @@ TEST_F(LoopPeelingTest, TwoBackedgeLoopWithCounter) {
IsInt32Add(phi, IsInt32Constant(1)),
IsInt32Add(phi, IsInt32Constant(2)), loop));
Capture<Node*> merge;
EXPECT_THAT(
r, IsReturn(IsPhi(MachineRepresentation::kTagged, phi, IsInt32Constant(0),
AllOf(CaptureEq(&merge), IsMerge(b1.if_false, b1f))),
start(), CaptureEq(&merge)));
EXPECT_THAT(exit, IsMerge(b1.if_false, b1f));
EXPECT_THAT(exit_marker, IsPhi(MachineRepresentation::kTagged, phi,
IsInt32Constant(0), exit));
EXPECT_THAT(r, IsReturn(exit_marker, start(), exit));
}
TEST_F(LoopPeelingTest, TwoExitLoop_nope) {
TEST_F(LoopPeelingTest, TwoExitLoop) {
Node* p0 = Parameter(0);
Node* loop = graph()->NewNode(common()->Loop(2), start(), start());
Branch b1 = NewBranch(p0, loop);
Branch b2 = NewBranch(p0, b1.if_true);
loop->ReplaceInput(1, b2.if_true);
Node* merge = graph()->NewNode(common()->Merge(2), b1.if_false, b2.if_false);
InsertReturn(p0, start(), merge);
{
LoopTree* loop_tree = GetLoopTree();
LoopTree::Loop* loop = loop_tree->outer_loops()[0];
EXPECT_FALSE(LoopPeeler::CanPeel(loop_tree, loop));
}
Node* exit1 = graph()->NewNode(common()->LoopExit(), b1.if_false, loop);
Node* exit2 = graph()->NewNode(common()->LoopExit(), b2.if_false, loop);
Node* merge = graph()->NewNode(common()->Merge(2), exit1, exit2);
Node* r = InsertReturn(p0, start(), merge);
PeeledIteration* peeled = PeelOne();
Node* b1p = ExpectPeeled(b1.branch, peeled);
Node* if_true1p = ExpectPeeled(b1.if_true, peeled);
Node* if_false1p = ExpectPeeled(b1.if_false, peeled);
Node* b2p = ExpectPeeled(b2.branch, peeled);
Node* if_true2p = ExpectPeeled(b2.if_true, peeled);
Node* if_false2p = ExpectPeeled(b2.if_false, peeled);
EXPECT_THAT(b1p, IsBranch(p0, start()));
EXPECT_THAT(if_true1p, IsIfTrue(b1p));
EXPECT_THAT(if_false1p, IsIfFalse(b1p));
EXPECT_THAT(b2p, IsBranch(p0, if_true1p));
EXPECT_THAT(if_true2p, IsIfTrue(b2p));
EXPECT_THAT(if_false2p, IsIfFalse(b2p));
EXPECT_THAT(exit1, IsMerge(b1.if_false, if_false1p));
EXPECT_THAT(exit2, IsMerge(b2.if_false, if_false2p));
EXPECT_THAT(loop, IsLoop(if_true2p, b2.if_true));
EXPECT_THAT(merge, IsMerge(exit1, exit2));
EXPECT_THAT(r, IsReturn(p0, start(), merge));
}
const Operator kMockCall(IrOpcode::kCall, Operator::kNoProperties, "MockCall",
0, 0, 1, 1, 1, 2);
TEST_F(LoopPeelingTest, TwoExitLoopWithCall_nope) {
TEST_F(LoopPeelingTest, SimpleLoopWithUnmarkedExit) {
Node* p0 = Parameter(0);
Node* loop = graph()->NewNode(common()->Loop(2), start(), start());
Branch b1 = NewBranch(p0, loop);
Branch b = NewBranch(p0, loop);
loop->ReplaceInput(1, b.if_true);
Node* call = graph()->NewNode(&kMockCall, b1.if_true);
Node* if_success = graph()->NewNode(common()->IfSuccess(), call);
Node* if_exception = graph()->NewNode(
common()->IfException(IfExceptionHint::kLocallyUncaught), call, call);
loop->ReplaceInput(1, if_success);
Node* merge = graph()->NewNode(common()->Merge(2), b1.if_false, if_exception);
InsertReturn(p0, start(), merge);
InsertReturn(p0, start(), b.if_false);
{
LoopTree* loop_tree = GetLoopTree();