AuroraMiddleware/SPIRV-Tools
1
0
Fork 0
mirror of https://github.com/KhronosGroup/SPIRV-Tools synced 2024-10-18 11:10:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add a loop fusion pass.

Browse Source 
This pass will look for adjacent loops that are compatible and legal to
be fused.

Loops are compatible if:

- they both have one induction variable
- they have the same upper and lower bounds
    - same initial value
    - same condition
- they have the same update step
- they are adjacent
- there are no break/continue in either of them

Fusion is legal if:

- fused loops do not have any dependencies with dependence distance
  greater than 0 that did not exist in the original loops.
- there are no function calls in the loops (could have side-effects)
- there are no barriers in the loops

It will fuse all such loops as long as the number of registers used for
the fused loop stays under the threshold defined by
max_registers_per_loop.
This commit is contained in:
Toomas Remmelg 2018-04-20 15:14:45 +01:00 committed by Diego Novillo
parent 9a5dd6fe88
commit 1dc2458060
21 changed files with 9863 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Android.mk
View File

@ -108,6 +108,8 @@ SPVTOOLS_OPT_SRC_FILES := \
source/opt/loop_dependence_helpers.cpp \
source/opt/loop_descriptor.cpp \
source/opt/loop_fission.cpp \
source/opt/loop_fusion.cpp \
source/opt/loop_fusion_pass.cpp \
source/opt/loop_peeling.cpp \
source/opt/loop_unroller.cpp \
source/opt/loop_unswitch_pass.cpp \

6
include/spirv-tools/optimizer.hpp
View File

@ -488,6 +488,12 @@ Optimizer::PassToken CreateLoopInvariantCodeMotionPass();
// given |threshold|.
Optimizer::PassToken CreateLoopFissionPass(size_t threshold);
// Creates a loop fusion pass.
// This pass will look for adjacent loops that are compatible and legal to be
// fused. The fuse all such loops as long as the register usage for the fused
// loop stays under the threshold defined by |max_registers_per_loop|.
Optimizer::PassToken CreateLoopFusionPass(size_t max_registers_per_loop);
// Creates a loop peeling pass.
// This pass will look for conditions inside a loop that are true or false only
// for the N first or last iteration. For loop with such condition, those N

4
source/opt/CMakeLists.txt
View File

@ -61,6 +61,8 @@ add_library(SPIRV-Tools-opt
loop_dependence.h
loop_descriptor.h
loop_fission.h
loop_fusion.h
loop_fusion_pass.h
loop_peeling.h
loop_unroller.h
loop_utils.h
@ -145,6 +147,8 @@ add_library(SPIRV-Tools-opt
loop_dependence_helpers.cpp
loop_descriptor.cpp
loop_fission.cpp
loop_fusion.cpp
loop_fusion_pass.cpp
loop_peeling.cpp
loop_utils.cpp
loop_unroller.cpp

26
source/opt/function.h
View File

@ -66,6 +66,13 @@ class Function {
template <typename T>
inline void AddBasicBlocks(T begin, T end, iterator ip);
// Move basic block with |id| to the position after |ip|. Both have to be
// contained in this function.
inline void MoveBasicBlockToAfter(uint32_t id, BasicBlock* ip);
// Delete all basic blocks that contain no instructions.
inline void RemoveEmptyBlocks();
// Saves the given function end instruction.
inline void SetFunctionEnd(std::unique_ptr<Instruction> end_inst);
@ -162,6 +169,25 @@ inline void Function::AddBasicBlocks(T src_begin, T src_end, iterator ip) {
std::make_move_iterator(src_end));
}
inline void Function::MoveBasicBlockToAfter(uint32_t id, BasicBlock* ip) {
auto block_to_move = std::move(*FindBlock(id).Get());
assert(block_to_move->GetParent() == ip->GetParent() &&
"Both blocks have to be in the same function.");
InsertBasicBlockAfter(std::move(block_to_move), ip);
blocks_.erase(std::find(std::begin(blocks_), std::end(blocks_), nullptr));
}
inline void Function::RemoveEmptyBlocks() {
auto first_empty =
std::remove_if(std::begin(blocks_), std::end(blocks_),
[](const std::unique_ptr<BasicBlock>& bb) -> bool {
return bb->GetLabelInst()->opcode() == SpvOpNop;
});
blocks_.erase(first_empty, std::end(blocks_));
}
inline void Function::SetFunctionEnd(std::unique_ptr<Instruction> end_inst) {
end_inst_ = std::move(end_inst);
}

50
source/opt/loop_dependence.cpp
View File

@ -192,11 +192,61 @@ bool LoopDependenceAnalysis::GetDependence(const ir::Instruction* source,
ir::Instruction* destination_access_chain =
GetOperandDefinition(destination, 0);
auto num_access_chains =
(source_access_chain->opcode() == SpvOpAccessChain) +
(destination_access_chain->opcode() == SpvOpAccessChain);
// If neither is an access chain, then they are load/store to a variable.
if (num_access_chains == 0) {
if (source_access_chain != destination_access_chain) {
// Not the same location, report independence
return true;
} else {
// Accessing the same variable
for (auto& entry : distance_vector->GetEntries()) {
entry = DistanceEntry();
}
return false;
}
}
// If only one is an access chain, it could be accessing a part of a struct
if (num_access_chains == 1) {
auto source_is_chain = source_access_chain->opcode() == SpvOpAccessChain;
auto access_chain =
source_is_chain ? source_access_chain : destination_access_chain;
auto variable =
source_is_chain ? destination_access_chain : source_access_chain;
auto location_in_chain = GetOperandDefinition(access_chain, 0);
if (variable != location_in_chain) {
// Not the same location, report independence
return true;
} else {
// Accessing the same variable
for (auto& entry : distance_vector->GetEntries()) {
entry = DistanceEntry();
}
return false;
}
}
// If the access chains aren't collecting from the same structure there is no
// dependence.
ir::Instruction* source_array = GetOperandDefinition(source_access_chain, 0);
ir::Instruction* destination_array =
GetOperandDefinition(destination_access_chain, 0);
// Nested access chains are not supported yet, bail out.
if (source_array->opcode() == SpvOpAccessChain ||
destination_array->opcode() == SpvOpAccessChain) {
for (auto& entry : distance_vector->GetEntries()) {
entry = DistanceEntry();
}
return false;
}
if (source_array != destination_array) {
PrintDebug("Proved independence through different arrays.");
return true;

36
source/opt/loop_descriptor.cpp
View File

@ -550,6 +550,29 @@ void LoopDescriptor::PopulateList(const Function* f) {
}
}
std::vector<ir::Loop*> LoopDescriptor::GetLoopsInBinaryLayoutOrder() {
std::vector<uint32_t> ids{};
for (size_t i = 0; i < NumLoops(); ++i) {
ids.push_back(GetLoopByIndex(i).GetHeaderBlock()->id());
}
std::vector<ir::Loop*> loops{};
if (!ids.empty()) {
auto function = GetLoopByIndex(0).GetHeaderBlock()->GetParent();
for (const auto& block : *function) {
auto block_id = block.id();
auto element = std::find(std::begin(ids), std::end(ids), block_id);
if (element != std::end(ids)) {
loops.push_back(&GetLoopByIndex(element - std::begin(ids)));
}
}
}
return loops;
}
ir::BasicBlock* Loop::FindConditionBlock() const {
if (!loop_merge_) {
return nullptr;
@ -856,6 +879,19 @@ ir::Instruction* Loop::FindConditionVariable(
return induction;
}
bool LoopDescriptor::CreatePreHeaderBlocksIfMissing() {
auto modified = false;
for (auto& loop : *this) {
if (!loop.GetPreHeaderBlock()) {
modified = true;
loop.GetOrCreatePreHeaderBlock();
}
}
return modified;
}
// Add and remove loops which have been marked for addition and removal to
// maintain the state of the loop descriptor class.
void LoopDescriptor::PostModificationCleanup() {

8
source/opt/loop_descriptor.h
View File

@ -437,6 +437,10 @@ class LoopDescriptor {
return *loops_[index];
}
// Returns the loops in |this| in the order their headers appear in the
// binary.
std::vector<ir::Loop*> GetLoopsInBinaryLayoutOrder();
// Returns the inner most loop that contains the basic block id |block_id|.
inline Loop* operator[](uint32_t block_id) const {
return FindLoopForBasicBlock(block_id);
@ -482,6 +486,10 @@ class LoopDescriptor {
loops_to_add_.emplace_back(std::make_pair(parent, loop_to_add));
}
// Checks all loops in |this| and will create pre-headers for all loops
// that don't have one. Returns |true| if any blocks were created.
bool CreatePreHeaderBlocksIfMissing();
// Should be called to preserve the LoopAnalysis after loops have been marked
// for addition with AddLoop or MarkLoopForRemoval.
void PostModificationCleanup();

731
source/opt/loop_fusion.cpp Normal file
View File

@ -0,0 +1,731 @@
// Copyright (c) 2018 Google LLC.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "opt/loop_fusion.h"
#include <algorithm>
#include <vector>
#include "opt/ir_context.h"
#include "opt/loop_dependence.h"
#include "opt/loop_descriptor.h"
namespace spvtools {
namespace opt {
namespace {
// Append all the loops nested in |loop| to |loops|.
void CollectChildren(ir::Loop* loop, std::vector<const ir::Loop*>* loops) {
for (auto child : *loop) {
loops->push_back(child);
if (child->NumImmediateChildren() != 0) {
CollectChildren(child, loops);
}
}
}
// Return the set of locations accessed by |stores| and |loads|.
std::set<ir::Instruction*> GetLocationsAccessed(
const std::map<ir::Instruction*, std::vector<ir::Instruction*>>& stores,
const std::map<ir::Instruction*, std::vector<ir::Instruction*>>& loads) {
std::set<ir::Instruction*> locations{};
for (const auto& kv : stores) {
locations.insert(std::get<0>(kv));
}
for (const auto& kv : loads) {
locations.insert(std::get<0>(kv));
}
return locations;
}
// Append all dependences from |sources| to |destinations| to |dependences|.
void GetDependences(std::vector<DistanceVector>* dependences,
LoopDependenceAnalysis* analysis,
const std::vector<ir::Instruction*>& sources,
const std::vector<ir::Instruction*>& destinations,
size_t num_entries) {
for (auto source : sources) {
for (auto destination : destinations) {
DistanceVector dist(num_entries);
if (!analysis->GetDependence(source, destination, &dist)) {
dependences->push_back(dist);
}
}
}
}
// Apped all instructions in |block| to |instructions|.
void AddInstructionsInBlock(std::vector<ir::Instruction*>* instructions,
ir::BasicBlock* block) {
for (auto& inst : *block) {
instructions->push_back(&inst);
}
instructions->push_back(block->GetLabelInst());
}
} // namespace
bool LoopFusion::UsedInContinueOrConditionBlock(
ir::Instruction* phi_instruction, ir::Loop* loop) {
auto condition_block = loop->FindConditionBlock()->id();
auto continue_block = loop->GetLatchBlock()->id();
auto not_used = context_->get_def_use_mgr()->WhileEachUser(
phi_instruction,
[this, condition_block, continue_block](ir::Instruction* instruction) {
auto block_id = context_->get_instr_block(instruction)->id();
return block_id != condition_block && block_id != continue_block;
});
return !not_used;
}
void LoopFusion::RemoveIfNotUsedContinueOrConditionBlock(
std::vector<ir::Instruction*>* instructions, ir::Loop* loop) {
instructions->erase(
std::remove_if(std::begin(*instructions), std::end(*instructions),
[this, loop](ir::Instruction* instruction) {
return !UsedInContinueOrConditionBlock(instruction,
loop);
}),
std::end(*instructions));
}
bool LoopFusion::AreCompatible() {
// Check that the loops are in the same function.
if (loop_0_->GetHeaderBlock()->GetParent() !=
loop_1_->GetHeaderBlock()->GetParent()) {
return false;
}
// Check that both loops have pre-header blocks.
if (!loop_0_->GetPreHeaderBlock() || !loop_1_->GetPreHeaderBlock()) {
return false;
}
// Check there are no breaks.
if (context_->cfg()->preds(loop_0_->GetMergeBlock()->id()).size() != 1 ||
context_->cfg()->preds(loop_1_->GetMergeBlock()->id()).size() != 1) {
return false;
}
// Check there are no continues.
if (context_->cfg()->preds(loop_0_->GetLatchBlock()->id()).size() != 1 ||
context_->cfg()->preds(loop_1_->GetLatchBlock()->id()).size() != 1) {
return false;
}
// |GetInductionVariables| returns all OpPhi in the header. Check that both
// loops have exactly one that is used in the continue and condition blocks.
std::vector<ir::Instruction*> inductions_0{}, inductions_1{};
loop_0_->GetInductionVariables(inductions_0);
RemoveIfNotUsedContinueOrConditionBlock(&inductions_0, loop_0_);
if (inductions_0.size() != 1) {
return false;
}
induction_0_ = inductions_0.front();
loop_1_->GetInductionVariables(inductions_1);
RemoveIfNotUsedContinueOrConditionBlock(&inductions_1, loop_1_);
if (inductions_1.size() != 1) {
return false;
}
induction_1_ = inductions_1.front();
if (!CheckInit()) {
return false;
}
if (!CheckCondition()) {
return false;
}
if (!CheckStep()) {
return false;
}
// Check adjacency, |loop_0_| should come just before |loop_1_|.
// There is always at least one block between loops, even if it's empty.
// We'll check at most 2 preceeding blocks.
auto pre_header_1 = loop_1_->GetPreHeaderBlock();
std::vector<ir::BasicBlock*> block_to_check{};
block_to_check.push_back(pre_header_1);
if (loop_0_->GetMergeBlock() != loop_1_->GetPreHeaderBlock()) {
// Follow CFG for one more block.
auto preds = context_->cfg()->preds(pre_header_1->id());
if (preds.size() == 1) {
auto block = &*containing_function_->FindBlock(preds.front());
if (block == loop_0_->GetMergeBlock()) {
block_to_check.push_back(block);
} else {
return false;
}
} else {
return false;
}
}
// Check that the separating blocks are either empty or only contains a store
// to a local variable that is never read (left behind by
// '--eliminate-local-multi-store'). Also allow OpPhi, since the loop could be
// in LCSSA form.
for (auto block : block_to_check) {
for (auto& inst : *block) {
if (inst.opcode() == SpvOpStore) {
// Get the definition of the target to check it's function scope so
// there are no observable side effects.
auto variable =
context_->get_def_use_mgr()->GetDef(inst.GetSingleWordInOperand(0));
if (variable->opcode() != SpvOpVariable ||
variable->GetSingleWordInOperand(0) != SpvStorageClassFunction) {
return false;
}
// Check the target is never loaded.
auto is_used = false;
context_->get_def_use_mgr()->ForEachUse(
inst.GetSingleWordInOperand(0),
[&is_used](ir::Instruction* use_inst, uint32_t) {
if (use_inst->opcode() == SpvOpLoad) {
is_used = true;
}
});
if (is_used) {
return false;
}
} else if (inst.opcode() == SpvOpPhi) {
if (inst.NumInOperands() != 2) {
return false;
}
} else if (inst.opcode() != SpvOpBranch) {
return false;
}
}
}
return true;
} // namespace opt
bool LoopFusion::ContainsBarriersOrFunctionCalls(ir::Loop* loop) {
for (const auto& block : loop->GetBlocks()) {
for (const auto& inst : *containing_function_->FindBlock(block)) {
auto opcode = inst.opcode();
if (opcode == SpvOpFunctionCall || opcode == SpvOpControlBarrier ||
opcode == SpvOpMemoryBarrier || opcode == SpvOpTypeNamedBarrier ||
opcode == SpvOpNamedBarrierInitialize ||
opcode == SpvOpMemoryNamedBarrier) {
return true;
}
}
}
return false;
}
bool LoopFusion::CheckInit() {
int64_t loop_0_init;
if (!loop_0_->GetInductionInitValue(induction_0_, &loop_0_init)) {
return false;
}
int64_t loop_1_init;
if (!loop_1_->GetInductionInitValue(induction_1_, &loop_1_init)) {
return false;
}
if (loop_0_init != loop_1_init) {
return false;
}
return true;
}
bool LoopFusion::CheckCondition() {
auto condition_0 = loop_0_->GetConditionInst();
auto condition_1 = loop_1_->GetConditionInst();
if (!loop_0_->IsSupportedCondition(condition_0->opcode()) ||
!loop_1_->IsSupportedCondition(condition_1->opcode())) {
return false;
}
if (condition_0->opcode() != condition_1->opcode()) {
return false;
}
for (uint32_t i = 0; i < condition_0->NumInOperandWords(); ++i) {
auto arg_0 = context_->get_def_use_mgr()->GetDef(
condition_0->GetSingleWordInOperand(i));
auto arg_1 = context_->get_def_use_mgr()->GetDef(
condition_1->GetSingleWordInOperand(i));
if (arg_0 == induction_0_ && arg_1 == induction_1_) {
continue;
}
if (arg_0 == induction_0_ && arg_1 != induction_1_) {
return false;
}
if (arg_1 == induction_1_ && arg_0 != induction_0_) {
return false;
}
if (arg_0 != arg_1) {
return false;
}
}
return true;
}
bool LoopFusion::CheckStep() {
auto scalar_analysis = context_->GetScalarEvolutionAnalysis();
SENode* induction_node_0 = scalar_analysis->SimplifyExpression(
scalar_analysis->AnalyzeInstruction(induction_0_));
if (!induction_node_0->AsSERecurrentNode()) {
return false;
}
SENode* induction_step_0 =
induction_node_0->AsSERecurrentNode()->GetCoefficient();
if (!induction_step_0->AsSEConstantNode()) {
return false;
}
SENode* induction_node_1 = scalar_analysis->SimplifyExpression(
scalar_analysis->AnalyzeInstruction(induction_1_));
if (!induction_node_1->AsSERecurrentNode()) {
return false;
}
SENode* induction_step_1 =
induction_node_1->AsSERecurrentNode()->GetCoefficient();
if (!induction_step_1->AsSEConstantNode()) {
return false;
}
if (*induction_step_0 != *induction_step_1) {
return false;
}
return true;
}
std::map<ir::Instruction*, std::vector<ir::Instruction*>>
LoopFusion::LocationToMemOps(const std::vector<ir::Instruction*>& mem_ops) {
std::map<ir::Instruction*, std::vector<ir::Instruction*>> location_map{};
for (auto instruction : mem_ops) {
auto access_location = context_->get_def_use_mgr()->GetDef(
instruction->GetSingleWordInOperand(0));
while (access_location->opcode() == SpvOpAccessChain) {
access_location = context_->get_def_use_mgr()->GetDef(
access_location->GetSingleWordInOperand(0));
}
location_map[access_location].push_back(instruction);
}
return location_map;
}
std::pair<std::vector<ir::Instruction*>, std::vector<ir::Instruction*>>
LoopFusion::GetLoadsAndStoresInLoop(ir::Loop* loop) {
std::vector<ir::Instruction*> loads{};
std::vector<ir::Instruction*> stores{};
for (auto block_id : loop->GetBlocks()) {
if (block_id == loop->GetLatchBlock()->id()) {
continue;
}
for (auto& instruction : *containing_function_->FindBlock(block_id)) {
if (instruction.opcode() == SpvOpLoad) {
loads.push_back(&instruction);
} else if (instruction.opcode() == SpvOpStore) {
stores.push_back(&instruction);
}
}
}
return std::make_pair(loads, stores);
}
bool LoopFusion::IsUsedInLoop(ir::Instruction* instruction, ir::Loop* loop) {
auto not_used = context_->get_def_use_mgr()->WhileEachUser(
instruction, [this, loop](ir::Instruction* user) {
auto block_id = context_->get_instr_block(user)->id();
return !loop->IsInsideLoop(block_id);
});
return !not_used;
}
bool LoopFusion::IsLegal() {
assert(AreCompatible() && "Fusion can't be legal, loops are not compatible.");
// Bail out if there are function calls as they could have side-effects that
// cause dependencies or if there are any barriers.
if (ContainsBarriersOrFunctionCalls(loop_0_) ||
ContainsBarriersOrFunctionCalls(loop_1_)) {
return false;
}
std::vector<ir::Instruction*> phi_instructions{};
loop_0_->GetInductionVariables(phi_instructions);
// Check no OpPhi in |loop_0_| is used in |loop_1_|.
for (auto phi_instruction : phi_instructions) {
if (IsUsedInLoop(phi_instruction, loop_1_)) {
return false;
}
}
// Check no LCSSA OpPhi in merge block of |loop_0_| is used in |loop_1_|.
auto phi_used = false;
loop_0_->GetMergeBlock()->ForEachPhiInst(
[this, &phi_used](ir::Instruction* phi_instruction) {
phi_used |= IsUsedInLoop(phi_instruction, loop_1_);
});
if (phi_used) {
return false;
}
// Grab loads & stores from both loops.
auto loads_stores_0 = GetLoadsAndStoresInLoop(loop_0_);
auto loads_stores_1 = GetLoadsAndStoresInLoop(loop_1_);
// Build memory location to operation maps.
auto load_locs_0 = LocationToMemOps(std::get<0>(loads_stores_0));
auto store_locs_0 = LocationToMemOps(std::get<1>(loads_stores_0));
auto load_locs_1 = LocationToMemOps(std::get<0>(loads_stores_1));
auto store_locs_1 = LocationToMemOps(std::get<1>(loads_stores_1));
// Get the locations accessed in both loops.
auto locations_0 = GetLocationsAccessed(store_locs_0, load_locs_0);
auto locations_1 = GetLocationsAccessed(store_locs_1, load_locs_1);
std::vector<ir::Instruction*> potential_clashes{};
std::set_intersection(std::begin(locations_0), std::end(locations_0),
std::begin(locations_1), std::end(locations_1),
std::back_inserter(potential_clashes));
// If the loops don't access the same variables, the fusion is legal.
if (potential_clashes.empty()) {
return true;
}
// Find variables that have at least one store.
std::vector<ir::Instruction*> potential_clashes_with_stores{};
for (auto location : potential_clashes) {
if (store_locs_0.find(location) != std::end(store_locs_0) ||
store_locs_1.find(location) != std::end(store_locs_1)) {
potential_clashes_with_stores.push_back(location);
}
}
// If there are only loads to the same variables, the fusion is legal.
if (potential_clashes_with_stores.empty()) {
return true;
}
// Else if loads and at least one store (across loops) to the same variable
// there is a potential dependence and we need to check the dependence
// distance.
// Find all the loops in this loop nest for the dependency analysis.
std::vector<const ir::Loop*> loops{};
// Find the parents.
for (auto current_loop = loop_0_; current_loop != nullptr;
current_loop = current_loop->GetParent()) {
loops.push_back(current_loop);
}
auto this_loop_position = loops.size() - 1;
std::reverse(std::begin(loops), std::end(loops));
// Find the children.
CollectChildren(loop_0_, &loops);
CollectChildren(loop_1_, &loops);
// Check that any dependes created are legal. That means the fused loops do
// not have any dependencies with dependence distance greater than 0 that did
// not exist in the original loops.
LoopDependenceAnalysis analysis(context_, loops);
analysis.GetScalarEvolution()->AddLoopsToPretendAreTheSame(
{loop_0_, loop_1_});
for (auto location : potential_clashes_with_stores) {
// Analyse dependences from |loop_0_| to |loop_1_|.
std::vector<DistanceVector> dependences;
// Read-After-Write.
GetDependences(&dependences, &analysis, store_locs_0[location],
load_locs_1[location], loops.size());
// Write-After-Read.
GetDependences(&dependences, &analysis, load_locs_0[location],
store_locs_1[location], loops.size());
// Write-After-Write.
GetDependences(&dependences, &analysis, store_locs_0[location],
store_locs_1[location], loops.size());
// Check that the induction variables either don't appear in the subscripts
// or the dependence distance is negative.
for (const auto& dependence : dependences) {
const auto& entry = dependence.GetEntries()[this_loop_position];
if ((entry.dependence_information ==
DistanceEntry::DependenceInformation::DISTANCE &&
entry.distance < 1) ||
(entry.dependence_information ==
DistanceEntry::DependenceInformation::IRRELEVANT)) {
continue;
} else {
return false;
}
}
}
return true;
}
void ReplacePhiParentWith(ir::Instruction* inst, uint32_t orig_block,
uint32_t new_block) {
if (inst->GetSingleWordInOperand(1) == orig_block) {
inst->SetInOperand(1, {new_block});
} else {
inst->SetInOperand(3, {new_block});
}
}
void LoopFusion::Fuse() {
assert(AreCompatible() && "Can't fuse, loops aren't compatible");
assert(IsLegal() && "Can't fuse, illegal");
// Save the pointers/ids, won't be found in the middle of doing modifications.
auto header_1 = loop_1_->GetHeaderBlock()->id();
auto condition_1 = loop_1_->FindConditionBlock()->id();
auto continue_1 = loop_1_->GetLatchBlock()->id();
auto continue_0 = loop_0_->GetLatchBlock()->id();
auto condition_block_of_0 = loop_0_->FindConditionBlock();
// Find the blocks whose branches need updating.
auto first_block_of_1 = &*(++containing_function_->FindBlock(condition_1));
auto last_block_of_1 = &*(--containing_function_->FindBlock(continue_1));
auto last_block_of_0 = &*(--containing_function_->FindBlock(continue_0));
// Update the branch for |last_block_of_loop_0| to go to |first_block_of_1|.
last_block_of_0->ForEachSuccessorLabel(
[first_block_of_1](uint32_t* succ) { *succ = first_block_of_1->id(); });
// Update the branch for the |last_block_of_loop_1| to go to the continue
// block of |loop_0_|.
last_block_of_1->ForEachSuccessorLabel(
[this](uint32_t* succ) { *succ = loop_0_->GetLatchBlock()->id(); });
// Update merge block id in the header of |loop_0_| to the merge block of
// |loop_1_|.
loop_0_->GetHeaderBlock()->ForEachInst([this](ir::Instruction* inst) {
if (inst->opcode() == SpvOpLoopMerge) {
inst->SetInOperand(0, {loop_1_->GetMergeBlock()->id()});
}
});
// Update condition branch target in |loop_0_| to the merge block of
// |loop_1_|.
condition_block_of_0->ForEachInst([this](ir::Instruction* inst) {
if (inst->opcode() == SpvOpBranchConditional) {
auto loop_0_merge_block_id = loop_0_->GetMergeBlock()->id();
if (inst->GetSingleWordInOperand(1) == loop_0_merge_block_id) {
inst->SetInOperand(1, {loop_1_->GetMergeBlock()->id()});
} else {
inst->SetInOperand(2, {loop_1_->GetMergeBlock()->id()});
}
}
});
// Move OpPhi instructions not corresponding to the induction variable from
// the header of |loop_1_| to the header of |loop_0_|.
std::vector<ir::Instruction*> instructions_to_move{};
for (auto& instruction : *loop_1_->GetHeaderBlock()) {
if (instruction.opcode() == SpvOpPhi && &instruction != induction_1_) {
instructions_to_move.push_back(&instruction);
}
}
for (auto& it : instructions_to_move) {
it->RemoveFromList();
it->InsertBefore(induction_0_);
}
// Update the OpPhi parents to the correct blocks in |loop_0_|.
loop_0_->GetHeaderBlock()->ForEachPhiInst([this](ir::Instruction* i) {
ReplacePhiParentWith(i, loop_1_->GetPreHeaderBlock()->id(),
loop_0_->GetPreHeaderBlock()->id());
ReplacePhiParentWith(i, loop_1_->GetLatchBlock()->id(),
loop_0_->GetLatchBlock()->id());
});
// Update instruction to block mapping & DefUseManager.
for (auto& phi_instruction : instructions_to_move) {
context_->set_instr_block(phi_instruction, loop_0_->GetHeaderBlock());
context_->get_def_use_mgr()->AnalyzeInstUse(phi_instruction);
}
// Replace the uses of the induction variable of |loop_1_| with that the
// induction variable of |loop_0_|.
context_->ReplaceAllUsesWith(induction_1_->result_id(),
induction_0_->result_id());
// Replace LCSSA OpPhi in merge block of |loop_0_|.
loop_0_->GetMergeBlock()->ForEachPhiInst(
[this](ir::Instruction* instruction) {
context_->ReplaceAllUsesWith(instruction->result_id(),
instruction->GetSingleWordInOperand(0));
});
// Update LCSSA OpPhi in merge block of |loop_1_|.
loop_1_->GetMergeBlock()->ForEachPhiInst(
[condition_block_of_0](ir::Instruction* instruction) {
instruction->SetInOperand(1, {condition_block_of_0->id()});
});
// Move the continue block of |loop_0_| after the last block of |loop_1_|.
containing_function_->MoveBasicBlockToAfter(continue_0, last_block_of_1);
// Gather all instructions to be killed from |loop_1_| (induction variable
// initialisation, header, condition and continue blocks).
std::vector<ir::Instruction*> instr_to_delete{};
AddInstructionsInBlock(&instr_to_delete, loop_1_->GetPreHeaderBlock());
AddInstructionsInBlock(&instr_to_delete, loop_1_->GetHeaderBlock());
AddInstructionsInBlock(&instr_to_delete, loop_1_->FindConditionBlock());
AddInstructionsInBlock(&instr_to_delete, loop_1_->GetLatchBlock());
// There was an additional empty block between the loops, kill that too.
if (loop_0_->GetMergeBlock() != loop_1_->GetPreHeaderBlock()) {
AddInstructionsInBlock(&instr_to_delete, loop_0_->GetMergeBlock());
}
// Update the CFG, so it wouldn't need invalidating.
auto cfg = context_->cfg();
cfg->ForgetBlock(loop_1_->GetPreHeaderBlock());
cfg->ForgetBlock(loop_1_->GetHeaderBlock());
cfg->ForgetBlock(loop_1_->FindConditionBlock());
cfg->ForgetBlock(loop_1_->GetLatchBlock());
if (loop_0_->GetMergeBlock() != loop_1_->GetPreHeaderBlock()) {
cfg->ForgetBlock(loop_0_->GetMergeBlock());
}
cfg->RemoveEdge(last_block_of_0->id(), loop_0_->GetLatchBlock()->id());
cfg->AddEdge(last_block_of_0->id(), first_block_of_1->id());
cfg->AddEdge(last_block_of_1->id(), loop_0_->GetLatchBlock()->id());
cfg->AddEdge(loop_0_->GetLatchBlock()->id(), loop_1_->GetHeaderBlock()->id());
cfg->AddEdge(condition_block_of_0->id(), loop_1_->GetMergeBlock()->id());
// Update DefUseManager.
auto def_use_mgr = context_->get_def_use_mgr();
// Uses of labels that are in updated branches need analysing.
def_use_mgr->AnalyzeInstUse(last_block_of_0->terminator());
def_use_mgr->AnalyzeInstUse(last_block_of_1->terminator());
def_use_mgr->AnalyzeInstUse(loop_0_->GetHeaderBlock()->GetLoopMergeInst());
def_use_mgr->AnalyzeInstUse(condition_block_of_0->terminator());
// Update the LoopDescriptor, so it wouldn't need invalidating.
auto ld = context_->GetLoopDescriptor(containing_function_);
// Create a copy, so the iterator wouldn't be invalidated.
std::vector<ir::Loop*> loops_to_add_remove{};
for (auto child_loop : *loop_1_) {
loops_to_add_remove.push_back(child_loop);
}
for (auto child_loop : loops_to_add_remove) {
loop_1_->RemoveChildLoop(child_loop);
loop_0_->AddNestedLoop(child_loop);
}
auto loop_1_blocks = loop_1_->GetBlocks();
for (auto block : loop_1_blocks) {
loop_1_->RemoveBasicBlock(block);
if (block != header_1 && block != condition_1 && block != continue_1) {
loop_0_->AddBasicBlock(block);
if ((*ld)[block] == loop_1_) {
ld->SetBasicBlockToLoop(block, loop_0_);
}
}
if ((*ld)[block] == loop_1_) {
ld->ForgetBasicBlock(block);
}
}
loop_1_->RemoveBasicBlock(loop_1_->GetPreHeaderBlock()->id());
ld->ForgetBasicBlock(loop_1_->GetPreHeaderBlock()->id());
if (loop_0_->GetMergeBlock() != loop_1_->GetPreHeaderBlock()) {
loop_0_->RemoveBasicBlock(loop_0_->GetMergeBlock()->id());
ld->ForgetBasicBlock(loop_0_->GetMergeBlock()->id());
}
loop_0_->SetMergeBlock(loop_1_->GetMergeBlock());
loop_1_->ClearBlocks();
ld->RemoveLoop(loop_1_);
// Kill unnessecary instructions and remove all empty blocks.
for (auto inst : instr_to_delete) {
context_->KillInst(inst);
}
containing_function_->RemoveEmptyBlocks();
// Invalidate analyses.
context_->InvalidateAnalysesExceptFor(
ir::IRContext::Analysis::kAnalysisInstrToBlockMapping |
ir::IRContext::Analysis::kAnalysisLoopAnalysis |
ir::IRContext::Analysis::kAnalysisDefUse |
ir::IRContext::Analysis::kAnalysisCFG);
}
} // namespace opt
} // namespace spvtools

114
source/opt/loop_fusion.h Normal file
View File

@ -0,0 +1,114 @@
// Copyright (c) 2018 Google LLC.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef SOURCE_OPT_LOOP_FUSION_H_
#define SOURCE_OPT_LOOP_FUSION_H_
#include <map>
#include <set>
#include <vector>
#include "opt/ir_context.h"
#include "opt/loop_descriptor.h"
#include "opt/loop_utils.h"
#include "opt/scalar_analysis.h"
namespace spvtools {
namespace opt {
class LoopFusion {
public:
LoopFusion(ir::IRContext* context, ir::Loop* loop_0, ir::Loop* loop_1)
: context_(context),
loop_0_(loop_0),
loop_1_(loop_1),
containing_function_(loop_0->GetHeaderBlock()->GetParent()) {}
// Checks if the |loop_0| and |loop_1| are compatible for fusion.
// That means:
// * they both have one induction variable
// * they have the same upper and lower bounds
// - same inital value
// - same condition
// * they have the same update step
// * they are adjacent, with |loop_0| appearing before |loop_1|
// * there are no break/continue in either of them
// * they both have pre-header blocks (required for ScalarEvolutionAnalysis
// and dependence checking).
bool AreCompatible();
// Checks if compatible |loop_0| and |loop_1| are legal to fuse.
// * fused loops do not have any dependencies with dependence distance greater
// than 0 that did not exist in the original loops.
// * there are no function calls in the loops (could have side-effects)
bool IsLegal();
// Perform the actual fusion of |loop_0_| and |loop_1_|. The loops have to be
// compatible and the fusion has to be legal.
void Fuse();
private:
// Check that the initial values are the same.
bool CheckInit();
// Check that the conditions are the same.
bool CheckCondition();
// Check that the steps are the same.
bool CheckStep();
// Returns |true| if |instruction| is used in the continue or condition block
// of |loop|.
bool UsedInContinueOrConditionBlock(ir::Instruction* instruction,
ir::Loop* loop);
// Remove entries in |instructions| that are not used in the continue or
// condition block of |loop|.
void RemoveIfNotUsedContinueOrConditionBlock(
std::vector<ir::Instruction*>* instructions, ir::Loop* loop);
// Returns |true| if |instruction| is used in |loop|.
bool IsUsedInLoop(ir::Instruction* instruction, ir::Loop* loop);
// Returns |true| if |loop| has at least one barrier or function call.
bool ContainsBarriersOrFunctionCalls(ir::Loop* loop);
// Get all instructions in the |loop| (except in the latch block) that have
// the opcode |opcode|.
std::pair<std::vector<ir::Instruction*>, std::vector<ir::Instruction*>>
GetLoadsAndStoresInLoop(ir::Loop* loop);
// Given a vector of memory operations (OpLoad/OpStore), constructs a map from
// variables to the loads/stores that those variables.
std::map<ir::Instruction*, std::vector<ir::Instruction*>> LocationToMemOps(
const std::vector<ir::Instruction*>& mem_ops);
ir::IRContext* context_;
// The original loops to be fused.
ir::Loop* loop_0_;
ir::Loop* loop_1_;
// The function that contains |loop_0_| and |loop_1_|.
ir::Function* containing_function_ = nullptr;
// The induction variables for |loop_0_| and |loop_1_|.
ir::Instruction* induction_0_ = nullptr;
ir::Instruction* induction_1_ = nullptr;
};
} // namespace opt
} // namespace spvtools
#endif // SOURCE_OPT_LOOP_FUSION_H_

70
source/opt/loop_fusion_pass.cpp Normal file
View File

@ -0,0 +1,70 @@
// Copyright (c) 2018 Google LLC.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "opt/loop_fusion_pass.h"
#include "opt/ir_context.h"
#include "opt/loop_descriptor.h"
#include "opt/loop_fusion.h"
#include "opt/register_pressure.h"
namespace spvtools {
namespace opt {
Pass::Status LoopFusionPass::Process(ir::IRContext* c) {
bool modified = false;
ir::Module* module = c->module();
// Process each function in the module
for (ir::Function& f : *module) {
modified |= ProcessFunction(&f);
}
return modified ? Status::SuccessWithChange : Status::SuccessWithoutChange;
}
bool LoopFusionPass::ProcessFunction(ir::Function* function) {
auto c = function->context();
ir::LoopDescriptor& ld = *c->GetLoopDescriptor(function);
// If a loop doesn't have a preheader needs then it needs to be created. Make
// sure to return Status::SuccessWithChange in that case.
auto modified = ld.CreatePreHeaderBlocksIfMissing();
// TODO(tremmelg): Could the only loop that |loop| could possibly be fused be
// picked out so don't have to check every loop
for (auto& loop_0 : ld) {
for (auto& loop_1 : ld) {
LoopFusion fusion(c, &loop_0, &loop_1);
if (fusion.AreCompatible() && fusion.IsLegal()) {
RegisterLiveness liveness(c, function);
RegisterLiveness::RegionRegisterLiveness reg_pressure{};
liveness.SimulateFusion(loop_0, loop_1, &reg_pressure);
if (reg_pressure.used_registers_ <= max_registers_per_loop_) {
fusion.Fuse();
// Recurse, as the current iterators will have been invalidated.
ProcessFunction(function);
return true;
}
}
}
}
return modified;
}
} // namespace opt
} // namespace spvtools

51
source/opt/loop_fusion_pass.h Normal file
View File

@ -0,0 +1,51 @@
// Copyright (c) 2018 Google LLC.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef SOURCE_OPT_LOOP_FUSION_PASS_H_
#define SOURCE_OPT_LOOP_FUSION_PASS_H_
#include "opt/pass.h"
namespace spvtools {
namespace opt {
// Implements a loop fusion pass.
// This pass will look for adjacent loops that are compatible and legal to be
// fused. It will fuse all such loops as long as the register usage for the
// fused loop stays under the threshold defined by |max_registers_per_loop|.
class LoopFusionPass : public Pass {
public:
explicit LoopFusionPass(size_t max_registers_per_loop)
: Pass(), max_registers_per_loop_(max_registers_per_loop) {}
const char* name() const override { return "loop-fusion"; }
// Processes the given |module|. Returns Status::Failure if errors occur when
// processing. Returns the corresponding Status::Success if processing is
// succesful to indicate whether changes have been made to the modue.
Status Process(ir::IRContext* c) override;
private:
// Fuse loops in |function| if compatible, legal and the fused loop won't use
// too many registers.
bool ProcessFunction(ir::Function* function);
// The maximum number of registers a fused loop is allowed to use.
size_t max_registers_per_loop_;
};
} // namespace opt
} // namespace spvtools
#endif // SOURCE_OPT_LOOP_FUSION_PASS_H_

5
source/opt/optimizer.cpp
View File

@ -384,6 +384,11 @@ Optimizer::PassToken CreateLoopFissionPass(size_t threshold) {
MakeUnique<opt::LoopFissionPass>(threshold));
}
Optimizer::PassToken CreateLoopFusionPass(size_t max_registers_per_loop) {
return MakeUnique<Optimizer::PassToken::Impl>(
MakeUnique<opt::LoopFusionPass>(max_registers_per_loop));
}
Optimizer::PassToken CreateLoopInvariantCodeMotionPass() {
return MakeUnique<Optimizer::PassToken::Impl>(MakeUnique<opt::LICMPass>());
}

1
source/opt/passes.h
View File

@ -43,6 +43,7 @@
#include "local_single_store_elim_pass.h"
#include "local_ssa_elim_pass.h"
#include "loop_fission.h"
#include "loop_fusion_pass.h"
#include "loop_peeling.h"
#include "loop_unroller.h"
#include "loop_unswitch_pass.h"

21
source/opt/scalar_analysis.cpp
View File

@ -49,7 +49,7 @@ namespace opt {
uint32_t SENode::NumberOfNodes = 0;
ScalarEvolutionAnalysis::ScalarEvolutionAnalysis(ir::IRContext* context)
: context_(context) {
: context_(context), pretend_equal_{} {
// Create and cached the CantComputeNode.
cached_cant_compute_ =
GetCachedOrAdd(std::unique_ptr<SECantCompute>(new SECantCompute(this)));
@ -80,7 +80,15 @@ SENode* ScalarEvolutionAnalysis::CreateRecurrentExpression(
if (offset->IsCantCompute() || coefficient->IsCantCompute())
return CreateCantComputeNode();
std::unique_ptr<SERecurrentNode> phi_node{new SERecurrentNode(this, loop)};
const ir::Loop* loop_to_use = nullptr;
if (pretend_equal_[loop]) {
loop_to_use = pretend_equal_[loop];
} else {
loop_to_use = loop;
}
std::unique_ptr<SERecurrentNode> phi_node{
new SERecurrentNode(this, loop_to_use)};
phi_node->AddOffset(offset);
phi_node->AddCoefficient(coefficient);
@ -270,7 +278,14 @@ SENode* ScalarEvolutionAnalysis::AnalyzePhiInstruction(
loop->GetHeaderBlock() != basic_block)
return recurrent_node_map_[phi] = CreateCantComputeNode();
std::unique_ptr<SERecurrentNode> phi_node{new SERecurrentNode(this, loop)};
const ir::Loop* loop_to_use = nullptr;
if (pretend_equal_[loop]) {
loop_to_use = pretend_equal_[loop];
} else {
loop_to_use = loop;
}
std::unique_ptr<SERecurrentNode> phi_node{
new SERecurrentNode(this, loop_to_use)};
// We add the node to this map to allow it to be returned before the node is
// fully built. This is needed as the subsequent call to AnalyzeInstruction

12
source/opt/scalar_analysis.h
View File

@ -120,6 +120,14 @@ class ScalarEvolutionAnalysis {
SENode* UpdateChildNode(SENode* parent, SENode* child, SENode* new_child);
// The loops in |loop_pair| will be considered the same when constructing
// SERecurrentNode objects. This enables analysing dependencies that will be
// created during loop fusion.
void AddLoopsToPretendAreTheSame(
const std::pair<const ir::Loop*, const ir::Loop*>& loop_pair) {
pretend_equal_[std::get<1>(loop_pair)] = std::get<0>(loop_pair);
}
private:
SENode* AnalyzeConstant(const ir::Instruction* inst);
@ -158,6 +166,10 @@ class ScalarEvolutionAnalysis {
// managed by they set.
std::unordered_set<std::unique_ptr<SENode>, SENodeHash, NodePointersEquality>
node_cache_;
// Loops that should be considered the same for performing analysis for loop
// fusion.
std::map<const ir::Loop*, const ir::Loop*> pretend_equal_;
};
// Wrapping class to manipulate SENode pointer using + - * / operators.

22
test/opt/loop_optimizations/CMakeLists.txt
View File

@ -115,4 +115,26 @@ add_spvtools_unittest(TARGET loop_fission
LIBS SPIRV-Tools-opt
)
add_spvtools_unittest(TARGET fusion_compatibility
SRCS ../function_utils.h
fusion_compatibility.cpp
LIBS SPIRV-Tools-opt
)
add_spvtools_unittest(TARGET fusion_illegal
SRCS ../function_utils.h
fusion_illegal.cpp
LIBS SPIRV-Tools-opt
)
add_spvtools_unittest(TARGET fusion_legal
SRCS ../function_utils.h
fusion_legal.cpp
LIBS SPIRV-Tools-opt
)
add_spvtools_unittest(TARGET fusion_pass
SRCS ../function_utils.h
fusion_pass.cpp
LIBS SPIRV-Tools-opt
)

1790
test/opt/loop_optimizations/fusion_compatibility.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

1591
test/opt/loop_optimizations/fusion_illegal.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

4581
test/opt/loop_optimizations/fusion_legal.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

721
test/opt/loop_optimizations/fusion_pass.cpp Normal file
View File

@ -0,0 +1,721 @@
// Copyright (c) 2018 Google LLC.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <gmock/gmock.h>
#ifdef SPIRV_EFFCEE
#include "effcee/effcee.h"
#endif
#include "../pass_fixture.h"
namespace {
using namespace spvtools;
using FusionPassTest = PassTest<::testing::Test>;
#ifdef SPIRV_EFFCEE
/*
Generated from the following GLSL + --eliminate-local-multi-store
#version 440 core
void main() {
int[10] a;
int[10] b;
for (int i = 0; i < 10; i++) {
a[i] = a[i]*2;
}
for (int i = 0; i < 10; i++) {
b[i] = a[i]+2;
}
}
*/
TEST_F(FusionPassTest, SimpleFusion) {
const std::string text = R"(
; CHECK: OpPhi
; CHECK: OpLoad
; CHECK: OpStore
; CHECK-NOT: OpPhi
; CHECK: OpLoad
; CHECK: OpStore
OpCapability Shader
%1 = OpExtInstImport "GLSL.std.450"
OpMemoryModel Logical GLSL450
OpEntryPoint Fragment %4 "main"
OpExecutionMode %4 OriginUpperLeft
OpSource GLSL 440
OpName %4 "main"
OpName %8 "i"
OpName %23 "a"
OpName %34 "i"
OpName %42 "b"
%2 = OpTypeVoid
%3 = OpTypeFunction %2
%6 = OpTypeInt 32 1
%7 = OpTypePointer Function %6
%9 = OpConstant %6 0
%16 = OpConstant %6 10
%17 = OpTypeBool
%19 = OpTypeInt 32 0
%20 = OpConstant %19 10
%21 = OpTypeArray %6 %20
%22 = OpTypePointer Function %21
%28 = OpConstant %6 2
%32 = OpConstant %6 1
%4 = OpFunction %2 None %3
%5 = OpLabel
%8 = OpVariable %7 Function
%23 = OpVariable %22 Function
%34 = OpVariable %7 Function
%42 = OpVariable %22 Function
OpStore %8 %9
OpBranch %10
%10 = OpLabel
%51 = OpPhi %6 %9 %5 %33 %13
OpLoopMerge %12 %13 None
OpBranch %14
%14 = OpLabel
%18 = OpSLessThan %17 %51 %16
OpBranchConditional %18 %11 %12
%11 = OpLabel
%26 = OpAccessChain %7 %23 %51
%27 = OpLoad %6 %26
%29 = OpIMul %6 %27 %28
%30 = OpAccessChain %7 %23 %51
OpStore %30 %29
OpBranch %13
%13 = OpLabel
%33 = OpIAdd %6 %51 %32
OpStore %8 %33
OpBranch %10
%12 = OpLabel
OpStore %34 %9
OpBranch %35
%35 = OpLabel
%52 = OpPhi %6 %9 %12 %50 %38
OpLoopMerge %37 %38 None
OpBranch %39
%39 = OpLabel
%41 = OpSLessThan %17 %52 %16
OpBranchConditional %41 %36 %37
%36 = OpLabel
%45 = OpAccessChain %7 %23 %52
%46 = OpLoad %6 %45
%47 = OpIAdd %6 %46 %28
%48 = OpAccessChain %7 %42 %52
OpStore %48 %47
OpBranch %38
%38 = OpLabel
%50 = OpIAdd %6 %52 %32
OpStore %34 %50
OpBranch %35
%37 = OpLabel
OpReturn
OpFunctionEnd
)";
SinglePassRunAndMatch<opt::LoopFusionPass>(text, true, 20);
}
/*
Generated from the following GLSL + --eliminate-local-multi-store
#version 440 core
void main() {
int[10] a;
int[10] b;
int[10] c;
for (int i = 0; i < 10; i++) {
a[i] = b[i] + 1;
}
for (int i = 0; i < 10; i++) {
c[i] = a[i] + 2;
}
for (int i = 0; i < 10; i++) {
b[i] = c[i] + 10;
}
}
*/
TEST_F(FusionPassTest, ThreeLoopsFused) {
const std::string text = R"(
; CHECK: OpPhi
; CHECK: OpLoad
; CHECK: OpStore
; CHECK-NOT: OpPhi
; CHECK: OpLoad
; CHECK: OpStore
; CHECK-NOT: OpPhi
; CHECK: OpLoad
; CHECK: OpStore
OpCapability Shader
%1 = OpExtInstImport "GLSL.std.450"
OpMemoryModel Logical GLSL450
OpEntryPoint Fragment %4 "main"
OpExecutionMode %4 OriginUpperLeft
OpSource GLSL 440
OpName %4 "main"
OpName %8 "i"
OpName %23 "a"
OpName %25 "b"
OpName %34 "i"
OpName %42 "c"
OpName %52 "i"
%2 = OpTypeVoid
%3 = OpTypeFunction %2
%6 = OpTypeInt 32 1
%7 = OpTypePointer Function %6
%9 = OpConstant %6 0
%16 = OpConstant %6 10
%17 = OpTypeBool
%19 = OpTypeInt 32 0
%20 = OpConstant %19 10
%21 = OpTypeArray %6 %20
%22 = OpTypePointer Function %21
%29 = OpConstant %6 1
%47 = OpConstant %6 2
%4 = OpFunction %2 None %3
%5 = OpLabel
%8 = OpVariable %7 Function
%23 = OpVariable %22 Function
%25 = OpVariable %22 Function
%34 = OpVariable %7 Function
%42 = OpVariable %22 Function
%52 = OpVariable %7 Function
OpStore %8 %9
OpBranch %10
%10 = OpLabel
%68 = OpPhi %6 %9 %5 %33 %13
OpLoopMerge %12 %13 None
OpBranch %14
%14 = OpLabel
%18 = OpSLessThan %17 %68 %16
OpBranchConditional %18 %11 %12
%11 = OpLabel
%27 = OpAccessChain %7 %25 %68
%28 = OpLoad %6 %27
%30 = OpIAdd %6 %28 %29
%31 = OpAccessChain %7 %23 %68
OpStore %31 %30
OpBranch %13
%13 = OpLabel
%33 = OpIAdd %6 %68 %29
OpStore %8 %33
OpBranch %10
%12 = OpLabel
OpStore %34 %9
OpBranch %35
%35 = OpLabel
%69 = OpPhi %6 %9 %12 %51 %38
OpLoopMerge %37 %38 None
OpBranch %39
%39 = OpLabel
%41 = OpSLessThan %17 %69 %16
OpBranchConditional %41 %36 %37
%36 = OpLabel
%45 = OpAccessChain %7 %23 %69
%46 = OpLoad %6 %45
%48 = OpIAdd %6 %46 %47
%49 = OpAccessChain %7 %42 %69
OpStore %49 %48
OpBranch %38
%38 = OpLabel
%51 = OpIAdd %6 %69 %29
OpStore %34 %51
OpBranch %35
%37 = OpLabel
OpStore %52 %9
OpBranch %53
%53 = OpLabel
%70 = OpPhi %6 %9 %37 %67 %56
OpLoopMerge %55 %56 None
OpBranch %57
%57 = OpLabel
%59 = OpSLessThan %17 %70 %16
OpBranchConditional %59 %54 %55
%54 = OpLabel
%62 = OpAccessChain %7 %42 %70
%63 = OpLoad %6 %62
%64 = OpIAdd %6 %63 %16
%65 = OpAccessChain %7 %25 %70
OpStore %65 %64
OpBranch %56
%56 = OpLabel
%67 = OpIAdd %6 %70 %29
OpStore %52 %67
OpBranch %53
%55 = OpLabel
OpReturn
OpFunctionEnd
)";
SinglePassRunAndMatch<opt::LoopFusionPass>(text, true, 20);
}
/*
Generated from the following GLSL + --eliminate-local-multi-store
#version 440 core
void main() {
int[10][10] a;
int[10][10] b;
int[10][10] c;
// Legal both
for (int i = 0; i < 10; i++) {
for (int j = 0; j < 10; j++) {
c[i][j] = a[i][j] + 2;
}
}
for (int i = 0; i < 10; i++) {
for (int j = 0; j < 10; j++) {
b[i][j] = c[i][j] + 10;
}
}
}
*/
TEST_F(FusionPassTest, NestedLoopsFused) {
const std::string text = R"(
; CHECK: OpPhi
; CHECK: OpPhi
; CHECK: OpLoad
; CHECK: OpStore
; CHECK-NOT: OpPhi
; CHECK: OpLoad
; CHECK: OpStore
OpCapability Shader
%1 = OpExtInstImport "GLSL.std.450"
OpMemoryModel Logical GLSL450
OpEntryPoint Fragment %4 "main"
OpExecutionMode %4 OriginUpperLeft
OpSource GLSL 440
OpName %4 "main"
OpName %8 "i"
OpName %19 "j"
OpName %32 "c"
OpName %35 "a"
OpName %48 "i"
OpName %56 "j"
OpName %64 "b"
%2 = OpTypeVoid
%3 = OpTypeFunction %2
%6 = OpTypeInt 32 1
%7 = OpTypePointer Function %6
%9 = OpConstant %6 0
%16 = OpConstant %6 10
%17 = OpTypeBool
%27 = OpTypeInt 32 0
%28 = OpConstant %27 10
%29 = OpTypeArray %6 %28
%30 = OpTypeArray %29 %28
%31 = OpTypePointer Function %30
%40 = OpConstant %6 2
%44 = OpConstant %6 1
%4 = OpFunction %2 None %3
%5 = OpLabel
%8 = OpVariable %7 Function
%19 = OpVariable %7 Function
%32 = OpVariable %31 Function
%35 = OpVariable %31 Function
%48 = OpVariable %7 Function
%56 = OpVariable %7 Function
%64 = OpVariable %31 Function
OpStore %8 %9
OpBranch %10
%10 = OpLabel
%77 = OpPhi %6 %9 %5 %47 %13
OpLoopMerge %12 %13 None
OpBranch %14
%14 = OpLabel
%18 = OpSLessThan %17 %77 %16
OpBranchConditional %18 %11 %12
%11 = OpLabel
OpStore %19 %9
OpBranch %20
%20 = OpLabel
%81 = OpPhi %6 %9 %11 %45 %23
OpLoopMerge %22 %23 None
OpBranch %24
%24 = OpLabel
%26 = OpSLessThan %17 %81 %16
OpBranchConditional %26 %21 %22
%21 = OpLabel
%38 = OpAccessChain %7 %35 %77 %81
%39 = OpLoad %6 %38
%41 = OpIAdd %6 %39 %40
%42 = OpAccessChain %7 %32 %77 %81
OpStore %42 %41
OpBranch %23
%23 = OpLabel
%45 = OpIAdd %6 %81 %44
OpStore %19 %45
OpBranch %20
%22 = OpLabel
OpBranch %13
%13 = OpLabel
%47 = OpIAdd %6 %77 %44
OpStore %8 %47
OpBranch %10
%12 = OpLabel
OpStore %48 %9
OpBranch %49
%49 = OpLabel
%78 = OpPhi %6 %9 %12 %76 %52
OpLoopMerge %51 %52 None
OpBranch %53
%53 = OpLabel
%55 = OpSLessThan %17 %78 %16
OpBranchConditional %55 %50 %51
%50 = OpLabel
OpStore %56 %9
OpBranch %57
%57 = OpLabel
%79 = OpPhi %6 %9 %50 %74 %60
OpLoopMerge %59 %60 None
OpBranch %61
%61 = OpLabel
%63 = OpSLessThan %17 %79 %16
OpBranchConditional %63 %58 %59
%58 = OpLabel
%69 = OpAccessChain %7 %32 %78 %79
%70 = OpLoad %6 %69
%71 = OpIAdd %6 %70 %16
%72 = OpAccessChain %7 %64 %78 %79
OpStore %72 %71
OpBranch %60
%60 = OpLabel
%74 = OpIAdd %6 %79 %44
OpStore %56 %74
OpBranch %57
%59 = OpLabel
OpBranch %52
%52 = OpLabel
%76 = OpIAdd %6 %78 %44
OpStore %48 %76
OpBranch %49
%51 = OpLabel
OpReturn
OpFunctionEnd
)";
SinglePassRunAndMatch<opt::LoopFusionPass>(text, true, 20);
}
/*
Generated from the following GLSL + --eliminate-local-multi-store
#version 440 core
void main() {
// Can't fuse, different step
for (int i = 0; i < 10; i++) {}
for (int j = 0; j < 10; j=j+2) {}
}
*/
TEST_F(FusionPassTest, Incompatible) {
const std::string text = R"(
; CHECK: OpPhi
; CHECK-NEXT: OpLoopMerge
; CHECK: OpPhi
; CHECK-NEXT: OpLoopMerge
OpCapability Shader
%1 = OpExtInstImport "GLSL.std.450"
OpMemoryModel Logical GLSL450
OpEntryPoint Fragment %4 "main"
OpExecutionMode %4 OriginUpperLeft
OpSource GLSL 440
OpName %4 "main"
OpName %8 "i"
OpName %22 "j"
%2 = OpTypeVoid
%3 = OpTypeFunction %2
%6 = OpTypeInt 32 1
%7 = OpTypePointer Function %6
%9 = OpConstant %6 0
%16 = OpConstant %6 10
%17 = OpTypeBool
%20 = OpConstant %6 1
%31 = OpConstant %6 2
%4 = OpFunction %2 None %3
%5 = OpLabel
%8 = OpVariable %7 Function
%22 = OpVariable %7 Function
OpStore %8 %9
OpBranch %10
%10 = OpLabel
%33 = OpPhi %6 %9 %5 %21 %13
OpLoopMerge %12 %13 None
OpBranch %14
%14 = OpLabel
%18 = OpSLessThan %17 %33 %16
OpBranchConditional %18 %11 %12
%11 = OpLabel
OpBranch %13
%13 = OpLabel
%21 = OpIAdd %6 %33 %20
OpStore %8 %21
OpBranch %10
%12 = OpLabel
OpStore %22 %9
OpBranch %23
%23 = OpLabel
%34 = OpPhi %6 %9 %12 %32 %26
OpLoopMerge %25 %26 None
OpBranch %27
%27 = OpLabel
%29 = OpSLessThan %17 %34 %16
OpBranchConditional %29 %24 %25
%24 = OpLabel
OpBranch %26
%26 = OpLabel
%32 = OpIAdd %6 %34 %31
OpStore %22 %32
OpBranch %23
%25 = OpLabel
OpReturn
OpFunctionEnd
)";
SinglePassRunAndMatch<opt::LoopFusionPass>(text, true, 20);
}
/*
Generated from the following GLSL + --eliminate-local-multi-store
#version 440 core
void main() {
int[10] a;
int[10] b;
int[10] c;
// Illegal, loop-independent dependence will become a
// backward loop-carried antidependence
for (int i = 0; i < 10; i++) {
a[i] = b[i] + 1;
}
for (int i = 0; i < 10; i++) {
c[i] = a[i+1] + 2;
}
}
*/
TEST_F(FusionPassTest, Illegal) {
std::string text = R"(
; CHECK: OpPhi
; CHECK-NEXT: OpLoopMerge
; CHECK: OpLoad
; CHECK: OpStore
; CHECK: OpPhi
; CHECK-NEXT: OpLoopMerge
; CHECK: OpLoad
; CHECK: OpStore
OpCapability Shader
%1 = OpExtInstImport "GLSL.std.450"
OpMemoryModel Logical GLSL450
OpEntryPoint Fragment %4 "main"
OpExecutionMode %4 OriginUpperLeft
OpSource GLSL 440
OpName %4 "main"
OpName %8 "i"
OpName %23 "a"
OpName %25 "b"
OpName %34 "i"
OpName %42 "c"
%2 = OpTypeVoid
%3 = OpTypeFunction %2
%6 = OpTypeInt 32 1
%7 = OpTypePointer Function %6
%9 = OpConstant %6 0
%16 = OpConstant %6 10
%17 = OpTypeBool
%19 = OpTypeInt 32 0
%20 = OpConstant %19 10
%21 = OpTypeArray %6 %20
%22 = OpTypePointer Function %21
%29 = OpConstant %6 1
%48 = OpConstant %6 2
%4 = OpFunction %2 None %3
%5 = OpLabel
%8 = OpVariable %7 Function
%23 = OpVariable %22 Function
%25 = OpVariable %22 Function
%34 = OpVariable %7 Function
%42 = OpVariable %22 Function
OpStore %8 %9
OpBranch %10
%10 = OpLabel
%53 = OpPhi %6 %9 %5 %33 %13
OpLoopMerge %12 %13 None
OpBranch %14
%14 = OpLabel
%18 = OpSLessThan %17 %53 %16
OpBranchConditional %18 %11 %12
%11 = OpLabel
%27 = OpAccessChain %7 %25 %53
%28 = OpLoad %6 %27
%30 = OpIAdd %6 %28 %29
%31 = OpAccessChain %7 %23 %53
OpStore %31 %30
OpBranch %13
%13 = OpLabel
%33 = OpIAdd %6 %53 %29
OpStore %8 %33
OpBranch %10
%12 = OpLabel
OpStore %34 %9
OpBranch %35
%35 = OpLabel
%54 = OpPhi %6 %9 %12 %52 %38
OpLoopMerge %37 %38 None
OpBranch %39
%39 = OpLabel
%41 = OpSLessThan %17 %54 %16
OpBranchConditional %41 %36 %37
%36 = OpLabel
%45 = OpIAdd %6 %54 %29
%46 = OpAccessChain %7 %23 %45
%47 = OpLoad %6 %46
%49 = OpIAdd %6 %47 %48
%50 = OpAccessChain %7 %42 %54
OpStore %50 %49
OpBranch %38
%38 = OpLabel
%52 = OpIAdd %6 %54 %29
OpStore %34 %52
OpBranch %35
%37 = OpLabel
OpReturn
OpFunctionEnd
)";
SinglePassRunAndMatch<opt::LoopFusionPass>(text, true, 20);
}
/*
Generated from the following GLSL + --eliminate-local-multi-store
#version 440 core
void main() {
int[10] a;
int[10] b;
for (int i = 0; i < 10; i++) {
a[i] = a[i]*2;
}
for (int i = 0; i < 10; i++) {
b[i] = a[i]+2;
}
}
*/
TEST_F(FusionPassTest, TooManyRegisters) {
const std::string text = R"(
; CHECK: OpPhi
; CHECK-NEXT: OpLoopMerge
; CHECK: OpLoad
; CHECK: OpStore
; CHECK: OpPhi
; CHECK-NEXT: OpLoopMerge
; CHECK: OpLoad
; CHECK: OpStore
OpCapability Shader
%1 = OpExtInstImport "GLSL.std.450"
OpMemoryModel Logical GLSL450
OpEntryPoint Fragment %4 "main"
OpExecutionMode %4 OriginUpperLeft
OpSource GLSL 440
OpName %4 "main"
OpName %8 "i"
OpName %23 "a"
OpName %34 "i"
OpName %42 "b"
%2 = OpTypeVoid
%3 = OpTypeFunction %2
%6 = OpTypeInt 32 1
%7 = OpTypePointer Function %6
%9 = OpConstant %6 0
%16 = OpConstant %6 10
%17 = OpTypeBool
%19 = OpTypeInt 32 0
%20 = OpConstant %19 10
%21 = OpTypeArray %6 %20
%22 = OpTypePointer Function %21
%28 = OpConstant %6 2
%32 = OpConstant %6 1
%4 = OpFunction %2 None %3
%5 = OpLabel
%8 = OpVariable %7 Function
%23 = OpVariable %22 Function
%34 = OpVariable %7 Function
%42 = OpVariable %22 Function
OpStore %8 %9
OpBranch %10
%10 = OpLabel
%51 = OpPhi %6 %9 %5 %33 %13
OpLoopMerge %12 %13 None
OpBranch %14
%14 = OpLabel
%18 = OpSLessThan %17 %51 %16
OpBranchConditional %18 %11 %12
%11 = OpLabel
%26 = OpAccessChain %7 %23 %51
%27 = OpLoad %6 %26
%29 = OpIMul %6 %27 %28
%30 = OpAccessChain %7 %23 %51
OpStore %30 %29
OpBranch %13
%13 = OpLabel
%33 = OpIAdd %6 %51 %32
OpStore %8 %33
OpBranch %10
%12 = OpLabel
OpStore %34 %9
OpBranch %35
%35 = OpLabel
%52 = OpPhi %6 %9 %12 %50 %38
OpLoopMerge %37 %38 None
OpBranch %39
%39 = OpLabel
%41 = OpSLessThan %17 %52 %16
OpBranchConditional %41 %36 %37
%36 = OpLabel
%45 = OpAccessChain %7 %23 %52
%46 = OpLoad %6 %45
%47 = OpIAdd %6 %46 %28
%48 = OpAccessChain %7 %42 %52
OpStore %48 %47
OpBranch %38
%38 = OpLabel
%50 = OpIAdd %6 %52 %32
OpStore %34 %50
OpBranch %35
%37 = OpLabel
OpReturn
OpFunctionEnd
)";
SinglePassRunAndMatch<opt::LoopFusionPass>(text, true, 5);
}
#endif
} // namespace

24
tools/opt/opt.cpp
View File

@ -180,6 +180,13 @@ Options (in lexicographical order):
Splits any top level loops in which the register pressure has exceeded
a given threshold. The threshold must follow the use of this flag and
must be a positive integer value.
--loop-fusion
Identifies adjacent loops with the same lower and upper bound.
If this is legal, then merge the loops into a single loop.
Includes heuristics to ensure it does not increase number of
registers too much, while reducing the number of loads from
memory. Takes an additional positive integer argument to set
the maximum number of registers.
--loop-unroll
Fully unrolls loops marked with the Unroll flag
--loop-unroll-partial
@ -418,6 +425,21 @@ OptStatus ParseLoopFissionArg(int argc, const char** argv, int argi,
fprintf(
stderr,
"error: --loop-fission must be followed by a positive integer value\n");
}
OptStatus ParseLoopFusionArg(int argc, const char** argv, int argi,
Optimizer* optimizer) {
if (argi < argc) {
int max_registers_per_loop = atoi(argv[argi]);
if (max_registers_per_loop > 0) {
optimizer->RegisterPass(
CreateLoopFusionPass(static_cast<size_t>(max_registers_per_loop)));
return {OPT_CONTINUE, 0};
}
}
fprintf(stderr,
"error: --loop-loop-fusion must be followed by a positive "
"integer\n");
return {OPT_STOP, 1};
}
@ -577,6 +599,8 @@ OptStatus ParseFlags(int argc, const char** argv, Optimizer* optimizer,
optimizer->RegisterPass(CreateCopyPropagateArraysPass());
} else if (0 == strcmp(cur_arg, "--loop-fission")) {
OptStatus status = ParseLoopFissionArg(argc, argv, ++argi, optimizer);
} else if (0 == strcmp(cur_arg, "--loop-fusion")) {
OptStatus status = ParseLoopFusionArg(argc, argv, ++argi, optimizer);
if (status.action != OPT_CONTINUE) {
return status;
}