From 5d954d650688a2f069f0ff7693be57f5995bdf03 Mon Sep 17 00:00:00 2001 From: mtrofin Date: Tue, 25 Aug 2015 07:47:26 -0700 Subject: [PATCH] [turbofan] Deferred blocks splintering. This change encompasses what is necessary to enable stack checks in loops without suffering large regressions. Primarily, it consists of a new mechanism for dealing with deferred blocks by "splintering", rather than splitting, inside deferred blocks. My initial change was splitting along deferred block boundaries, but the regression introduced by stackchecks wasn't resolved conclusively. After investigation, it appears that just splitting ranges along cold block boundaries leads to a greater opportunity for moves on the hot path, hence the suboptimal outcome. The alternative "splinters" ranges rather than splitting them. While splitting creates 2 ranges and links them (parent-child), in contrast, splintering creates a new independent range with no parent-child relation to the original. The original range appears as if it has a liveness hole in the place of the splintered one. All thus obtained ranges are then register allocated with no change to the register allocator. The splinters (cold blocks) do not conflict with the hot path ranges, by construction. The hot path ones have less pressure to split, because we remove a source of conflicts. After allocation, we merge the splinters back to their original ranges and continue the pipeline. We leverage the previous changes made for deferred blocks (determining where to spill, for example). Review URL: https://codereview.chromium.org/1305393003 Cr-Commit-Position: refs/heads/master@{#30357} --- BUILD.gn | 4 +- src/bit-vector.cc | 2 +- src/compiler/ast-graph-builder.cc | 4 +- src/compiler/instruction.cc | 3 +- src/compiler/instruction.h | 14 +- src/compiler/live-range-separator.cc | 172 +++++++++++++ src/compiler/live-range-separator.h | 60 +++++ src/compiler/pipeline.cc | 28 +- src/compiler/preprocess-live-ranges.cc | 169 ------------- src/compiler/preprocess-live-ranges.h | 35 --- src/compiler/register-allocator.cc | 239 ++++++++++++++++-- src/compiler/register-allocator.h | 54 +++- src/flag-definitions.h | 7 +- .../compiler/register-allocator-unittest.cc | 2 +- tools/gyp/v8.gyp | 4 +- 15 files changed, 541 insertions(+), 256 deletions(-) create mode 100644 src/compiler/live-range-separator.cc create mode 100644 src/compiler/live-range-separator.h delete mode 100644 src/compiler/preprocess-live-ranges.cc delete mode 100644 src/compiler/preprocess-live-ranges.h diff --git a/BUILD.gn b/BUILD.gn index 0f20932d7c..f0dbc083cd 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -781,6 +781,8 @@ source_set("v8_base") { "src/compiler/jump-threading.h", "src/compiler/linkage.cc", "src/compiler/linkage.h", + "src/compiler/live-range-separator.cc", + "src/compiler/live-range-separator.h", "src/compiler/liveness-analyzer.cc", "src/compiler/liveness-analyzer.h", "src/compiler/load-elimination.cc", @@ -819,8 +821,6 @@ source_set("v8_base") { "src/compiler/pipeline.h", "src/compiler/pipeline-statistics.cc", "src/compiler/pipeline-statistics.h", - "src/compiler/preprocess-live-ranges.cc", - "src/compiler/preprocess-live-ranges.h", "src/compiler/raw-machine-assembler.cc", "src/compiler/raw-machine-assembler.h", "src/compiler/register-allocator.cc", diff --git a/src/bit-vector.cc b/src/bit-vector.cc index 198b24273c..cdd00f89c4 100644 --- a/src/bit-vector.cc +++ b/src/bit-vector.cc @@ -21,7 +21,7 @@ void BitVector::Print() { PrintF("%d", i); } } - PrintF("}"); + PrintF("}\n"); } #endif diff --git a/src/compiler/ast-graph-builder.cc b/src/compiler/ast-graph-builder.cc index 11dc456e3d..e6acf4d157 100644 --- a/src/compiler/ast-graph-builder.cc +++ b/src/compiler/ast-graph-builder.cc @@ -2928,9 +2928,7 @@ void AstGraphBuilder::VisitInScope(Statement* stmt, Scope* s, Node* context) { void AstGraphBuilder::VisitIterationBody(IterationStatement* stmt, LoopBuilder* loop) { ControlScopeForIteration scope(this, stmt, loop); - // TODO(mstarzinger): For now we only allow to interrupt non-asm.js code, - // which is a gigantic hack and should be extended to all code at some point. - if (!info()->shared_info()->asm_function()) { + if (FLAG_turbo_loop_stackcheck) { Node* node = NewNode(javascript()->StackCheck()); PrepareFrameState(node, stmt->StackCheckId()); } diff --git a/src/compiler/instruction.cc b/src/compiler/instruction.cc index 9aebb9a17a..1f6f7c9333 100644 --- a/src/compiler/instruction.cc +++ b/src/compiler/instruction.cc @@ -417,7 +417,8 @@ InstructionBlock::InstructionBlock(Zone* zone, RpoNumber rpo_number, handler_(handler), needs_frame_(false), must_construct_frame_(false), - must_deconstruct_frame_(false) {} + must_deconstruct_frame_(false), + last_deferred_(RpoNumber::Invalid()) {} size_t InstructionBlock::PredecessorIndexOf(RpoNumber rpo_number) const { diff --git a/src/compiler/instruction.h b/src/compiler/instruction.h index 4f6a515f11..a0718f3c21 100644 --- a/src/compiler/instruction.h +++ b/src/compiler/instruction.h @@ -778,9 +778,13 @@ class RpoNumber final { return other.index_ == this->index_ + 1; } - bool operator==(RpoNumber other) const { - return this->index_ == other.index_; - } + // Comparison operators. + bool operator==(RpoNumber other) const { return index_ == other.index_; } + bool operator!=(RpoNumber other) const { return index_ != other.index_; } + bool operator>(RpoNumber other) const { return index_ > other.index_; } + bool operator<(RpoNumber other) const { return index_ < other.index_; } + bool operator<=(RpoNumber other) const { return index_ <= other.index_; } + bool operator>=(RpoNumber other) const { return index_ >= other.index_; } private: explicit RpoNumber(int32_t index) : index_(index) {} @@ -992,6 +996,9 @@ class InstructionBlock final : public ZoneObject { bool must_deconstruct_frame() const { return must_deconstruct_frame_; } void mark_must_deconstruct_frame() { must_deconstruct_frame_ = true; } + void set_last_deferred(RpoNumber last) { last_deferred_ = last; } + RpoNumber last_deferred() const { return last_deferred_; } + private: Successors successors_; Predecessors predecessors_; @@ -1007,6 +1014,7 @@ class InstructionBlock final : public ZoneObject { bool needs_frame_; bool must_construct_frame_; bool must_deconstruct_frame_; + RpoNumber last_deferred_; }; typedef ZoneDeque ConstantDeque; diff --git a/src/compiler/live-range-separator.cc b/src/compiler/live-range-separator.cc new file mode 100644 index 0000000000..94b159787c --- /dev/null +++ b/src/compiler/live-range-separator.cc @@ -0,0 +1,172 @@ +// Copyright 2015 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "src/compiler/live-range-separator.h" +#include "src/compiler/register-allocator.h" + +namespace v8 { +namespace internal { +namespace compiler { + + +#define TRACE(...) \ + do { \ + if (FLAG_trace_alloc) PrintF(__VA_ARGS__); \ + } while (false) + + +namespace { + +// Starting from a deferred block, find the last consecutive deferred block. +RpoNumber GetLastDeferredBlock(const InstructionBlock *block, + const InstructionSequence *code) { + DCHECK(block->IsDeferred()); + RpoNumber first = block->rpo_number(); + + RpoNumber last = first; + for (int i = first.ToInt(); i < code->InstructionBlockCount(); ++i) { + RpoNumber at_i = RpoNumber::FromInt(i); + const InstructionBlock *block_at_i = code->InstructionBlockAt(at_i); + if (!block_at_i->IsDeferred()) break; + last = at_i; + } + + return last; +} + + +// Delimits consecutive deferred block sequences. +void AssociateDeferredBlockSequences(InstructionSequence *code) { + for (int blk_id = 0; blk_id < code->InstructionBlockCount(); ++blk_id) { + InstructionBlock *block = + code->InstructionBlockAt(RpoNumber::FromInt(blk_id)); + if (!block->IsDeferred()) continue; + RpoNumber last = GetLastDeferredBlock(block, code); + block->set_last_deferred(last); + // We know last is still deferred, and that last + 1, is not (or is an + // invalid index). So skip over last + 1 and continue from last + 2. This + // way, we visit each block exactly once, and the total complexity of this + // function is O(n), n being jthe number of blocks. + blk_id = last.ToInt() + 1; + } +} + + +// If the live range has a liveness hole right between start and end, +// we don't need to splinter it. +bool IsIntervalAlreadyExcluded(const LiveRange *range, LifetimePosition start, + LifetimePosition end) { + for (UseInterval *interval = range->first_interval(); interval != nullptr; + interval = interval->next()) { + if (interval->start() <= start && start < interval->end()) return false; + if (interval->start() < end && end <= interval->end()) return false; + } + return true; +} + + +void CreateSplinter(LiveRange *range, RegisterAllocationData *data, + LifetimePosition first_cut, LifetimePosition last_cut) { + DCHECK(!range->IsChild()); + DCHECK(!range->IsSplinter()); + // We can ignore ranges that live solely in deferred blocks. + // If a range ends right at the end of a deferred block, it is marked by + // the range builder as ending at gap start of the next block - since the + // end is a position where the variable isn't live. We need to take that + // into consideration. + LifetimePosition max_allowed_end = last_cut.NextFullStart(); + + if (first_cut <= range->Start() && max_allowed_end >= range->End()) { + return; + } + + LifetimePosition start = Max(first_cut, range->Start()); + LifetimePosition end = Min(last_cut, range->End()); + // Skip ranges that have a hole where the deferred block(s) are. + if (IsIntervalAlreadyExcluded(range, start, end)) return; + + if (start < end) { + // Ensure the original range has a spill range associated, before it gets + // splintered. Splinters will point to it. This way, when attempting to + // reuse spill slots of splinters, during allocation, we avoid clobbering + // such slots. + if (range->MayRequireSpillRange()) { + data->CreateSpillRangeForLiveRange(range); + } + LiveRange *result = data->NewChildRangeFor(range); + Zone *zone = data->allocation_zone(); + range->Splinter(start, end, result, zone); + } +} + + +// Splinter all ranges live inside successive deferred blocks. +// No control flow analysis is performed. After the register allocation, we will +// merge the splinters back into the original ranges, and then rely on the +// range connector to properly connect them. +void SplinterRangesInDeferredBlocks(RegisterAllocationData *data) { + InstructionSequence *code = data->code(); + int code_block_count = code->InstructionBlockCount(); + Zone *zone = data->allocation_zone(); + ZoneVector &in_sets = data->live_in_sets(); + + for (int i = 0; i < code_block_count; ++i) { + InstructionBlock *block = code->InstructionBlockAt(RpoNumber::FromInt(i)); + if (!block->IsDeferred()) continue; + + RpoNumber last_deferred = block->last_deferred(); + i = last_deferred.ToInt(); + + LifetimePosition first_cut = LifetimePosition::GapFromInstructionIndex( + block->first_instruction_index()); + + LifetimePosition last_cut = LifetimePosition::GapFromInstructionIndex( + static_cast(code->instructions().size())); + + const BitVector *in_set = in_sets[i]; + InstructionBlock *last = code->InstructionBlockAt(last_deferred); + const BitVector *out_set = LiveRangeBuilder::ComputeLiveOut(last, data); + last_cut = LifetimePosition::GapFromInstructionIndex( + last->last_instruction_index()); + + BitVector ranges_to_splinter(*in_set, zone); + ranges_to_splinter.Union(*out_set); + BitVector::Iterator iterator(&ranges_to_splinter); + + while (!iterator.Done()) { + int range_id = iterator.Current(); + iterator.Advance(); + + LiveRange *range = data->live_ranges()[range_id]; + CreateSplinter(range, data, first_cut, last_cut); + } + } +} +} // namespace + + +void LiveRangeSeparator::Splinter() { + AssociateDeferredBlockSequences(data()->code()); + SplinterRangesInDeferredBlocks(data()); +} + + +void LiveRangeMerger::Merge() { + int live_range_count = static_cast(data()->live_ranges().size()); + for (int i = 0; i < live_range_count; ++i) { + LiveRange *range = data()->live_ranges()[i]; + if (range == nullptr || range->IsEmpty() || range->IsChild() || + !range->IsSplinter()) { + continue; + } + LiveRange *splinter_parent = range->splintered_from(); + + splinter_parent->Merge(range, data()); + } +} + + +} // namespace compiler +} // namespace internal +} // namespace v8 diff --git a/src/compiler/live-range-separator.h b/src/compiler/live-range-separator.h new file mode 100644 index 0000000000..c8e6edc20b --- /dev/null +++ b/src/compiler/live-range-separator.h @@ -0,0 +1,60 @@ +// Copyright 2015 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef V8_LIVE_RANGE_SEPARATOR_H_ +#define V8_LIVE_RANGE_SEPARATOR_H_ + + +#include +namespace v8 { +namespace internal { + +class Zone; + +namespace compiler { + +class RegisterAllocationData; + + +// A register allocation pair of transformations: splinter and merge live ranges +class LiveRangeSeparator final : public ZoneObject { + public: + LiveRangeSeparator(RegisterAllocationData* data, Zone* zone) + : data_(data), zone_(zone) {} + + void Splinter(); + + private: + RegisterAllocationData* data() const { return data_; } + Zone* zone() const { return zone_; } + + RegisterAllocationData* const data_; + Zone* const zone_; + + DISALLOW_COPY_AND_ASSIGN(LiveRangeSeparator); +}; + + +class LiveRangeMerger final : public ZoneObject { + public: + LiveRangeMerger(RegisterAllocationData* data, Zone* zone) + : data_(data), zone_(zone) {} + + void Merge(); + + private: + RegisterAllocationData* data() const { return data_; } + Zone* zone() const { return zone_; } + + RegisterAllocationData* const data_; + Zone* const zone_; + + DISALLOW_COPY_AND_ASSIGN(LiveRangeMerger); +}; + + +} // namespace compiler +} // namespace internal +} // namespace v8 +#endif // V8_LIVE_RANGE_SEPARATOR_H_ diff --git a/src/compiler/pipeline.cc b/src/compiler/pipeline.cc index ffeec468c0..3a542e6890 100644 --- a/src/compiler/pipeline.cc +++ b/src/compiler/pipeline.cc @@ -35,6 +35,7 @@ #include "src/compiler/js-type-feedback-lowering.h" #include "src/compiler/js-typed-lowering.h" #include "src/compiler/jump-threading.h" +#include "src/compiler/live-range-separator.h" #include "src/compiler/load-elimination.h" #include "src/compiler/loop-analysis.h" #include "src/compiler/loop-peeling.h" @@ -42,7 +43,6 @@ #include "src/compiler/move-optimizer.h" #include "src/compiler/osr.h" #include "src/compiler/pipeline-statistics.h" -#include "src/compiler/preprocess-live-ranges.h" #include "src/compiler/register-allocator.h" #include "src/compiler/register-allocator-verifier.h" #include "src/compiler/schedule.h" @@ -778,13 +778,13 @@ struct BuildLiveRangesPhase { }; -struct PreprocessLiveRangesPhase { - static const char* phase_name() { return "preprocess live ranges"; } +struct SplinterLiveRangesPhase { + static const char* phase_name() { return "splinter live ranges"; } void Run(PipelineData* data, Zone* temp_zone) { - PreprocessLiveRanges live_range_preprocessor( - data->register_allocation_data(), temp_zone); - live_range_preprocessor.PreprocessRanges(); + LiveRangeSeparator live_range_splinterer(data->register_allocation_data(), + temp_zone); + live_range_splinterer.Splinter(); } }; @@ -813,6 +813,16 @@ struct AllocateDoubleRegistersPhase { }; +struct MergeSplintersPhase { + static const char* phase_name() { return "merge splintered ranges"; } + void Run(PipelineData* pipeline_data, Zone* temp_zone) { + RegisterAllocationData* data = pipeline_data->register_allocation_data(); + LiveRangeMerger live_range_merger(data, temp_zone); + live_range_merger.Merge(); + } +}; + + struct LocateSpillSlotsPhase { static const char* phase_name() { return "locate spill slots"; } @@ -1350,14 +1360,14 @@ void Pipeline::AllocateRegisters(const RegisterConfiguration* config, CHECK(!data->register_allocation_data()->ExistsUseWithoutDefinition()); } - if (FLAG_turbo_preprocess_ranges) { - Run(); - } + Run(); // TODO(mtrofin): re-enable greedy once we have bots for range preprocessing. Run>(); Run>(); + Run(); + if (FLAG_turbo_frame_elision) { Run(); Run(); diff --git a/src/compiler/preprocess-live-ranges.cc b/src/compiler/preprocess-live-ranges.cc deleted file mode 100644 index fee3a3d98c..0000000000 --- a/src/compiler/preprocess-live-ranges.cc +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright 2015 the V8 project authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "src/compiler/preprocess-live-ranges.h" -#include "src/compiler/register-allocator.h" - -namespace v8 { -namespace internal { -namespace compiler { - - -#define TRACE(...) \ - do { \ - if (FLAG_trace_alloc) PrintF(__VA_ARGS__); \ - } while (false) - - -namespace { - -LiveRange* Split(LiveRange* range, RegisterAllocationData* data, - LifetimePosition pos) { - DCHECK(range->Start() < pos && pos < range->End()); - DCHECK(pos.IsStart() || pos.IsGapPosition() || - (data->code() - ->GetInstructionBlock(pos.ToInstructionIndex()) - ->last_instruction_index() != pos.ToInstructionIndex())); - LiveRange* result = data->NewChildRangeFor(range); - range->SplitAt(pos, result, data->allocation_zone()); - TRACE("Split range %d(v%d) @%d => %d.\n", range->id(), - range->TopLevel()->id(), pos.ToInstructionIndex(), result->id()); - return result; -} - - -LifetimePosition GetSplitPositionForInstruction(const LiveRange* range, - int instruction_index) { - LifetimePosition ret = LifetimePosition::Invalid(); - - ret = LifetimePosition::GapFromInstructionIndex(instruction_index); - if (range->Start() >= ret || ret >= range->End()) { - return LifetimePosition::Invalid(); - } - return ret; -} - - -LiveRange* SplitRangeAfterBlock(LiveRange* range, RegisterAllocationData* data, - const InstructionBlock* block) { - const InstructionSequence* code = data->code(); - int last_index = block->last_instruction_index(); - int outside_index = static_cast(code->instructions().size()); - bool has_handler = false; - for (auto successor_id : block->successors()) { - const InstructionBlock* successor = code->InstructionBlockAt(successor_id); - if (successor->IsHandler()) { - has_handler = true; - } - outside_index = Min(outside_index, successor->first_instruction_index()); - } - int split_at = has_handler ? outside_index : last_index; - LifetimePosition after_block = - GetSplitPositionForInstruction(range, split_at); - - if (after_block.IsValid()) { - return Split(range, data, after_block); - } - - return range; -} - - -int GetFirstInstructionIndex(const UseInterval* interval) { - int ret = interval->start().ToInstructionIndex(); - if (!interval->start().IsGapPosition() && !interval->start().IsStart()) { - ++ret; - } - return ret; -} - - -bool DoesSubsequenceClobber(const InstructionSequence* code, int start, - int end) { - for (int i = start; i <= end; ++i) { - if (code->InstructionAt(i)->IsCall()) return true; - } - return false; -} - - -void SplitRangeAtDeferredBlocksWithCalls(LiveRange* range, - RegisterAllocationData* data) { - DCHECK(!range->IsFixed()); - DCHECK(!range->spilled()); - if (range->TopLevel()->HasSpillOperand()) { - TRACE( - "Skipping deferred block analysis for live range %d because it has a " - "spill operand.\n", - range->TopLevel()->id()); - return; - } - - const InstructionSequence* code = data->code(); - LiveRange* current_subrange = range; - - UseInterval* interval = current_subrange->first_interval(); - - while (interval != nullptr) { - int first_index = GetFirstInstructionIndex(interval); - int last_index = interval->end().ToInstructionIndex(); - - if (last_index > code->LastInstructionIndex()) { - last_index = code->LastInstructionIndex(); - } - - interval = interval->next(); - - for (int index = first_index; index <= last_index;) { - const InstructionBlock* block = code->GetInstructionBlock(index); - int last_block_index = static_cast(block->last_instruction_index()); - int last_covered_index = Min(last_index, last_block_index); - int working_index = index; - index = block->last_instruction_index() + 1; - - if (!block->IsDeferred() || - !DoesSubsequenceClobber(code, working_index, last_covered_index)) { - continue; - } - - TRACE("Deferred block B%d clobbers range %d(v%d).\n", - block->rpo_number().ToInt(), current_subrange->id(), - current_subrange->TopLevel()->id()); - LifetimePosition block_start = - GetSplitPositionForInstruction(current_subrange, working_index); - LiveRange* block_and_after = nullptr; - if (block_start.IsValid()) { - block_and_after = Split(current_subrange, data, block_start); - } else { - block_and_after = current_subrange; - } - LiveRange* next = SplitRangeAfterBlock(block_and_after, data, block); - if (next != current_subrange) interval = next->first_interval(); - current_subrange = next; - break; - } - } -} -} - - -void PreprocessLiveRanges::PreprocessRanges() { - SplitRangesAroundDeferredBlocks(); -} - - -void PreprocessLiveRanges::SplitRangesAroundDeferredBlocks() { - size_t live_range_count = data()->live_ranges().size(); - for (size_t i = 0; i < live_range_count; i++) { - LiveRange* range = data()->live_ranges()[i]; - if (range != nullptr && !range->IsEmpty() && !range->spilled() && - !range->IsFixed() && !range->IsChild()) { - SplitRangeAtDeferredBlocksWithCalls(range, data()); - } - } -} - -} // namespace compiler -} // namespace internal -} // namespace v8 diff --git a/src/compiler/preprocess-live-ranges.h b/src/compiler/preprocess-live-ranges.h deleted file mode 100644 index aa852fc7ca..0000000000 --- a/src/compiler/preprocess-live-ranges.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2015 the V8 project authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef V8_PREPROCESS_LIVE_RANGES_H_ -#define V8_PREPROCESS_LIVE_RANGES_H_ - -#include "src/compiler/register-allocator.h" - -namespace v8 { -namespace internal { -namespace compiler { - - -class PreprocessLiveRanges final { - public: - PreprocessLiveRanges(RegisterAllocationData* data, Zone* zone) - : data_(data), zone_(zone) {} - void PreprocessRanges(); - - private: - void SplitRangesAroundDeferredBlocks(); - - RegisterAllocationData* data() { return data_; } - Zone* zone() { return zone_; } - - RegisterAllocationData* data_; - Zone* zone_; -}; - - -} // namespace compiler -} // namespace internal -} // namespace v8 -#endif // V8_PREPROCESS_LIVE_RANGES_H_ diff --git a/src/compiler/register-allocator.cc b/src/compiler/register-allocator.cc index 101a10ae5b..4c1aabd576 100644 --- a/src/compiler/register-allocator.cc +++ b/src/compiler/register-allocator.cc @@ -253,6 +253,7 @@ LiveRange::LiveRange(int id, MachineType machine_type) first_pos_(nullptr), parent_(nullptr), next_(nullptr), + splintered_from_(nullptr), spill_operand_(nullptr), spills_at_definition_(nullptr), current_interval_(nullptr), @@ -447,9 +448,9 @@ void LiveRange::SetSpillOperand(InstructionOperand* operand) { void LiveRange::SetSpillRange(SpillRange* spill_range) { - DCHECK(HasNoSpillType() || HasSpillRange()); + DCHECK(!HasSpillOperand()); DCHECK(spill_range); - set_spill_type(SpillType::kSpillRange); + DCHECK(!IsChild()); spill_range_ = spill_range; } @@ -668,6 +669,146 @@ void LiveRange::SplitAt(LifetimePosition position, LiveRange* result, } +void LiveRange::Splinter(LifetimePosition start, LifetimePosition end, + LiveRange* result, Zone* zone) { + DCHECK(start != Start() || end != End()); + DCHECK(start < end); + + result->set_spill_type(spill_type()); + + if (start <= Start()) { + DCHECK(end < End()); + SplitAt(end, result, zone); + next_ = nullptr; + } else if (end >= End()) { + DCHECK(start > Start()); + SplitAt(start, result, zone); + next_ = nullptr; + } else { + DCHECK(start < End() && Start() < end); + + const int kInvalidId = std::numeric_limits::max(); + + SplitAt(start, result, zone); + + LiveRange end_part(kInvalidId, this->machine_type()); + result->SplitAt(end, &end_part, zone); + + next_ = end_part.next_; + last_interval_->set_next(end_part.first_interval_); + last_interval_ = end_part.last_interval_; + + + if (first_pos_ == nullptr) { + first_pos_ = end_part.first_pos_; + } else { + UsePosition* pos = first_pos_; + for (; pos->next() != nullptr; pos = pos->next()) { + } + pos->set_next(end_part.first_pos_); + } + } + result->next_ = nullptr; + result->parent_ = nullptr; + + result->SetSplinteredFrom(this); +} + + +void LiveRange::SetSplinteredFrom(LiveRange* splinter_parent) { + // The splinter parent is always the original "Top". + DCHECK(splinter_parent->Start() < Start()); + DCHECK(!splinter_parent->IsChild()); + + splintered_from_ = splinter_parent; + if (!HasSpillOperand()) { + SetSpillRange(splinter_parent->spill_range_); + } +} + + +void LiveRange::AppendChild(LiveRange* other) { + DCHECK(!other->IsChild()); + next_ = other; + + other->UpdateParentForAllChildren(TopLevel()); + TopLevel()->UpdateSpillRangePostMerge(other); +} + + +void LiveRange::UpdateSpillRangePostMerge(LiveRange* merged) { + DCHECK(!IsChild()); + DCHECK(merged->TopLevel() == this); + + if (HasNoSpillType() && merged->HasSpillRange()) { + set_spill_type(merged->spill_type()); + DCHECK(GetSpillRange()->live_ranges().size() > 0); + merged->spill_range_ = nullptr; + merged->bits_ = SpillTypeField::update(merged->bits_, + LiveRange::SpillType::kNoSpillType); + } +} + + +void LiveRange::UpdateParentForAllChildren(LiveRange* new_parent) { + LiveRange* child = this; + for (; child != nullptr; child = child->next()) { + child->parent_ = new_parent; + } +} + + +LiveRange* LiveRange::GetLastChild() { + LiveRange* ret = this; + for (; ret->next() != nullptr; ret = ret->next()) { + } + return ret; +} + + +void LiveRange::Merge(LiveRange* other, RegisterAllocationData* data) { + DCHECK(!IsChild()); + DCHECK(!other->IsChild()); + DCHECK(Start() < other->Start()); + + LiveRange* last_other = other->GetLastChild(); + LiveRange* last_me = GetLastChild(); + + // Simple case: we just append at the end. + if (last_me->End() <= other->Start()) return last_me->AppendChild(other); + + DCHECK(last_me->End() > last_other->End()); + + // In the more general case, we need to find the ranges between which to + // insert. + LiveRange* insertion_point = this; + for (; insertion_point->next() != nullptr && + insertion_point->next()->Start() <= other->Start(); + insertion_point = insertion_point->next()) { + } + + // When we splintered the original range, we reconstituted the original range + // into one range without children, but with discontinuities. To merge the + // splinter back in, we need to split the range - or a child obtained after + // register allocation splitting. + LiveRange* after = insertion_point->next(); + if (insertion_point->End() > other->Start()) { + LiveRange* new_after = data->NewChildRangeFor(insertion_point); + insertion_point->SplitAt(other->Start(), new_after, + data->allocation_zone()); + new_after->set_spilled(insertion_point->spilled()); + if (!new_after->spilled()) + new_after->set_assigned_register(insertion_point->assigned_register()); + after = new_after; + } + + last_other->next_ = after; + insertion_point->next_ = other; + other->UpdateParentForAllChildren(TopLevel()); + TopLevel()->UpdateSpillRangePostMerge(other); +} + + // This implements an ordering on live ranges so that they are ordered by their // start positions. This is needed for the correctness of the register // allocation algorithm. If two live ranges start at the same offset then there @@ -913,8 +1054,15 @@ std::ostream& operator<<(std::ostream& os, SpillRange::SpillRange(LiveRange* parent, Zone* zone) - : live_ranges_(zone), assigned_slot_(kUnassignedSlot) { + : live_ranges_(zone), + assigned_slot_(kUnassignedSlot), + byte_width_(GetByteWidth(parent->machine_type())), + kind_(parent->kind()) { + // Spill ranges are created for top level, non-splintered ranges. This is so + // that, when merging decisions are made, we consider the full extent of the + // virtual register, and avoid clobbering it. DCHECK(!parent->IsChild()); + DCHECK(!parent->IsSplinter()); UseInterval* result = nullptr; UseInterval* node = nullptr; // Copy the intervals for all ranges. @@ -934,7 +1082,6 @@ SpillRange::SpillRange(LiveRange* parent, Zone* zone) use_interval_ = result; live_ranges().push_back(parent); end_position_ = node->end(); - DCHECK(!parent->HasSpillRange()); parent->SetSpillRange(this); } @@ -1056,7 +1203,6 @@ RegisterAllocationData::RegisterAllocationData( RegisterConfiguration::kMaxGeneralRegisters); DCHECK(this->config()->num_double_registers() <= RegisterConfiguration::kMaxDoubleRegisters); - spill_ranges().reserve(8); assigned_registers_ = new (code_zone()) BitVector(this->config()->num_general_registers(), code_zone()); assigned_double_registers_ = new (code_zone()) @@ -1100,14 +1246,20 @@ LiveRange* RegisterAllocationData::NewLiveRange(int index, } -LiveRange* RegisterAllocationData::NewChildRangeFor(LiveRange* range) { +LiveRange* RegisterAllocationData::NextLiveRange(MachineType machine_type) { int vreg = virtual_register_count_++; if (vreg >= static_cast(live_ranges().size())) { live_ranges().resize(vreg + 1, nullptr); } - auto child = new (allocation_zone()) LiveRange(vreg, range->machine_type()); - DCHECK_NULL(live_ranges()[vreg]); - live_ranges()[vreg] = child; + LiveRange* ret = NewLiveRange(vreg, machine_type); + return ret; +} + + +LiveRange* RegisterAllocationData::NewChildRangeFor(LiveRange* range) { + auto child = NextLiveRange(range->machine_type()); + DCHECK_NULL(live_ranges()[child->id()]); + live_ranges()[child->id()] = child; return child; } @@ -1155,9 +1307,28 @@ bool RegisterAllocationData::ExistsUseWithoutDefinition() { SpillRange* RegisterAllocationData::AssignSpillRangeToLiveRange( LiveRange* range) { + DCHECK(!range->IsChild()); + DCHECK(!range->HasSpillOperand()); + + SpillRange* spill_range = range->GetAllocatedSpillRange(); + if (spill_range == nullptr) { + DCHECK(!range->IsSplinter()); + spill_range = new (allocation_zone()) SpillRange(range, allocation_zone()); + } + range->set_spill_type(LiveRange::SpillType::kSpillRange); + + spill_ranges().insert(spill_range); + return spill_range; +} + + +SpillRange* RegisterAllocationData::CreateSpillRangeForLiveRange( + LiveRange* range) { + DCHECK(!range->HasSpillOperand()); + DCHECK(!range->IsChild()); + DCHECK(!range->IsSplinter()); auto spill_range = new (allocation_zone()) SpillRange(range, allocation_zone()); - spill_ranges().push_back(spill_range); return spill_range; } @@ -1230,6 +1401,23 @@ void RegisterAllocationData::Print(const MoveOperands* move) { } +void RegisterAllocationData::Print(const SpillRange* spill_range) { + OFStream os(stdout); + os << "{" << std::endl; + for (LiveRange* range : spill_range->live_ranges()) { + os << range->id() << " "; + } + os << std::endl; + + for (UseInterval* interval = spill_range->interval(); interval != nullptr; + interval = interval->next()) { + os << '[' << interval->start() << ", " << interval->end() << ')' + << std::endl; + } + os << "}" << std::endl; +} + + ConstraintBuilder::ConstraintBuilder(RegisterAllocationData* data) : data_(data) {} @@ -1474,22 +1662,25 @@ LiveRangeBuilder::LiveRangeBuilder(RegisterAllocationData* data, : data_(data), phi_hints_(local_zone) {} -BitVector* LiveRangeBuilder::ComputeLiveOut(const InstructionBlock* block) { +BitVector* LiveRangeBuilder::ComputeLiveOut(const InstructionBlock* block, + RegisterAllocationData* data) { // Compute live out for the given block, except not including backward // successor edges. - auto live_out = new (allocation_zone()) - BitVector(code()->VirtualRegisterCount(), allocation_zone()); + Zone* zone = data->allocation_zone(); + const InstructionSequence* code = data->code(); + + auto live_out = new (zone) BitVector(code->VirtualRegisterCount(), zone); // Process all successor blocks. for (auto succ : block->successors()) { - // Add values live on entry to the successor. Note the successor's - // live_in will not be computed yet for backwards edges. - auto live_in = live_in_sets()[succ.ToSize()]; + // Add values live on entry to the successor. + if (succ <= block->rpo_number()) continue; + auto live_in = data->live_in_sets()[succ.ToSize()]; if (live_in != nullptr) live_out->Union(*live_in); // All phi input operands corresponding to this successor edge are live // out from this block. - auto successor = code()->InstructionBlockAt(succ); + auto successor = code->InstructionBlockAt(succ); size_t index = successor->PredecessorIndexOf(block->rpo_number()); DCHECK(index < successor->PredecessorCount()); for (auto phi : successor->phis()) { @@ -1834,7 +2025,7 @@ void LiveRangeBuilder::BuildLiveRanges() { for (int block_id = code()->InstructionBlockCount() - 1; block_id >= 0; --block_id) { auto block = code()->InstructionBlockAt(RpoNumber::FromInt(block_id)); - auto live = ComputeLiveOut(block); + auto live = ComputeLiveOut(block, data()); // Initially consider all live_out values live for the entire block. We // will shorten these intervals if necessary. AddInitialIntervals(block, live); @@ -2597,13 +2788,15 @@ OperandAssigner::OperandAssigner(RegisterAllocationData* data) : data_(data) {} void OperandAssigner::AssignSpillSlots() { - auto& spill_ranges = data()->spill_ranges(); + ZoneSet& spill_ranges = data()->spill_ranges(); // Merge disjoint spill ranges - for (size_t i = 0; i < spill_ranges.size(); i++) { - auto range = spill_ranges[i]; + for (auto i = spill_ranges.begin(), end = spill_ranges.end(); i != end; ++i) { + SpillRange* range = *i; if (range->IsEmpty()) continue; - for (size_t j = i + 1; j < spill_ranges.size(); j++) { - auto other = spill_ranges[j]; + auto j = i; + j++; + for (; j != end; ++j) { + SpillRange* other = *j; if (!other->IsEmpty()) { range->TryMerge(other); } diff --git a/src/compiler/register-allocator.h b/src/compiler/register-allocator.h index 2e63d36e12..c537bbe7f9 100644 --- a/src/compiler/register-allocator.h +++ b/src/compiler/register-allocator.h @@ -273,7 +273,7 @@ class UsePosition final : public ZoneObject { class SpillRange; - +class RegisterAllocationData; // Representation of SSA values' live ranges as a collection of (continuous) // intervals over the instruction ordering. @@ -355,6 +355,9 @@ class LiveRange final : public ZoneObject { // All uses following the given position will be moved from this // live range to the result live range. void SplitAt(LifetimePosition position, LiveRange* result, Zone* zone); + void Splinter(LifetimePosition start, LifetimePosition end, LiveRange* result, + Zone* zone); + void Merge(LiveRange* other, RegisterAllocationData* data); // Returns nullptr when no register is hinted, otherwise sets register_index. UsePosition* FirstHintPosition(int* register_index) const; @@ -384,6 +387,12 @@ class LiveRange final : public ZoneObject { DCHECK(spill_type() == SpillType::kSpillOperand); return spill_operand_; } + + SpillRange* GetAllocatedSpillRange() const { + DCHECK(spill_type() != SpillType::kSpillOperand); + return spill_range_; + } + SpillRange* GetSpillRange() const { DCHECK(spill_type() == SpillType::kSpillRange); return spill_range_; @@ -395,6 +404,11 @@ class LiveRange final : public ZoneObject { return spill_type() == SpillType::kSpillOperand; } bool HasSpillRange() const { return spill_type() == SpillType::kSpillRange; } + bool MayRequireSpillRange() const { + DCHECK(!IsChild() && !IsSplinter()); + return !HasSpillOperand() && spill_range_ == nullptr; + } + AllocatedOperand GetSpillRangeOperand() const; void SpillAtDefinition(Zone* zone, int gap_index, @@ -458,17 +472,35 @@ class LiveRange final : public ZoneObject { static const float kInvalidWeight; static const float kMaxWeight; - private: + LiveRange* splintered_from() const { + DCHECK(!IsChild()); + return splintered_from_; + } + bool IsSplinter() const { + DCHECK(!IsChild()); + return splintered_from_ != nullptr; + } + void set_spill_type(SpillType value) { bits_ = SpillTypeField::update(bits_, value); } + private: + void AppendChild(LiveRange* other); + void UpdateParentForAllChildren(LiveRange* new_parent); + void UpdateSpillRangePostMerge(LiveRange* merged); + + void SetSplinteredFrom(LiveRange* splinter_parent); + + void set_spilled(bool value) { bits_ = SpilledField::update(bits_, value); } UseInterval* FirstSearchIntervalForPosition(LifetimePosition position) const; void AdvanceLastProcessedMarker(UseInterval* to_start_of, LifetimePosition but_not_past) const; + LiveRange* GetLastChild(); + typedef BitField SpilledField; typedef BitField HasSlotUseField; typedef BitField IsPhiField; @@ -485,6 +517,7 @@ class LiveRange final : public ZoneObject { UsePosition* first_pos_; LiveRange* parent_; LiveRange* next_; + LiveRange* splintered_from_; union { // Correct value determined by spill_type() InstructionOperand* spill_operand_; @@ -543,10 +576,13 @@ class SpillRange final : public ZoneObject { DCHECK_NE(kUnassignedSlot, assigned_slot_); return assigned_slot_; } + const ZoneVector& live_ranges() const { return live_ranges_; } + ZoneVector& live_ranges() { return live_ranges_; } + int byte_width() const { return byte_width_; } + RegisterKind kind() const { return kind_; } private: LifetimePosition End() const { return end_position_; } - ZoneVector& live_ranges() { return live_ranges_; } bool IsIntersectingWith(SpillRange* other) const; // Merge intervals, making sure the use intervals are sorted void MergeDisjointIntervals(UseInterval* other); @@ -555,6 +591,8 @@ class SpillRange final : public ZoneObject { UseInterval* use_interval_; LifetimePosition end_position_; int assigned_slot_; + int byte_width_; + RegisterKind kind_; DISALLOW_COPY_AND_ASSIGN(SpillRange); }; @@ -612,7 +650,7 @@ class RegisterAllocationData final : public ZoneObject { return fixed_double_live_ranges_; } ZoneVector& live_in_sets() { return live_in_sets_; } - ZoneVector& spill_ranges() { return spill_ranges_; } + ZoneSet& spill_ranges() { return spill_ranges_; } DelayedReferences& delayed_references() { return delayed_references_; } InstructionSequence* code() const { return code_; } // This zone is for datastructures only needed during register allocation @@ -630,9 +668,11 @@ class RegisterAllocationData final : public ZoneObject { LiveRange* LiveRangeFor(int index); // Creates a new live range. LiveRange* NewLiveRange(int index, MachineType machine_type); + LiveRange* NextLiveRange(MachineType machine_type); LiveRange* NewChildRangeFor(LiveRange* range); SpillRange* AssignSpillRangeToLiveRange(LiveRange* range); + SpillRange* CreateSpillRangeForLiveRange(LiveRange* range); MoveOperands* AddGapMove(int index, Instruction::GapPosition position, const InstructionOperand& from, @@ -656,6 +696,7 @@ class RegisterAllocationData final : public ZoneObject { void Print(const LiveRange* range, bool with_children = false); void Print(const InstructionOperand& op); void Print(const MoveOperands* move); + void Print(const SpillRange* spill_range); private: Zone* const allocation_zone_; @@ -668,7 +709,7 @@ class RegisterAllocationData final : public ZoneObject { ZoneVector live_ranges_; ZoneVector fixed_live_ranges_; ZoneVector fixed_double_live_ranges_; - ZoneVector spill_ranges_; + ZoneSet spill_ranges_; DelayedReferences delayed_references_; BitVector* assigned_registers_; BitVector* assigned_double_registers_; @@ -721,6 +762,8 @@ class LiveRangeBuilder final : public ZoneObject { // Phase 3: compute liveness of all virtual register. void BuildLiveRanges(); + static BitVector* ComputeLiveOut(const InstructionBlock* block, + RegisterAllocationData* data); private: RegisterAllocationData* data() const { return data_; } @@ -737,7 +780,6 @@ class LiveRangeBuilder final : public ZoneObject { void Verify() const; // Liveness analysis support. - BitVector* ComputeLiveOut(const InstructionBlock* block); void AddInitialIntervals(const InstructionBlock* block, BitVector* live_out); void ProcessInstructions(const InstructionBlock* block, BitVector* live); void ProcessPhis(const InstructionBlock* block, BitVector* live); diff --git a/src/flag-definitions.h b/src/flag-definitions.h index 51d716e291..17a69bff86 100644 --- a/src/flag-definitions.h +++ b/src/flag-definitions.h @@ -402,9 +402,14 @@ DEFINE_BOOL(omit_map_checks_for_leaf_maps, true, DEFINE_BOOL(turbo, false, "enable TurboFan compiler") DEFINE_BOOL(turbo_shipping, true, "enable TurboFan compiler on subset") DEFINE_BOOL(turbo_greedy_regalloc, false, "use the greedy register allocator") -DEFINE_BOOL(turbo_preprocess_ranges, false, +DEFINE_BOOL(turbo_preprocess_ranges, true, "run pre-register allocation heuristics") +DEFINE_BOOL(turbo_loop_stackcheck, true, "enable stack checks in loops") +// TODO(mtrofin): remove these implications, as they are here just for trybot +// purposes. DEFINE_IMPLICATION(turbo_greedy_regalloc, turbo_preprocess_ranges) +DEFINE_IMPLICATION(turbo_greedy_regalloc, turbo_loop_stackcheck) + DEFINE_IMPLICATION(turbo, turbo_asm_deoptimization) DEFINE_STRING(turbo_filter, "~~", "optimization filter for TurboFan compiler") DEFINE_BOOL(trace_turbo, false, "trace generated TurboFan IR") diff --git a/test/unittests/compiler/register-allocator-unittest.cc b/test/unittests/compiler/register-allocator-unittest.cc index 23a118b6ad..6cdaa4630d 100644 --- a/test/unittests/compiler/register-allocator-unittest.cc +++ b/test/unittests/compiler/register-allocator-unittest.cc @@ -686,7 +686,7 @@ TEST_F(RegisterAllocatorTest, MultipleDeferredBlockSpills) { EXPECT_TRUE(IsParallelMovePresent(call_in_b1, Instruction::START, sequence(), Reg(var3_reg), Slot(var3_slot))); EXPECT_TRUE(IsParallelMovePresent(end_of_b1, Instruction::START, sequence(), - Slot(var3_slot), Reg())); + Slot(var3_slot), Reg(var3_reg))); EXPECT_TRUE(IsParallelMovePresent(call_in_b2, Instruction::START, sequence(), Reg(var3_reg), Slot(var3_slot))); diff --git a/tools/gyp/v8.gyp b/tools/gyp/v8.gyp index 432c7ab060..e7d33b15c2 100644 --- a/tools/gyp/v8.gyp +++ b/tools/gyp/v8.gyp @@ -540,6 +540,8 @@ '../../src/compiler/linkage.h', '../../src/compiler/liveness-analyzer.cc', '../../src/compiler/liveness-analyzer.h', + '../../src/compiler/live-range-separator.cc', + '../../src/compiler/live-range-separator.h', '../../src/compiler/load-elimination.cc', '../../src/compiler/load-elimination.h', '../../src/compiler/loop-analysis.cc', @@ -577,8 +579,6 @@ '../../src/compiler/pipeline.h', '../../src/compiler/pipeline-statistics.cc', '../../src/compiler/pipeline-statistics.h', - '../../src/compiler/preprocess-live-ranges.cc', - '../../src/compiler/preprocess-live-ranges.h', '../../src/compiler/raw-machine-assembler.cc', '../../src/compiler/raw-machine-assembler.h', '../../src/compiler/register-allocator.cc',