Ryujinx/ARMeilleure/CodeGen/RegisterAllocators/LiveInterval.cs
FICTURE7 69093cf2d6
Optimize LSRA (#2563)
* Optimize `TryAllocateRegWithtoutSpill` a bit

* Add a fast path for when all registers are live.
* Do not query `GetOverlapPosition` if the register is already in use
  (i.e: free position is 0).

* Do not allocate child split list if not parent

* Turn `LiveRange` into a reference struct

`LiveRange` is now a reference wrapping struct like `Operand` and
`Operation`.

It has also been changed into a singly linked-list. In micro-benchmarks
traversing the linked-list was faster than binary search on `List<T>`.
Even for quite large input sizes (e.g: 1,000,000), surprisingly.

Could be because the code gen for traversing the linked-list is much
much cleaner and there is no virtual dispatch happening when checking if
intervals overlaps.

* Turn `LiveInterval` into an iterator

The LSRA allocates in forward order and never inspect previous
`LiveInterval` once they are expired. Something similar can be done for
the `LiveRange`s within the `LiveInterval`s themselves.

The `LiveInterval` is turned into a iterator which expires `LiveRange`
within it. The iterator is moved forward along with interval walking
code, i.e: AllocateInterval(context, interval, cIndex).

* Remove `LinearScanAllocator.Sources`

Local methods are less susceptible to do allocations than lambdas.

* Optimize `GetOverlapPosition(interval)` a bit

Time complexity should be in O(n+m) instead of O(nm) now.

* Optimize `NumberLocals` a bit

Use the same idea as in `HybridAllocator` to store the visited state
in the MSB of the Operand's value instead of using a `HashSet<T>`.

* Optimize `InsertSplitCopies` a bit

Avoid allocating a redundant `CopyResolver`.

* Optimize `InsertSplitCopiesAtEdges` a bit

Avoid redundant allocations of `CopyResolver`.

* Use stack allocation for `freePositions`

Avoid redundant computations.

* Add `UseList`

Replace `SortedIntegerList` with an even more specialized data
structure. It allocates memory on the arena allocators and does not
require copying use positions when splitting it.

* Turn `LiveInterval` into a reference struct

`LiveInterval` is now a reference wrapping struct like `Operand` and
`Operation`.

The rationale behind turning this in a reference wrapping struct is
because a `LiveInterval` is associated with each local variable, and
these intervals may themselves be split further. I've seen translations
having up to 8000 local variables.

To make the `LiveInterval` unmanaged, a new data structure called
`LiveIntervalList` was added to store child splits. This differs from
`SortedList<,>` because it can contain intervals with the same start
position.

Really wished we got some more of C++ template in C#. :^(

* Optimize `GetChildSplit` a bit

No need to inspect the remaining ranges if we've reached a range which
starts after position, since the split list is ordered.

* Optimize `CopyResolver` a bit

Lazily allocate the fill, spill and parallel copy structures since most
of the time only one of them is needed.

* Optimize `BitMap.Enumerator` a bit

Marking `MoveNext` as `AggressiveInlining` allows RyuJIT to promote the
`Enumerator` struct into registers completely, reducing load/store code
a lot since it does not have to store the struct on the stack for ABI
purposes.

* Use stack allocation for `use/blockedPositions`

* Optimize `AllocateWithSpill` a bit

* Address feedback

* Make `LiveInterval.AddRange(,)` more conservative

Produces no diff against master, but just for good measure.
2021-10-08 18:15:44 -03:00

394 lines
10 KiB
C#

using ARMeilleure.IntermediateRepresentation;
using System;
using System.Collections.Generic;
using System.Diagnostics;
namespace ARMeilleure.CodeGen.RegisterAllocators
{
unsafe readonly struct LiveInterval : IComparable<LiveInterval>
{
public const int NotFound = -1;
private struct Data
{
public int End;
public int SpillOffset;
public LiveRange FirstRange;
public LiveRange PrevRange;
public LiveRange CurrRange;
public LiveInterval Parent;
public UseList Uses;
public LiveIntervalList Children;
public Operand Local;
public Register Register;
public bool IsFixed;
}
private readonly Data* _data;
private ref int End => ref _data->End;
private ref LiveRange FirstRange => ref _data->FirstRange;
private ref LiveRange CurrRange => ref _data->CurrRange;
private ref LiveRange PrevRange => ref _data->PrevRange;
private ref LiveInterval Parent => ref _data->Parent;
private ref UseList Uses => ref _data->Uses;
private ref LiveIntervalList Children => ref _data->Children;
public Operand Local => _data->Local;
public ref Register Register => ref _data->Register;
public ref int SpillOffset => ref _data->SpillOffset;
public bool IsFixed => _data->IsFixed;
public bool IsEmpty => FirstRange == default;
public bool IsSplit => Children.Count != 0;
public bool IsSpilled => SpillOffset != -1;
public int UsesCount => Uses.Count;
public LiveInterval(Operand local = default, LiveInterval parent = default)
{
_data = Allocators.LiveIntervals.Allocate<Data>();
*_data = default;
_data->IsFixed = false;
_data->Local = local;
Parent = parent == default ? this : parent;
Uses = new UseList();
Children = new LiveIntervalList();
FirstRange = default;
CurrRange = default;
PrevRange = default;
SpillOffset = -1;
}
public LiveInterval(Register register) : this(local: default, parent: default)
{
_data->IsFixed = true;
Register = register;
}
public void Reset()
{
PrevRange = default;
CurrRange = FirstRange;
}
public void Forward(int position)
{
LiveRange prev = PrevRange;
LiveRange curr = CurrRange;
while (curr != default && curr.Start < position && !curr.Overlaps(position))
{
prev = curr;
curr = curr.Next;
}
PrevRange = prev;
CurrRange = curr;
}
public int GetStart()
{
Debug.Assert(!IsEmpty, "Empty LiveInterval cannot have a start position.");
return FirstRange.Start;
}
public void SetStart(int position)
{
if (FirstRange != default)
{
Debug.Assert(position != FirstRange.End);
FirstRange.Start = position;
}
else
{
FirstRange = new LiveRange(position, position + 1);
End = position + 1;
}
}
public int GetEnd()
{
Debug.Assert(!IsEmpty, "Empty LiveInterval cannot have an end position.");
return End;
}
public void AddRange(int start, int end)
{
Debug.Assert(start < end, $"Invalid range start position {start}, {end}");
if (FirstRange != default)
{
// If the new range ends exactly where the first range start, then coalesce together.
if (end == FirstRange.Start)
{
FirstRange.Start = start;
return;
}
// If the new range is already contained, then coalesce together.
else if (FirstRange.Overlaps(start, end))
{
FirstRange.Start = Math.Min(FirstRange.Start, start);
FirstRange.End = Math.Max(FirstRange.End, end);
End = Math.Max(End, end);
Debug.Assert(FirstRange.Next == default || !FirstRange.Overlaps(FirstRange.Next));
return;
}
}
FirstRange = new LiveRange(start, end, FirstRange);
End = Math.Max(End, end);
Debug.Assert(FirstRange.Next == default || !FirstRange.Overlaps(FirstRange.Next));
}
public void AddUsePosition(int position)
{
Uses.Add(position);
}
public bool Overlaps(int position)
{
LiveRange curr = CurrRange;
while (curr != default && curr.Start <= position)
{
if (curr.Overlaps(position))
{
return true;
}
curr = curr.Next;
}
return false;
}
public bool Overlaps(LiveInterval other)
{
return GetOverlapPosition(other) != NotFound;
}
public int GetOverlapPosition(LiveInterval other)
{
LiveRange a = CurrRange;
LiveRange b = other.CurrRange;
while (a != default)
{
while (b != default && b.Start < a.Start)
{
if (a.Overlaps(b))
{
return a.Start;
}
b = b.Next;
}
if (b == default)
{
break;
}
else if (a.Overlaps(b))
{
return a.Start;
}
a = a.Next;
}
return NotFound;
}
public ReadOnlySpan<LiveInterval> SplitChildren()
{
return Parent.Children.Span;
}
public ReadOnlySpan<int> UsePositions()
{
return Uses.Span;
}
public int FirstUse()
{
return Uses.FirstUse;
}
public int NextUseAfter(int position)
{
return Uses.NextUse(position);
}
public LiveInterval Split(int position)
{
LiveInterval result = new(Local, Parent);
result.End = End;
LiveRange prev = PrevRange;
LiveRange curr = CurrRange;
while (curr != default && curr.Start < position && !curr.Overlaps(position))
{
prev = curr;
curr = curr.Next;
}
if (curr.Start >= position)
{
prev.Next = default;
result.FirstRange = curr;
End = prev.End;
}
else
{
result.FirstRange = new LiveRange(position, curr.End, curr.Next);
curr.End = position;
curr.Next = default;
End = curr.End;
}
result.Uses = Uses.Split(position);
AddSplitChild(result);
Debug.Assert(!IsEmpty, "Left interval is empty after split.");
Debug.Assert(!result.IsEmpty, "Right interval is empty after split.");
// Make sure the iterator in the new split is pointing to the start.
result.Reset();
return result;
}
private void AddSplitChild(LiveInterval child)
{
Debug.Assert(!child.IsEmpty, "Trying to insert an empty interval.");
Parent.Children.Add(child);
}
public LiveInterval GetSplitChild(int position)
{
if (Overlaps(position))
{
return this;
}
foreach (LiveInterval splitChild in SplitChildren())
{
if (splitChild.Overlaps(position))
{
return splitChild;
}
else if (splitChild.GetStart() > position)
{
break;
}
}
return default;
}
public bool TrySpillWithSiblingOffset()
{
foreach (LiveInterval splitChild in SplitChildren())
{
if (splitChild.IsSpilled)
{
Spill(splitChild.SpillOffset);
return true;
}
}
return false;
}
public void Spill(int offset)
{
SpillOffset = offset;
}
public int CompareTo(LiveInterval interval)
{
if (FirstRange == default || interval.FirstRange == default)
{
return 0;
}
return GetStart().CompareTo(interval.GetStart());
}
public bool Equals(LiveInterval interval)
{
return interval._data == _data;
}
public override bool Equals(object obj)
{
return obj is LiveInterval interval && Equals(interval);
}
public static bool operator ==(LiveInterval a, LiveInterval b)
{
return a.Equals(b);
}
public static bool operator !=(LiveInterval a, LiveInterval b)
{
return !a.Equals(b);
}
public override int GetHashCode()
{
return HashCode.Combine((IntPtr)_data);
}
public override string ToString()
{
LiveInterval self = this;
IEnumerable<string> GetRanges()
{
LiveRange curr = self.CurrRange;
while (curr != default)
{
if (curr == self.CurrRange)
{
yield return "*" + curr;
}
else
{
yield return curr.ToString();
}
curr = curr.Next;
}
}
return string.Join(", ", GetRanges());
}
}
}