mirror of
https://github.com/Ryujinx/Ryujinx.git
synced 2024-11-09 20:28:36 +00:00
69093cf2d6
* Optimize `TryAllocateRegWithtoutSpill` a bit * Add a fast path for when all registers are live. * Do not query `GetOverlapPosition` if the register is already in use (i.e: free position is 0). * Do not allocate child split list if not parent * Turn `LiveRange` into a reference struct `LiveRange` is now a reference wrapping struct like `Operand` and `Operation`. It has also been changed into a singly linked-list. In micro-benchmarks traversing the linked-list was faster than binary search on `List<T>`. Even for quite large input sizes (e.g: 1,000,000), surprisingly. Could be because the code gen for traversing the linked-list is much much cleaner and there is no virtual dispatch happening when checking if intervals overlaps. * Turn `LiveInterval` into an iterator The LSRA allocates in forward order and never inspect previous `LiveInterval` once they are expired. Something similar can be done for the `LiveRange`s within the `LiveInterval`s themselves. The `LiveInterval` is turned into a iterator which expires `LiveRange` within it. The iterator is moved forward along with interval walking code, i.e: AllocateInterval(context, interval, cIndex). * Remove `LinearScanAllocator.Sources` Local methods are less susceptible to do allocations than lambdas. * Optimize `GetOverlapPosition(interval)` a bit Time complexity should be in O(n+m) instead of O(nm) now. * Optimize `NumberLocals` a bit Use the same idea as in `HybridAllocator` to store the visited state in the MSB of the Operand's value instead of using a `HashSet<T>`. * Optimize `InsertSplitCopies` a bit Avoid allocating a redundant `CopyResolver`. * Optimize `InsertSplitCopiesAtEdges` a bit Avoid redundant allocations of `CopyResolver`. * Use stack allocation for `freePositions` Avoid redundant computations. * Add `UseList` Replace `SortedIntegerList` with an even more specialized data structure. It allocates memory on the arena allocators and does not require copying use positions when splitting it. * Turn `LiveInterval` into a reference struct `LiveInterval` is now a reference wrapping struct like `Operand` and `Operation`. The rationale behind turning this in a reference wrapping struct is because a `LiveInterval` is associated with each local variable, and these intervals may themselves be split further. I've seen translations having up to 8000 local variables. To make the `LiveInterval` unmanaged, a new data structure called `LiveIntervalList` was added to store child splits. This differs from `SortedList<,>` because it can contain intervals with the same start position. Really wished we got some more of C++ template in C#. :^( * Optimize `GetChildSplit` a bit No need to inspect the remaining ranges if we've reached a range which starts after position, since the split list is ordered. * Optimize `CopyResolver` a bit Lazily allocate the fill, spill and parallel copy structures since most of the time only one of them is needed. * Optimize `BitMap.Enumerator` a bit Marking `MoveNext` as `AggressiveInlining` allows RyuJIT to promote the `Enumerator` struct into registers completely, reducing load/store code a lot since it does not have to store the struct on the stack for ABI purposes. * Use stack allocation for `use/blockedPositions` * Optimize `AllocateWithSpill` a bit * Address feedback * Make `LiveInterval.AddRange(,)` more conservative Produces no diff against master, but just for good measure.
394 lines
10 KiB
C#
394 lines
10 KiB
C#
using ARMeilleure.IntermediateRepresentation;
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Diagnostics;
|
|
|
|
namespace ARMeilleure.CodeGen.RegisterAllocators
|
|
{
|
|
unsafe readonly struct LiveInterval : IComparable<LiveInterval>
|
|
{
|
|
public const int NotFound = -1;
|
|
|
|
private struct Data
|
|
{
|
|
public int End;
|
|
public int SpillOffset;
|
|
|
|
public LiveRange FirstRange;
|
|
public LiveRange PrevRange;
|
|
public LiveRange CurrRange;
|
|
|
|
public LiveInterval Parent;
|
|
|
|
public UseList Uses;
|
|
public LiveIntervalList Children;
|
|
|
|
public Operand Local;
|
|
public Register Register;
|
|
|
|
public bool IsFixed;
|
|
}
|
|
|
|
private readonly Data* _data;
|
|
|
|
private ref int End => ref _data->End;
|
|
private ref LiveRange FirstRange => ref _data->FirstRange;
|
|
private ref LiveRange CurrRange => ref _data->CurrRange;
|
|
private ref LiveRange PrevRange => ref _data->PrevRange;
|
|
private ref LiveInterval Parent => ref _data->Parent;
|
|
private ref UseList Uses => ref _data->Uses;
|
|
private ref LiveIntervalList Children => ref _data->Children;
|
|
|
|
public Operand Local => _data->Local;
|
|
public ref Register Register => ref _data->Register;
|
|
public ref int SpillOffset => ref _data->SpillOffset;
|
|
|
|
public bool IsFixed => _data->IsFixed;
|
|
public bool IsEmpty => FirstRange == default;
|
|
public bool IsSplit => Children.Count != 0;
|
|
public bool IsSpilled => SpillOffset != -1;
|
|
|
|
public int UsesCount => Uses.Count;
|
|
|
|
public LiveInterval(Operand local = default, LiveInterval parent = default)
|
|
{
|
|
_data = Allocators.LiveIntervals.Allocate<Data>();
|
|
*_data = default;
|
|
|
|
_data->IsFixed = false;
|
|
_data->Local = local;
|
|
|
|
Parent = parent == default ? this : parent;
|
|
Uses = new UseList();
|
|
Children = new LiveIntervalList();
|
|
|
|
FirstRange = default;
|
|
CurrRange = default;
|
|
PrevRange = default;
|
|
|
|
SpillOffset = -1;
|
|
}
|
|
|
|
public LiveInterval(Register register) : this(local: default, parent: default)
|
|
{
|
|
_data->IsFixed = true;
|
|
|
|
Register = register;
|
|
}
|
|
|
|
public void Reset()
|
|
{
|
|
PrevRange = default;
|
|
CurrRange = FirstRange;
|
|
}
|
|
|
|
public void Forward(int position)
|
|
{
|
|
LiveRange prev = PrevRange;
|
|
LiveRange curr = CurrRange;
|
|
|
|
while (curr != default && curr.Start < position && !curr.Overlaps(position))
|
|
{
|
|
prev = curr;
|
|
curr = curr.Next;
|
|
}
|
|
|
|
PrevRange = prev;
|
|
CurrRange = curr;
|
|
}
|
|
|
|
public int GetStart()
|
|
{
|
|
Debug.Assert(!IsEmpty, "Empty LiveInterval cannot have a start position.");
|
|
|
|
return FirstRange.Start;
|
|
}
|
|
|
|
public void SetStart(int position)
|
|
{
|
|
if (FirstRange != default)
|
|
{
|
|
Debug.Assert(position != FirstRange.End);
|
|
|
|
FirstRange.Start = position;
|
|
}
|
|
else
|
|
{
|
|
FirstRange = new LiveRange(position, position + 1);
|
|
End = position + 1;
|
|
}
|
|
}
|
|
|
|
public int GetEnd()
|
|
{
|
|
Debug.Assert(!IsEmpty, "Empty LiveInterval cannot have an end position.");
|
|
|
|
return End;
|
|
}
|
|
|
|
public void AddRange(int start, int end)
|
|
{
|
|
Debug.Assert(start < end, $"Invalid range start position {start}, {end}");
|
|
|
|
if (FirstRange != default)
|
|
{
|
|
// If the new range ends exactly where the first range start, then coalesce together.
|
|
if (end == FirstRange.Start)
|
|
{
|
|
FirstRange.Start = start;
|
|
|
|
return;
|
|
}
|
|
// If the new range is already contained, then coalesce together.
|
|
else if (FirstRange.Overlaps(start, end))
|
|
{
|
|
FirstRange.Start = Math.Min(FirstRange.Start, start);
|
|
FirstRange.End = Math.Max(FirstRange.End, end);
|
|
End = Math.Max(End, end);
|
|
|
|
Debug.Assert(FirstRange.Next == default || !FirstRange.Overlaps(FirstRange.Next));
|
|
return;
|
|
}
|
|
}
|
|
|
|
FirstRange = new LiveRange(start, end, FirstRange);
|
|
End = Math.Max(End, end);
|
|
|
|
Debug.Assert(FirstRange.Next == default || !FirstRange.Overlaps(FirstRange.Next));
|
|
}
|
|
|
|
public void AddUsePosition(int position)
|
|
{
|
|
Uses.Add(position);
|
|
}
|
|
|
|
public bool Overlaps(int position)
|
|
{
|
|
LiveRange curr = CurrRange;
|
|
|
|
while (curr != default && curr.Start <= position)
|
|
{
|
|
if (curr.Overlaps(position))
|
|
{
|
|
return true;
|
|
}
|
|
|
|
curr = curr.Next;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
public bool Overlaps(LiveInterval other)
|
|
{
|
|
return GetOverlapPosition(other) != NotFound;
|
|
}
|
|
|
|
public int GetOverlapPosition(LiveInterval other)
|
|
{
|
|
LiveRange a = CurrRange;
|
|
LiveRange b = other.CurrRange;
|
|
|
|
while (a != default)
|
|
{
|
|
while (b != default && b.Start < a.Start)
|
|
{
|
|
if (a.Overlaps(b))
|
|
{
|
|
return a.Start;
|
|
}
|
|
|
|
b = b.Next;
|
|
}
|
|
|
|
if (b == default)
|
|
{
|
|
break;
|
|
}
|
|
else if (a.Overlaps(b))
|
|
{
|
|
return a.Start;
|
|
}
|
|
|
|
a = a.Next;
|
|
}
|
|
|
|
return NotFound;
|
|
}
|
|
|
|
public ReadOnlySpan<LiveInterval> SplitChildren()
|
|
{
|
|
return Parent.Children.Span;
|
|
}
|
|
|
|
public ReadOnlySpan<int> UsePositions()
|
|
{
|
|
return Uses.Span;
|
|
}
|
|
|
|
public int FirstUse()
|
|
{
|
|
return Uses.FirstUse;
|
|
}
|
|
|
|
public int NextUseAfter(int position)
|
|
{
|
|
return Uses.NextUse(position);
|
|
}
|
|
|
|
public LiveInterval Split(int position)
|
|
{
|
|
LiveInterval result = new(Local, Parent);
|
|
result.End = End;
|
|
|
|
LiveRange prev = PrevRange;
|
|
LiveRange curr = CurrRange;
|
|
|
|
while (curr != default && curr.Start < position && !curr.Overlaps(position))
|
|
{
|
|
prev = curr;
|
|
curr = curr.Next;
|
|
}
|
|
|
|
if (curr.Start >= position)
|
|
{
|
|
prev.Next = default;
|
|
|
|
result.FirstRange = curr;
|
|
|
|
End = prev.End;
|
|
}
|
|
else
|
|
{
|
|
result.FirstRange = new LiveRange(position, curr.End, curr.Next);
|
|
|
|
curr.End = position;
|
|
curr.Next = default;
|
|
|
|
End = curr.End;
|
|
}
|
|
|
|
result.Uses = Uses.Split(position);
|
|
|
|
AddSplitChild(result);
|
|
|
|
Debug.Assert(!IsEmpty, "Left interval is empty after split.");
|
|
Debug.Assert(!result.IsEmpty, "Right interval is empty after split.");
|
|
|
|
// Make sure the iterator in the new split is pointing to the start.
|
|
result.Reset();
|
|
|
|
return result;
|
|
}
|
|
|
|
private void AddSplitChild(LiveInterval child)
|
|
{
|
|
Debug.Assert(!child.IsEmpty, "Trying to insert an empty interval.");
|
|
|
|
Parent.Children.Add(child);
|
|
}
|
|
|
|
public LiveInterval GetSplitChild(int position)
|
|
{
|
|
if (Overlaps(position))
|
|
{
|
|
return this;
|
|
}
|
|
|
|
foreach (LiveInterval splitChild in SplitChildren())
|
|
{
|
|
if (splitChild.Overlaps(position))
|
|
{
|
|
return splitChild;
|
|
}
|
|
else if (splitChild.GetStart() > position)
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
return default;
|
|
}
|
|
|
|
public bool TrySpillWithSiblingOffset()
|
|
{
|
|
foreach (LiveInterval splitChild in SplitChildren())
|
|
{
|
|
if (splitChild.IsSpilled)
|
|
{
|
|
Spill(splitChild.SpillOffset);
|
|
|
|
return true;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
public void Spill(int offset)
|
|
{
|
|
SpillOffset = offset;
|
|
}
|
|
|
|
public int CompareTo(LiveInterval interval)
|
|
{
|
|
if (FirstRange == default || interval.FirstRange == default)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
return GetStart().CompareTo(interval.GetStart());
|
|
}
|
|
|
|
public bool Equals(LiveInterval interval)
|
|
{
|
|
return interval._data == _data;
|
|
}
|
|
|
|
public override bool Equals(object obj)
|
|
{
|
|
return obj is LiveInterval interval && Equals(interval);
|
|
}
|
|
|
|
public static bool operator ==(LiveInterval a, LiveInterval b)
|
|
{
|
|
return a.Equals(b);
|
|
}
|
|
|
|
public static bool operator !=(LiveInterval a, LiveInterval b)
|
|
{
|
|
return !a.Equals(b);
|
|
}
|
|
|
|
public override int GetHashCode()
|
|
{
|
|
return HashCode.Combine((IntPtr)_data);
|
|
}
|
|
|
|
public override string ToString()
|
|
{
|
|
LiveInterval self = this;
|
|
|
|
IEnumerable<string> GetRanges()
|
|
{
|
|
LiveRange curr = self.CurrRange;
|
|
|
|
while (curr != default)
|
|
{
|
|
if (curr == self.CurrRange)
|
|
{
|
|
yield return "*" + curr;
|
|
}
|
|
else
|
|
{
|
|
yield return curr.ToString();
|
|
}
|
|
|
|
curr = curr.Next;
|
|
}
|
|
}
|
|
|
|
return string.Join(", ", GetRanges());
|
|
}
|
|
}
|
|
} |