Add inlined on translation call counting (#2190)

* Add EntryTable<TEntry>

* Add on translation call counting

* Add Counter

* Add PPTC support

* Make Counter a generic & use a 32-bit counter instead

* Return false on overflow

* Set PPTC version

* Print more information about the rejit queue

* Make Counter<T> disposable

* Remove Block.TailCall since it is not used anymore

* Apply suggestions from code review

Address gdkchan's feedback

Co-authored-by: gdkchan <gab.dark.100@gmail.com>

* Fix more stale docs

* Remove rejit requests queue logging

* Make Counter<T> finalizable

Most certainly quite an odd use case.

* Make EntryTable<T>.TryAllocate set entry to default

* Re-trigger CI

* Dispose Counters before they hit the finalizer queue

* Re-trigger CI

Just for good measure...

* Make EntryTable<T> expandable

* EntryTable is now expandable instead of being a fixed slab.
* Remove EntryTable<T>.TryAllocate
* Remove Counter<T>.TryCreate

Address LDj3SNuD's feedback

* Apply suggestions from code review

Address LDj3SNuD's feedback

Co-authored-by: LDj3SNuD <35856442+LDj3SNuD@users.noreply.github.com>

* Remove useless return

* POH approach, but the sequel

* Revert "POH approach, but the sequel"

This reverts commit 5f5abaa247.

The sequel got shelved

* Add extra documentation

Co-authored-by: gdkchan <gab.dark.100@gmail.com>
Co-authored-by: LDj3SNuD <35856442+LDj3SNuD@users.noreply.github.com>
This commit is contained in:
FICTURE7 2021-04-19 01:43:53 +04:00 committed by GitHub
parent bb43219f1d
commit 89791ba68d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 468 additions and 92 deletions

View file

@ -0,0 +1,99 @@
using System;
namespace ARMeilleure.Common
{
/// <summary>
/// Represents a numeric counter which can be used for instrumentation of compiled code.
/// </summary>
/// <typeparam name="T">Type of the counter</typeparam>
class Counter<T> : IDisposable where T : unmanaged
{
private bool _disposed;
private readonly int _index;
private readonly EntryTable<T> _countTable;
/// <summary>
/// Initializes a new instance of the <see cref="Counter{T}"/> class from the specified
/// <see cref="EntryTable{T}"/> instance and index.
/// </summary>
/// <param name="countTable"><see cref="EntryTable{T}"/> instance</param>
/// <param name="index">Index in the <see cref="EntryTable{T}"/></param>
/// <exception cref="ArgumentNullException"><paramref name="countTable"/> is <see langword="null"/></exception>
/// <exception cref="ArgumentException"><typeparamref name="T"/> is unsupported</exception>
public Counter(EntryTable<T> countTable)
{
if (typeof(T) != typeof(byte) && typeof(T) != typeof(sbyte) &&
typeof(T) != typeof(short) && typeof(T) != typeof(ushort) &&
typeof(T) != typeof(int) && typeof(T) != typeof(uint) &&
typeof(T) != typeof(long) && typeof(T) != typeof(ulong) &&
typeof(T) != typeof(nint) && typeof(T) != typeof(nuint) &&
typeof(T) != typeof(float) && typeof(T) != typeof(double))
{
throw new ArgumentException("Counter does not support the specified type.");
}
_countTable = countTable ?? throw new ArgumentNullException(nameof(countTable));
_index = countTable.Allocate();
}
/// <summary>
/// Gets a reference to the value of the counter.
/// </summary>
/// <exception cref="ObjectDisposedException"><see cref="Counter{T}"/> instance was disposed</exception>
/// <remarks>
/// This can refer to freed memory if the owning <see cref="EntryTable{TEntry}"/> is disposed.
/// </remarks>
public ref T Value
{
get
{
if (_disposed)
{
throw new ObjectDisposedException(null);
}
return ref _countTable.GetValue(_index);
}
}
/// <summary>
/// Releases all resources used by the <see cref="Counter{T}"/> instance.
/// </summary>
public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
/// <summary>
/// Releases all unmanaged and optionally managed resources used by the <see cref="Counter{T}"/> instance.
/// </summary>
/// <param name="disposing"><see langword="true"/> to dispose managed resources also; otherwise just unmanaged resouces</param>
protected virtual void Dispose(bool disposing)
{
if (!_disposed)
{
try
{
// The index into the EntryTable is essentially an unmanaged resource since we allocate and free the
// resource ourselves.
_countTable.Free(_index);
}
catch (ObjectDisposedException)
{
// Can happen because _countTable may be disposed before the Counter instance.
}
_disposed = true;
}
}
/// <summary>
/// Frees resources used by the <see cref="Counter{T}"/> instance.
/// </summary>
~Counter()
{
Dispose(false);
}
}
}

View file

@ -0,0 +1,196 @@
using System;
using System.Collections.Generic;
using System.Numerics;
using System.Runtime.InteropServices;
namespace ARMeilleure.Common
{
/// <summary>
/// Represents an expandable table of the type <typeparamref name="TEntry"/>, whose entries will remain at the same
/// address through out the table's lifetime.
/// </summary>
/// <typeparam name="TEntry">Type of the entry in the table</typeparam>
class EntryTable<TEntry> : IDisposable where TEntry : unmanaged
{
private bool _disposed;
private int _freeHint;
private readonly int _pageCapacity; // Number of entries per page.
private readonly int _pageLogCapacity;
private readonly Dictionary<int, IntPtr> _pages;
private readonly BitMap _allocated;
/// <summary>
/// Initializes a new instance of the <see cref="EntryTable{TEntry}"/> class with the desired page size in
/// bytes.
/// </summary>
/// <param name="pageSize">Desired page size in bytes</param>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="pageSize"/> is less than 0</exception>
/// <exception cref="ArgumentException"><typeparamref name="TEntry"/>'s size is zero</exception>
/// <remarks>
/// The actual page size may be smaller or larger depending on the size of <typeparamref name="TEntry"/>.
/// </remarks>
public unsafe EntryTable(int pageSize = 4096)
{
if (pageSize < 0)
{
throw new ArgumentOutOfRangeException(nameof(pageSize), "Page size cannot be negative.");
}
if (sizeof(TEntry) == 0)
{
throw new ArgumentException("Size of TEntry cannot be zero.");
}
_allocated = new BitMap();
_pages = new Dictionary<int, IntPtr>();
_pageLogCapacity = BitOperations.Log2((uint)(pageSize / sizeof(TEntry)));
_pageCapacity = 1 << _pageLogCapacity;
}
/// <summary>
/// Allocates an entry in the <see cref="EntryTable{TEntry}"/>.
/// </summary>
/// <returns>Index of entry allocated in the table</returns>
/// <exception cref="ObjectDisposedException"><see cref="EntryTable{TEntry}"/> instance was disposed</exception>
public int Allocate()
{
if (_disposed)
{
throw new ObjectDisposedException(null);
}
lock (_allocated)
{
if (_allocated.IsSet(_freeHint))
{
_freeHint = _allocated.FindFirstUnset();
}
int index = _freeHint++;
var page = GetPage(index);
_allocated.Set(index);
GetValue(page, index) = default;
return index;
}
}
/// <summary>
/// Frees the entry at the specified <paramref name="index"/>.
/// </summary>
/// <param name="index">Index of entry to free</param>
/// <exception cref="ObjectDisposedException"><see cref="EntryTable{TEntry}"/> instance was disposed</exception>
public void Free(int index)
{
if (_disposed)
{
throw new ObjectDisposedException(null);
}
lock (_allocated)
{
if (_allocated.IsSet(index))
{
_allocated.Clear(index);
_freeHint = index;
}
}
}
/// <summary>
/// Gets a reference to the entry at the specified allocated <paramref name="index"/>.
/// </summary>
/// <param name="index">Index of the entry</param>
/// <returns>Reference to the entry at the specified <paramref name="index"/></returns>
/// <exception cref="ObjectDisposedException"><see cref="EntryTable{TEntry}"/> instance was disposed</exception>
/// <exception cref="ArgumentException">Entry at <paramref name="index"/> is not allocated</exception>
public ref TEntry GetValue(int index)
{
if (_disposed)
{
throw new ObjectDisposedException(null);
}
lock (_allocated)
{
if (!_allocated.IsSet(index))
{
throw new ArgumentException("Entry at the specified index was not allocated", nameof(index));
}
var page = GetPage(index);
return ref GetValue(page, index);
}
}
/// <summary>
/// Gets a reference to the entry at using the specified <paramref name="index"/> from the specified
/// <paramref name="page"/>.
/// </summary>
/// <param name="page">Page to use</param>
/// <param name="index">Index to use</param>
/// <returns>Reference to the entry</returns>
private ref TEntry GetValue(Span<TEntry> page, int index)
{
return ref page[index & (_pageCapacity - 1)];
}
/// <summary>
/// Gets the page for the specified <see cref="index"/>.
/// </summary>
/// <param name="index">Index to use</param>
/// <returns>Page for the specified <see cref="index"/></returns>
private unsafe Span<TEntry> GetPage(int index)
{
var pageIndex = (int)((uint)(index & ~(_pageCapacity - 1)) >> _pageLogCapacity);
if (!_pages.TryGetValue(pageIndex, out IntPtr page))
{
page = Marshal.AllocHGlobal(sizeof(TEntry) * _pageCapacity);
_pages.Add(pageIndex, page);
}
return new Span<TEntry>((void*)page, _pageCapacity);
}
/// <summary>
/// Releases all resources used by the <see cref="EntryTable{TEntry}"/> instance.
/// </summary>
public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
/// <summary>
/// Releases all unmanaged and optionally managed resources used by the <see cref="EntryTable{TEntry}{T}"/>
/// instance.
/// </summary>
/// <param name="disposing"><see langword="true"/> to dispose managed resources also; otherwise just unmanaged resouces</param>
protected virtual void Dispose(bool disposing)
{
if (!_disposed)
{
foreach (var page in _pages.Values)
{
Marshal.FreeHGlobal(page);
}
_disposed = true;
}
}
/// <summary>
/// Frees resources used by the <see cref="EntryTable{TEntry}"/> instance.
/// </summary>
~EntryTable()
{
Dispose(false);
}
}
}

View file

@ -11,7 +11,6 @@ namespace ARMeilleure.Decoders
public Block Next { get; set; }
public Block Branch { get; set; }
public bool TailCall { get; set; }
public bool Exit { get; set; }
public List<OpCode> OpCodes { get; }

View file

@ -58,7 +58,7 @@ namespace ARMeilleure.Decoders.Optimizations
return blocks.ToArray(); // Nothing to do here.
}
// Mark branches outside of contiguous region as exit blocks.
// Mark branches whose target is outside of the contiguous region as an exit block.
for (int i = startBlockIndex; i <= endBlockIndex; i++)
{
Block block = blocks[i];
@ -66,7 +66,6 @@ namespace ARMeilleure.Decoders.Optimizations
if (block.Branch != null && (block.Branch.Address > endBlock.EndAddress || block.Branch.EndAddress < startBlock.Address))
{
block.Branch.Exit = true;
block.Branch.TailCall = true;
}
}

View file

@ -200,7 +200,7 @@ namespace ARMeilleure.Instructions
}
}
public static void EmitTailContinue(ArmEmitterContext context, Operand address, bool allowRejit)
public static void EmitTailContinue(ArmEmitterContext context, Operand address)
{
// Left option here as it may be useful if we need to return to managed rather than tail call in future.
// (eg. for debug)
@ -218,9 +218,7 @@ namespace ARMeilleure.Instructions
{
context.StoreToContext();
Operand fallbackAddr = context.Call(typeof(NativeInterface).GetMethod(allowRejit
? nameof(NativeInterface.GetFunctionAddress)
: nameof(NativeInterface.GetFunctionAddressWithoutRejit)), address);
Operand fallbackAddr = context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress)), address);
EmitNativeCall(context, fallbackAddr, isJump: true);
}

View file

@ -220,6 +220,11 @@ namespace ARMeilleure.Instructions
}
#endregion
public static void EnqueueForRejit(ulong address)
{
Context.Translator.EnqueueForRejit(address, GetContext().ExecutionMode);
}
public static void SignalMemoryTracking(ulong address, ulong size, bool write)
{
GetMemoryManager().SignalMemoryTracking(address, size, write);
@ -232,24 +237,14 @@ namespace ARMeilleure.Instructions
public static ulong GetFunctionAddress(ulong address)
{
return GetFunctionAddressWithHint(address, true);
}
public static ulong GetFunctionAddressWithoutRejit(ulong address)
{
return GetFunctionAddressWithHint(address, false);
}
private static ulong GetFunctionAddressWithHint(ulong address, bool hintRejit)
{
TranslatedFunction function = Context.Translator.GetOrTranslate(address, GetContext().ExecutionMode, hintRejit);
TranslatedFunction function = Context.Translator.GetOrTranslate(address, GetContext().ExecutionMode);
return (ulong)function.FuncPtr.ToInt64();
}
public static ulong GetIndirectFunctionAddress(ulong address, ulong entryAddress)
{
TranslatedFunction function = Context.Translator.GetOrTranslate(address, GetContext().ExecutionMode, hintRejit: true);
TranslatedFunction function = Context.Translator.GetOrTranslate(address, GetContext().ExecutionMode);
ulong ptr = (ulong)function.FuncPtr.ToInt64();

View file

@ -1,4 +1,5 @@
using ARMeilleure.Common;
using System.Runtime.CompilerServices;
namespace ARMeilleure.IntermediateRepresentation
{
@ -34,6 +35,11 @@ namespace ARMeilleure.IntermediateRepresentation
return Operand().With(value);
}
public static unsafe Operand Const<T>(ref T reference, int? index = null)
{
return Operand().With((long)Unsafe.AsPointer(ref reference), index != null, index);
}
public static Operand ConstF(float value)
{
return Operand().With(value);

View file

@ -1,3 +1,4 @@
using ARMeilleure.Common;
using ARMeilleure.Decoders;
using ARMeilleure.Instructions;
using ARMeilleure.IntermediateRepresentation;
@ -41,15 +42,23 @@ namespace ARMeilleure.Translation
public IMemoryManager Memory { get; }
public JumpTable JumpTable { get; }
public EntryTable<uint> CountTable { get; }
public ulong EntryAddress { get; }
public bool HighCq { get; }
public Aarch32Mode Mode { get; }
public ArmEmitterContext(IMemoryManager memory, JumpTable jumpTable, ulong entryAddress, bool highCq, Aarch32Mode mode)
public ArmEmitterContext(
IMemoryManager memory,
JumpTable jumpTable,
EntryTable<uint> countTable,
ulong entryAddress,
bool highCq,
Aarch32Mode mode)
{
Memory = memory;
JumpTable = jumpTable;
CountTable = countTable;
EntryAddress = entryAddress;
HighCq = highCq;
Mode = mode;

View file

@ -103,6 +103,7 @@ namespace ARMeilleure.Translation
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.Break)));
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.CheckSynchronization)));
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.EnqueueForRejit)));
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetCntfrqEl0)));
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetCntpctEl0)));
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetCntvctEl0)));
@ -113,7 +114,6 @@ namespace ARMeilleure.Translation
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFpscr))); // A32 only.
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFpsr)));
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress)));
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddressWithoutRejit)));
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetIndirectFunctionAddress)));
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetTpidr)));
SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetTpidr32))); // A32 only.

View file

@ -1,6 +1,7 @@
using ARMeilleure.CodeGen;
using ARMeilleure.CodeGen.Unwinding;
using ARMeilleure.CodeGen.X86;
using ARMeilleure.Common;
using ARMeilleure.Memory;
using ARMeilleure.Translation.Cache;
using Ryujinx.Common;
@ -27,7 +28,7 @@ namespace ARMeilleure.Translation.PTC
private const string OuterHeaderMagicString = "PTCohd\0\0";
private const string InnerHeaderMagicString = "PTCihd\0\0";
private const uint InternalVersion = 2169; //! To be incremented manually for each change to the ARMeilleure project.
private const uint InternalVersion = 2190; //! To be incremented manually for each change to the ARMeilleure project.
private const string ActualDir = "0";
private const string BackupDir = "1";
@ -38,6 +39,7 @@ namespace ARMeilleure.Translation.PTC
internal const int PageTablePointerIndex = -1; // Must be a negative value.
internal const int JumpPointerIndex = -2; // Must be a negative value.
internal const int DynamicPointerIndex = -3; // Must be a negative value.
internal const int CountTableIndex = -4; // Must be a negative value.
private const byte FillingByte = 0x00;
private const CompressionLevel SaveCompressionLevel = CompressionLevel.Fastest;
@ -538,7 +540,11 @@ namespace ARMeilleure.Translation.PTC
}
}
internal static void LoadTranslations(ConcurrentDictionary<ulong, TranslatedFunction> funcs, IMemoryManager memory, JumpTable jumpTable)
internal static void LoadTranslations(
ConcurrentDictionary<ulong, TranslatedFunction> funcs,
IMemoryManager memory,
JumpTable jumpTable,
EntryTable<uint> countTable)
{
if (AreCarriersEmpty())
{
@ -567,16 +573,18 @@ namespace ARMeilleure.Translation.PTC
{
byte[] code = ReadCode(index, infoEntry.CodeLength);
Counter<uint> callCounter = null;
if (infoEntry.RelocEntriesCount != 0)
{
RelocEntry[] relocEntries = GetRelocEntries(relocsReader, infoEntry.RelocEntriesCount);
PatchCode(code.AsSpan(), relocEntries, memory.PageTablePointer, jumpTable);
PatchCode(code, relocEntries, memory.PageTablePointer, jumpTable, countTable, out callCounter);
}
UnwindInfo unwindInfo = ReadUnwindInfo(unwindInfosReader);
TranslatedFunction func = FastTranslate(code, infoEntry.GuestSize, unwindInfo, infoEntry.HighCq);
TranslatedFunction func = FastTranslate(code, callCounter, infoEntry.GuestSize, unwindInfo, infoEntry.HighCq);
bool isAddressUnique = funcs.TryAdd(infoEntry.Address, func);
@ -670,8 +678,16 @@ namespace ARMeilleure.Translation.PTC
return relocEntries;
}
private static void PatchCode(Span<byte> code, RelocEntry[] relocEntries, IntPtr pageTablePointer, JumpTable jumpTable)
private static void PatchCode(
Span<byte> code,
RelocEntry[] relocEntries,
IntPtr pageTablePointer,
JumpTable jumpTable,
EntryTable<uint> countTable,
out Counter<uint> callCounter)
{
callCounter = null;
foreach (RelocEntry relocEntry in relocEntries)
{
ulong imm;
@ -688,6 +704,12 @@ namespace ARMeilleure.Translation.PTC
{
imm = (ulong)jumpTable.DynamicPointer.ToInt64();
}
else if (relocEntry.Index == CountTableIndex)
{
callCounter = new Counter<uint>(countTable);
unsafe { imm = (ulong)Unsafe.AsPointer(ref callCounter.Value); }
}
else if (Delegates.TryGetDelegateFuncPtrByIndex(relocEntry.Index, out IntPtr funcPtr))
{
imm = (ulong)funcPtr.ToInt64();
@ -722,7 +744,12 @@ namespace ARMeilleure.Translation.PTC
return new UnwindInfo(pushEntries, prologueSize);
}
private static TranslatedFunction FastTranslate(byte[] code, ulong guestSize, UnwindInfo unwindInfo, bool highCq)
private static TranslatedFunction FastTranslate(
byte[] code,
Counter<uint> callCounter,
ulong guestSize,
UnwindInfo unwindInfo,
bool highCq)
{
CompiledFunction cFunc = new CompiledFunction(code, unwindInfo);
@ -730,7 +757,7 @@ namespace ARMeilleure.Translation.PTC
GuestFunction gFunc = Marshal.GetDelegateForFunctionPointer<GuestFunction>(codePtr);
TranslatedFunction tFunc = new TranslatedFunction(gFunc, guestSize, highCq);
TranslatedFunction tFunc = new TranslatedFunction(gFunc, callCounter, guestSize, highCq);
return tFunc;
}
@ -771,7 +798,11 @@ namespace ARMeilleure.Translation.PTC
}
}
internal static void MakeAndSaveTranslations(ConcurrentDictionary<ulong, TranslatedFunction> funcs, IMemoryManager memory, JumpTable jumpTable)
internal static void MakeAndSaveTranslations(
ConcurrentDictionary<ulong, TranslatedFunction> funcs,
IMemoryManager memory,
JumpTable jumpTable,
EntryTable<uint> countTable)
{
var profiledFuncsToTranslate = PtcProfiler.GetProfiledFuncsToTranslate(funcs);
@ -813,7 +844,7 @@ namespace ARMeilleure.Translation.PTC
Debug.Assert(PtcProfiler.IsAddressInStaticCodeRange(address));
TranslatedFunction func = Translator.Translate(memory, jumpTable, address, item.mode, item.highCq);
TranslatedFunction func = Translator.Translate(memory, jumpTable, countTable, address, item.mode, item.highCq);
bool isAddressUnique = funcs.TryAdd(address, func);

View file

@ -1,24 +1,22 @@
using ARMeilleure.Common;
using System;
using System.Runtime.InteropServices;
using System.Threading;
namespace ARMeilleure.Translation
{
class TranslatedFunction
{
private const int MinCallsForRejit = 100;
private readonly GuestFunction _func; // Ensure that this delegate will not be garbage collected.
private int _callCount;
public Counter<uint> CallCounter { get; }
public ulong GuestSize { get; }
public bool HighCq { get; }
public IntPtr FuncPtr { get; }
public TranslatedFunction(GuestFunction func, ulong guestSize, bool highCq)
public TranslatedFunction(GuestFunction func, Counter<uint> callCounter, ulong guestSize, bool highCq)
{
_func = func;
CallCounter = callCounter;
GuestSize = guestSize;
HighCq = highCq;
FuncPtr = Marshal.GetFunctionPointerForDelegate(func);
@ -28,15 +26,5 @@ namespace ARMeilleure.Translation
{
return _func(context.NativeContextPtr);
}
public bool ShouldRejit()
{
return !HighCq && Interlocked.Increment(ref _callCount) == MinCallsForRejit;
}
public void ResetCallCount()
{
Interlocked.Exchange(ref _callCount, 0);
}
}
}

View file

@ -1,3 +1,4 @@
using ARMeilleure.Common;
using ARMeilleure.Decoders;
using ARMeilleure.Diagnostics;
using ARMeilleure.Instructions;
@ -10,7 +11,6 @@ using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Runtime;
using System.Threading;
@ -22,23 +22,27 @@ namespace ARMeilleure.Translation
{
public class Translator
{
private const int CountTableCapacity = 4 * 1024 * 1024;
private readonly IJitMemoryAllocator _allocator;
private readonly IMemoryManager _memory;
private readonly ConcurrentDictionary<ulong, TranslatedFunction> _funcs;
private readonly ConcurrentQueue<KeyValuePair<ulong, IntPtr>> _oldFuncs;
private readonly ConcurrentQueue<KeyValuePair<ulong, TranslatedFunction>> _oldFuncs;
private readonly ConcurrentDictionary<ulong, object> _backgroundSet;
private readonly ConcurrentStack<RejitRequest> _backgroundStack;
private readonly AutoResetEvent _backgroundTranslatorEvent;
private readonly ReaderWriterLock _backgroundTranslatorLock;
private JumpTable _jumpTable;
internal JumpTable JumpTable => _jumpTable;
internal EntryTable<uint> CountTable { get; }
private volatile int _threadCount;
// FIXME: Remove this once the init logic of the emulator will be redone.
public static ManualResetEvent IsReadyForTranslation = new ManualResetEvent(false);
public static readonly ManualResetEvent IsReadyForTranslation = new(false);
public Translator(IJitMemoryAllocator allocator, IMemoryManager memory)
{
@ -46,12 +50,15 @@ namespace ARMeilleure.Translation
_memory = memory;
_funcs = new ConcurrentDictionary<ulong, TranslatedFunction>();
_oldFuncs = new ConcurrentQueue<KeyValuePair<ulong, IntPtr>>();
_oldFuncs = new ConcurrentQueue<KeyValuePair<ulong, TranslatedFunction>>();
_backgroundSet = new ConcurrentDictionary<ulong, object>();
_backgroundStack = new ConcurrentStack<RejitRequest>();
_backgroundTranslatorEvent = new AutoResetEvent(false);
_backgroundTranslatorLock = new ReaderWriterLock();
CountTable = new EntryTable<uint>();
JitCache.Initialize(allocator);
DirectCallStubs.InitializeStubs();
@ -63,9 +70,16 @@ namespace ARMeilleure.Translation
{
_backgroundTranslatorLock.AcquireReaderLock(Timeout.Infinite);
if (_backgroundStack.TryPop(out RejitRequest request))
if (_backgroundStack.TryPop(out RejitRequest request) &&
_backgroundSet.TryRemove(request.Address, out _))
{
TranslatedFunction func = Translate(_memory, _jumpTable, request.Address, request.Mode, highCq: true);
TranslatedFunction func = Translate(
_memory,
_jumpTable,
CountTable,
request.Address,
request.Mode,
highCq: true);
_funcs.AddOrUpdate(request.Address, func, (key, oldFunc) =>
{
@ -89,7 +103,8 @@ namespace ARMeilleure.Translation
}
}
_backgroundTranslatorEvent.Set(); // Wake up any other background translator threads, to encourage them to exit.
// Wake up any other background translator threads, to encourage them to exit.
_backgroundTranslatorEvent.Set();
}
public void Execute(State.ExecutionContext context, ulong address)
@ -104,18 +119,21 @@ namespace ARMeilleure.Translation
if (Ptc.State == PtcState.Enabled)
{
Debug.Assert(_funcs.Count == 0);
Ptc.LoadTranslations(_funcs, _memory, _jumpTable);
Ptc.MakeAndSaveTranslations(_funcs, _memory, _jumpTable);
Ptc.LoadTranslations(_funcs, _memory, _jumpTable, CountTable);
Ptc.MakeAndSaveTranslations(_funcs, _memory, _jumpTable, CountTable);
}
PtcProfiler.Start();
Ptc.Disable();
// Simple heuristic, should be user configurable in future. (1 for 4 core/ht or less, 2 for 6 core+ht etc).
// All threads are normal priority except from the last, which just fills as much of the last core as the os lets it with a low priority.
// If we only have one rejit thread, it should be normal priority as highCq code is performance critical.
// TODO: Use physical cores rather than logical. This only really makes sense for processors with hyperthreading. Requires OS specific code.
// Simple heuristic, should be user configurable in future. (1 for 4 core/ht or less, 2 for 6 core + ht
// etc). All threads are normal priority except from the last, which just fills as much of the last core
// as the os lets it with a low priority. If we only have one rejit thread, it should be normal priority
// as highCq code is performance critical.
//
// TODO: Use physical cores rather than logical. This only really makes sense for processors with
// hyperthreading. Requires OS specific code.
int unboundedThreadCount = Math.Max(1, (Environment.ProcessorCount - 6) / 3);
int threadCount = Math.Min(4, unboundedThreadCount);
@ -156,6 +174,8 @@ namespace ARMeilleure.Translation
_jumpTable.Dispose();
_jumpTable = null;
CountTable.Dispose();
GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
}
}
@ -173,11 +193,11 @@ namespace ARMeilleure.Translation
return nextAddr;
}
internal TranslatedFunction GetOrTranslate(ulong address, ExecutionMode mode, bool hintRejit = false)
internal TranslatedFunction GetOrTranslate(ulong address, ExecutionMode mode)
{
if (!_funcs.TryGetValue(address, out TranslatedFunction func))
{
func = Translate(_memory, _jumpTable, address, mode, highCq: false);
func = Translate(_memory, _jumpTable, CountTable, address, mode, highCq: false);
TranslatedFunction getFunc = _funcs.GetOrAdd(address, func);
@ -193,18 +213,18 @@ namespace ARMeilleure.Translation
}
}
if (hintRejit && func.ShouldRejit())
{
_backgroundStack.Push(new RejitRequest(address, mode));
_backgroundTranslatorEvent.Set();
}
return func;
}
internal static TranslatedFunction Translate(IMemoryManager memory, JumpTable jumpTable, ulong address, ExecutionMode mode, bool highCq)
internal static TranslatedFunction Translate(
IMemoryManager memory,
JumpTable jumpTable,
EntryTable<uint> countTable,
ulong address,
ExecutionMode mode,
bool highCq)
{
ArmEmitterContext context = new ArmEmitterContext(memory, jumpTable, address, highCq, Aarch32Mode.User);
var context = new ArmEmitterContext(memory, jumpTable, countTable, address, highCq, Aarch32Mode.User);
Logger.StartPass(PassName.Decoding);
@ -216,6 +236,13 @@ namespace ARMeilleure.Translation
Logger.StartPass(PassName.Translation);
Counter<uint> counter = null;
if (!context.HighCq)
{
EmitRejitCheck(context, out counter);
}
EmitSynchronization(context);
if (blocks[0].Address != address)
@ -258,7 +285,7 @@ namespace ARMeilleure.Translation
Ptc.WriteInfoCodeRelocUnwindInfo(address, funcSize, highCq, ptcInfo);
}
return new TranslatedFunction(func, funcSize, highCq);
return new TranslatedFunction(func, counter, funcSize, highCq);
}
internal static void PreparePool(int groupId = 0)
@ -320,7 +347,7 @@ namespace ARMeilleure.Translation
if (block.Exit)
{
InstEmitFlowHelper.EmitTailContinue(context, Const(block.Address), block.TailCall);
InstEmitFlowHelper.EmitTailContinue(context, Const(block.Address));
}
else
{
@ -368,29 +395,43 @@ namespace ARMeilleure.Translation
return context.GetControlFlowGraph();
}
internal static void EmitRejitCheck(ArmEmitterContext context, out Counter<uint> counter)
{
const int MinsCallForRejit = 100;
counter = new Counter<uint>(context.CountTable);
Operand lblEnd = Label();
Operand address = Const(ref counter.Value, Ptc.CountTableIndex);
Operand curCount = context.Load(OperandType.I32, address);
Operand count = context.Add(curCount, Const(1));
context.Store(address, count);
context.BranchIf(lblEnd, curCount, Const(MinsCallForRejit), Comparison.NotEqual, BasicBlockFrequency.Cold);
context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.EnqueueForRejit)), Const(context.EntryAddress));
context.MarkLabel(lblEnd);
}
internal static void EmitSynchronization(EmitterContext context)
{
long countOffs = NativeContext.GetCounterOffset();
Operand countAddr = context.Add(context.LoadArgument(OperandType.I64, 0), Const(countOffs));
Operand count = context.Load(OperandType.I32, countAddr);
Operand lblNonZero = Label();
Operand lblExit = Label();
Operand countAddr = context.Add(context.LoadArgument(OperandType.I64, 0), Const(countOffs));
Operand count = context.Load(OperandType.I32, countAddr);
context.BranchIfTrue(lblNonZero, count, BasicBlockFrequency.Cold);
Operand running = context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.CheckSynchronization)));
context.BranchIfTrue(lblExit, running, BasicBlockFrequency.Cold);
context.Return(Const(0L));
context.MarkLabel(lblNonZero);
count = context.Subtract(count, Const(1));
context.Store(countAddr, count);
context.MarkLabel(lblExit);
@ -404,9 +445,18 @@ namespace ARMeilleure.Translation
// TODO: Completely remove functions overlapping the specified range from the cache.
}
internal void EnqueueForRejit(ulong guestAddress, ExecutionMode mode)
{
if (_backgroundSet.TryAdd(guestAddress, null))
{
_backgroundStack.Push(new RejitRequest(guestAddress, mode));
_backgroundTranslatorEvent.Set();
}
}
private void EnqueueForDeletion(ulong guestAddress, TranslatedFunction func)
{
_oldFuncs.Enqueue(new KeyValuePair<ulong, IntPtr>(guestAddress, func.FuncPtr));
_oldFuncs.Enqueue(new(guestAddress, func));
}
private void ClearJitCache()
@ -414,16 +464,20 @@ namespace ARMeilleure.Translation
// Ensure no attempt will be made to compile new functions due to rejit.
ClearRejitQueue(allowRequeue: false);
foreach (var kv in _funcs)
foreach (var func in _funcs.Values)
{
JitCache.Unmap(kv.Value.FuncPtr);
JitCache.Unmap(func.FuncPtr);
func.CallCounter?.Dispose();
}
_funcs.Clear();
while (_oldFuncs.TryDequeue(out var kv))
{
JitCache.Unmap(kv.Value);
JitCache.Unmap(kv.Value.FuncPtr);
kv.Value.CallCounter?.Dispose();
}
}
@ -435,10 +489,12 @@ namespace ARMeilleure.Translation
{
while (_backgroundStack.TryPop(out var request))
{
if (_funcs.TryGetValue(request.Address, out var func))
if (_funcs.TryGetValue(request.Address, out var func) && func.CallCounter != null)
{
func.ResetCallCount();
Volatile.Write(ref func.CallCounter.Value, 0);
}
_backgroundSet.TryRemove(request.Address, out _);
}
}
else