IPC refactor part 1: Use explicit separate threads to process requests (#1447)

* Changes to allow explicit management of service threads

* Remove now unused code

* Remove ThreadCounter, its no longer needed

* Allow and use separate server per service, also fix exit issues

* New policy change: PTC version now uses PR number

ARMeilleure/Instructions/InstEmitException.cs

@@ -1,7 +1,6 @@
 using ARMeilleure.Decoders;
 using ARMeilleure.Translation;
 
-using static ARMeilleure.Instructions.InstEmitFlowHelper;
 using static ARMeilleure.IntermediateRepresentation.OperandHelper;
 
 namespace ARMeilleure.Instructions
@@ -27,6 +26,8 @@ namespace ARMeilleure.Instructions
             context.Call(typeof(NativeInterface).GetMethod(name), Const(op.Address), Const(op.Id));
 
             context.LoadFromContext();
+
+            Translator.EmitSynchronization(context);
         }
 
         public static void Und(ArmEmitterContext context)

ARMeilleure/Instructions/InstEmitException32.cs

@@ -27,6 +27,8 @@ namespace ARMeilleure.Instructions
             context.Call(typeof(NativeInterface).GetMethod(name), Const(op.Address), Const(op.Id));
 
             context.LoadFromContext();
+
+            Translator.EmitSynchronization(context);
         }
     }
 }

ARMeilleure/Translation/PTC/Ptc.cs

@@ -21,7 +21,7 @@ namespace ARMeilleure.Translation.PTC
     {
         private const string HeaderMagic = "PTChd";
 
-        private const int InternalVersion = 1528; //! To be incremented manually for each change to the ARMeilleure project.
+        private const int InternalVersion = 1447; //! To be incremented manually for each change to the ARMeilleure project.
 
         private const string ActualDir = "0";
         private const string BackupDir = "1";

ARMeilleure/Translation/Translator.cs

@@ -290,7 +290,7 @@ namespace ARMeilleure.Translation
             return context.GetControlFlowGraph();
         }
 
-        private static void EmitSynchronization(EmitterContext context)
+        internal static void EmitSynchronization(EmitterContext context)
         {
             long countOffs = NativeContext.GetCounterOffset();
 

Ryujinx.HLE/HOS/Horizon.cs

@@ -30,7 +30,7 @@ using Ryujinx.HLE.Loaders.Executables;
 using Ryujinx.HLE.Utilities;
 using System;
 using System.IO;
-
+using System.Threading;
 
 namespace Ryujinx.HLE.HOS
 {
@@ -318,18 +318,19 @@ namespace Ryujinx.HLE.HOS
 
                 terminationThread.Start();
 
+                // Wait until the thread is actually started.
+                while (terminationThread.HostThread.ThreadState == ThreadState.Unstarted)
+                {
+                    Thread.Sleep(10);
+                }
+
+                // Wait until the termination thread is done terminating all the other threads.
+                terminationThread.HostThread.Join();
+
                 // Destroy nvservices channels as KThread could be waiting on some user events.
                 // This is safe as KThread that are likely to call ioctls are going to be terminated by the post handler hook on the SVC facade.
                 INvDrvServices.Destroy();
 
-                // This is needed as the IPC Dummy KThread is also counted in the ThreadCounter.
-                KernelContext.ThreadCounter.Signal();
-
-                // It's only safe to release resources once all threads
-                // have exited.
-                KernelContext.ThreadCounter.Signal();
-                KernelContext.ThreadCounter.Wait();
-
                 AudioRendererManager.Dispose();
 
                 KernelContext.Dispose();

Ryujinx.HLE/HOS/Kernel/KernelContext.cs

@@ -24,8 +24,6 @@ namespace Ryujinx.HLE.HOS.Kernel
         public Syscall Syscall { get; }
         public SyscallHandler SyscallHandler { get; }
 
-        public CountdownEvent ThreadCounter { get; }
-
         public KResourceLimit ResourceLimit { get; }
 
         public KMemoryRegionManager[] MemoryRegions { get; }
@@ -57,8 +55,6 @@ namespace Ryujinx.HLE.HOS.Kernel
 
             SyscallHandler = new SyscallHandler(this);
 
-            ThreadCounter = new CountdownEvent(1);
-
             ResourceLimit = new KResourceLimit(this);
 
             KernelInit.InitializeResourceLimit(ResourceLimit);

Ryujinx.HLE/HOS/Kernel/Process/KProcess.cs

@@ -791,19 +791,16 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
         private void InterruptHandler(object sender, EventArgs e)
         {
             KernelContext.Scheduler.ContextSwitch();
+            KernelContext.Scheduler.GetCurrentThread().HandlePostSyscall();
         }
 
         public void IncrementThreadCount()
         {
             Interlocked.Increment(ref _threadCount);
-
-            KernelContext.ThreadCounter.AddCount();
         }
 
         public void DecrementThreadCountAndTerminateIfZero()
         {
-            KernelContext.ThreadCounter.Signal();
-
             if (Interlocked.Decrement(ref _threadCount) == 0)
             {
                 Terminate();
@@ -812,8 +809,6 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
 
         public void DecrementToZeroWhileTerminatingCurrent()
         {
-            KernelContext.ThreadCounter.Signal();
-
             while (Interlocked.Decrement(ref _threadCount) != 0)
             {
                 Destroy();
@@ -1000,24 +995,29 @@ namespace Ryujinx.HLE.HOS.Kernel.Process
                 KernelContext.CriticalSection.Leave();
             }
 
-            KThread blockedThread = null;
+            while (true)
-
-            lock (_threadingLock)
             {
-                foreach (KThread thread in _threads)
+                KThread blockedThread = null;
-                {
-                    if (thread != currentThread && (thread.SchedFlags & ThreadSchedState.LowMask) != ThreadSchedState.TerminationPending)
-                    {
-                        thread.IncrementReferenceCount();
 
-                        blockedThread = thread;
+                lock (_threadingLock)
-                        break;
+                {
+                    foreach (KThread thread in _threads)
+                    {
+                        if (thread != currentThread && (thread.SchedFlags & ThreadSchedState.LowMask) != ThreadSchedState.TerminationPending)
+                        {
+                            thread.IncrementReferenceCount();
+
+                            blockedThread = thread;
+                            break;
+                        }
                     }
                 }
-            }
 
-            if (blockedThread != null)
+                if (blockedThread == null)
-            {
+                {
+                    break;
+                }
+
                 blockedThread.Terminate();
                 blockedThread.DecrementReferenceCount();
             }

Ryujinx.HLE/HOS/Kernel/SupervisorCall/Syscall.cs

@@ -2,14 +2,12 @@
 using Ryujinx.Common.Logging;
 using Ryujinx.Cpu;
 using Ryujinx.HLE.Exceptions;
-using Ryujinx.HLE.HOS.Ipc;
 using Ryujinx.HLE.HOS.Kernel.Common;
 using Ryujinx.HLE.HOS.Kernel.Ipc;
 using Ryujinx.HLE.HOS.Kernel.Memory;
 using Ryujinx.HLE.HOS.Kernel.Process;
 using Ryujinx.HLE.HOS.Kernel.Threading;
 using System;
-using System.Threading;
 
 namespace Ryujinx.HLE.HOS.Kernel.SupervisorCall
 {
@@ -26,29 +24,6 @@ namespace Ryujinx.HLE.HOS.Kernel.SupervisorCall
 
         // IPC
 
-        private struct HleIpcMessage
-        {
-            public KProcess Process { get; }
-            public KThread Thread { get; }
-            public KClientSession Session { get; }
-            public IpcMessage Message { get; }
-            public long MessagePtr { get; }
-
-            public HleIpcMessage(
-                KProcess process,
-                KThread thread,
-                KClientSession session,
-                IpcMessage message,
-                long messagePtr)
-            {
-                Process = process;
-                Thread = thread;
-                Session = session;
-                Message = message;
-                MessagePtr = messagePtr;
-            }
-        }
-
         public KernelResult ConnectToNamedPort(ulong namePtr, out int handle)
         {
             handle = 0;
@@ -135,16 +110,7 @@ namespace Ryujinx.HLE.HOS.Kernel.SupervisorCall
 
                 currentThread.Reschedule(ThreadSchedState.Paused);
 
-                IpcMessage message = new IpcMessage(messageData, (long)messagePtr);
+                clientSession.Service.Server.PushMessage(_device, currentThread, clientSession, messagePtr, messageSize);
-
-                ThreadPool.QueueUserWorkItem(ProcessIpcRequest, new HleIpcMessage(
-                    process,
-                    currentThread,
-                    clientSession,
-                    message,
-                    (long)messagePtr));
-
-                _context.ThreadCounter.AddCount();
 
                 _context.CriticalSection.Leave();
 
@@ -158,24 +124,6 @@ namespace Ryujinx.HLE.HOS.Kernel.SupervisorCall
             }
         }
 
-        private void ProcessIpcRequest(object state)
-        {
-            HleIpcMessage ipcMessage = (HleIpcMessage)state;
-
-            ipcMessage.Thread.ObjSyncResult = IpcHandler.IpcCall(
-                _device,
-                ipcMessage.Process,
-                ipcMessage.Process.CpuMemory,
-                ipcMessage.Thread,
-                ipcMessage.Session,
-                ipcMessage.Message,
-                ipcMessage.MessagePtr);
-
-            _context.ThreadCounter.Signal();
-
-            ipcMessage.Thread.Reschedule(ThreadSchedState.Running);
-        }
-
         private KernelResult SendSyncRequest(int handle)
         {
             KProcess currentProcess = _context.Scheduler.GetCurrentProcess();

Ryujinx.HLE/HOS/Kernel/Threading/KThread.cs

@@ -348,6 +348,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
                     if ((SchedFlags & ThreadSchedState.LowMask) == ThreadSchedState.Running)
                     {
                         // TODO: GIC distributor stuffs (sgir changes ect)
+                        Context.RequestInterrupt();
                     }
 
                     SignaledObj   = null;

Ryujinx.HLE/HOS/Services/Audio/IAudioOutManager.cs

@@ -14,7 +14,7 @@ namespace Ryujinx.HLE.HOS.Services.Audio
         private const int    DefaultSampleRate    = 48000;
         private const int    DefaultChannelsCount = 2;
 
-        public IAudioOutManager(ServiceCtx context) { }
+        public IAudioOutManager(ServiceCtx context) : base(new ServerBase("AudioOutServer")) { }
 
         [Command(0)]
         // ListAudioOuts() -> (u32 count, buffer<bytes, 6>)

Ryujinx.HLE/HOS/Services/CommandAttributes.cs

@@ -3,7 +3,7 @@
 namespace Ryujinx.HLE.HOS.Services
 {
     [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)]
-    public class CommandAttribute : Attribute
+    class CommandAttribute : Attribute
     {
         public readonly int Id;
 

Ryujinx.HLE/HOS/Services/IpcService.cs

@@ -15,13 +15,13 @@ namespace Ryujinx.HLE.HOS.Services
     {
         public IReadOnlyDictionary<int, MethodInfo> Commands { get; }
 
+        public ServerBase Server { get; private set; }
+
         private IdDictionary _domainObjects;
-
         private int _selfId;
-
         private bool _isDomain;
 
-        public IpcService()
+        public IpcService(ServerBase server = null)
         {
             Commands = Assembly.GetExecutingAssembly().GetTypes()
                 .Where(type => type == GetType())
@@ -30,8 +30,9 @@ namespace Ryujinx.HLE.HOS.Services
                 .Select(command => (((CommandAttribute)command).Id, methodInfo)))
                 .ToDictionary(command => command.Id, command => command.methodInfo);
 
-            _domainObjects = new IdDictionary();
+            Server = server;
 
+            _domainObjects = new IdDictionary();
             _selfId = -1;
         }
 
@@ -152,6 +153,8 @@ namespace Ryujinx.HLE.HOS.Services
         {
             IpcService service = context.Session.Service;
 
+            obj.TrySetServer(service.Server);
+
             if (service._isDomain)
             {
                 context.Response.ObjectIds.Add(service.Add(obj));
@@ -194,6 +197,18 @@ namespace Ryujinx.HLE.HOS.Services
             return obj is T ? (T)obj : null;
         }
 
+        public bool TrySetServer(ServerBase newServer)
+        {
+            if (Server == null)
+            {
+                Server = newServer;
+
+                return true;
+            }
+
+            return false;
+        }
+
         private int Add(IIpcService obj)
         {
             return _domainObjects.Add(obj);

Ryujinx.HLE/HOS/Services/Nv/INvDrvServices.cs

@@ -45,7 +45,7 @@ namespace Ryujinx.HLE.HOS.Services.Nv
 
         private bool _transferMemInitialized = false;
 
-        public INvDrvServices(ServiceCtx context)
+        public INvDrvServices(ServiceCtx context) : base(new ServerBase("NvservicesServer"))
         {
             _owner = null;
         }

Ryujinx.HLE/HOS/Services/ServerBase.cs

@@ -1,4 +1,5 @@
-using Ryujinx.Cpu;
+using Ryujinx.Common;
+using Ryujinx.HLE.HOS.Ipc;
 using Ryujinx.HLE.HOS.Kernel.Common;
 using Ryujinx.HLE.HOS.Kernel.Ipc;
 using Ryujinx.HLE.HOS.Kernel.Process;
@@ -6,19 +7,54 @@ using Ryujinx.HLE.HOS.Kernel.Threading;
 using System;
 using System.IO;
 
-namespace Ryujinx.HLE.HOS.Ipc
+namespace Ryujinx.HLE.HOS.Services
 {
-    static class IpcHandler
+    class ServerBase
     {
-        public static KernelResult IpcCall(
+        private struct IpcRequest
-            Switch         device,
-            KProcess       process,
-            MemoryManager  memory,
-            KThread        thread,
-            KClientSession session,
-            IpcMessage     request,
-            long           cmdPtr)
         {
+            public Switch Device { get; }
+            public KProcess Process => Thread?.Owner;
+            public KThread Thread { get; }
+            public KClientSession Session { get; }
+            public ulong MessagePtr { get; }
+            public ulong MessageSize { get; }
+
+            public IpcRequest(Switch device, KThread thread, KClientSession session, ulong messagePtr, ulong messageSize)
+            {
+                Device = device;
+                Thread = thread;
+                Session = session;
+                MessagePtr = messagePtr;
+                MessageSize = messageSize;
+            }
+
+            public void SignalDone(KernelResult result)
+            {
+                Thread.ObjSyncResult = result;
+                Thread.Reschedule(ThreadSchedState.Running);
+            }
+        }
+
+        private readonly AsyncWorkQueue<IpcRequest> _ipcProcessor;
+
+        public ServerBase(string name)
+        {
+            _ipcProcessor = new AsyncWorkQueue<IpcRequest>(Process, name);
+        }
+
+        public void PushMessage(Switch device, KThread thread, KClientSession session, ulong messagePtr, ulong messageSize)
+        {
+            _ipcProcessor.Add(new IpcRequest(device, thread, session, messagePtr, messageSize));
+        }
+
+        private void Process(IpcRequest message)
+        {
+            byte[] reqData = new byte[message.MessageSize];
+
+            message.Process.CpuMemory.Read(message.MessagePtr, reqData);
+
+            IpcMessage request = new IpcMessage(reqData, (long)message.MessagePtr);
             IpcMessage response = new IpcMessage();
 
             using (MemoryStream raw = new MemoryStream(request.RawData))
@@ -35,17 +71,17 @@ namespace Ryujinx.HLE.HOS.Ipc
                         BinaryWriter resWriter = new BinaryWriter(resMs);
 
                         ServiceCtx context = new ServiceCtx(
-                            device,
+                            message.Device,
-                            process,
+                            message.Process,
-                            memory,
+                            message.Process.CpuMemory,
-                            thread,
+                            message.Thread,
-                            session,
+                            message.Session,
                             request,
                             response,
                             reqReader,
                             resWriter);
 
-                        session.Service.CallMethod(context);
+                        message.Session.Service.CallMethod(context);
 
                         response.RawData = resMs.ToArray();
                     }
@@ -59,26 +95,19 @@ namespace Ryujinx.HLE.HOS.Ipc
                     switch (cmdId)
                     {
                         case 0:
-                        {
+                            request = FillResponse(response, 0, message.Session.Service.ConvertToDomain());
-                            request = FillResponse(response, 0, session.Service.ConvertToDomain());
-
                             break;
-                        }
 
                         case 3:
-                        {
                             request = FillResponse(response, 0, 0x1000);
-
                             break;
-                        }
 
                         // TODO: Whats the difference between IpcDuplicateSession/Ex?
                         case 2:
                         case 4:
-                        {
                             int unknown = reqReader.ReadInt32();
 
-                            if (process.HandleTable.GenerateHandle(session, out int handle) != KernelResult.Success)
+                            if (message.Process.HandleTable.GenerateHandle(message.Session, out int handle) != KernelResult.Success)
                             {
                                 throw new InvalidOperationException("Out of handles!");
                             }
@@ -88,25 +117,24 @@ namespace Ryujinx.HLE.HOS.Ipc
                             request = FillResponse(response, 0);
 
                             break;
-                        }
 
                         default: throw new NotImplementedException(cmdId.ToString());
                     }
                 }
                 else if (request.Type == IpcMessageType.CloseSession)
                 {
-                    // TODO
+                    message.SignalDone(KernelResult.PortRemoteClosed);
-                    return KernelResult.PortRemoteClosed;
+                    return;
                 }
                 else
                 {
                     throw new NotImplementedException(request.Type.ToString());
                 }
 
-                memory.Write((ulong)cmdPtr, response.GetBytes(cmdPtr));
+                message.Process.CpuMemory.Write(message.MessagePtr, response.GetBytes((long)message.MessagePtr));
             }
 
-            return KernelResult.Success;
+            message.SignalDone(KernelResult.Success);
         }
 
         private static IpcMessage FillResponse(IpcMessage response, long result, params int[] values)

Ryujinx.HLE/HOS/Services/ServiceAttributes.cs

@@ -3,7 +3,7 @@
 namespace Ryujinx.HLE.HOS.Services
 {
     [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)]
-    public class ServiceAttribute : Attribute
+    class ServiceAttribute : Attribute
     {
         public readonly string Name;
         public readonly object Parameter;

Ryujinx.HLE/HOS/Services/Sm/IUserInterface.cs

@@ -18,9 +18,11 @@ namespace Ryujinx.HLE.HOS.Services.Sm
 
         private ConcurrentDictionary<string, KPort> _registeredServices;
 
+        private readonly ServerBase _commonServer;
+
         private bool _isInitialized;
 
-        public IUserInterface(ServiceCtx context = null)
+        public IUserInterface(ServiceCtx context = null) : base(new ServerBase("SmServer"))
         {
             _registeredServices = new ConcurrentDictionary<string, KPort>();
 
@@ -28,6 +30,8 @@ namespace Ryujinx.HLE.HOS.Services.Sm
                 .SelectMany(type => type.GetCustomAttributes(typeof(ServiceAttribute), true)
                 .Select(service => (((ServiceAttribute)service).Name, type)))
                 .ToDictionary(service => service.Name, service => service.type);
+
+            _commonServer = new ServerBase("CommonServer");
         }
 
         public static void InitializePort(Horizon system)
@@ -36,7 +40,9 @@ namespace Ryujinx.HLE.HOS.Services.Sm
 
             port.ClientPort.SetName("sm:");
 
-            port.ClientPort.Service = new IUserInterface();
+            IUserInterface smService = new IUserInterface();
+
+            port.ClientPort.Service = smService;
         }
 
         [Command(0)]
@@ -81,8 +87,13 @@ namespace Ryujinx.HLE.HOS.Services.Sm
                 {
                     ServiceAttribute serviceAttribute = (ServiceAttribute)type.GetCustomAttributes(typeof(ServiceAttribute)).First(service => ((ServiceAttribute)service).Name == name);
 
-                    session.ClientSession.Service = serviceAttribute.Parameter != null ? (IpcService)Activator.CreateInstance(type, context, serviceAttribute.Parameter)
+                    IpcService service = serviceAttribute.Parameter != null
-                                                                                       : (IpcService)Activator.CreateInstance(type, context);
+                        ? (IpcService)Activator.CreateInstance(type, context, serviceAttribute.Parameter)
+                        : (IpcService)Activator.CreateInstance(type, context);
+
+                    service.TrySetServer(_commonServer);
+
+                    session.ClientSession.Service = service;
                 }
                 else
                 {

Ryujinx.HLE/HOS/Services/Sockets/Bsd/IClient.cs

@@ -102,7 +102,7 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Bsd
 
         private List<BsdSocket> _sockets = new List<BsdSocket>();
 
-        public IClient(ServiceCtx context, bool isPrivileged)
+        public IClient(ServiceCtx context, bool isPrivileged)  : base(new ServerBase("BsdServer"))
         {
             _isPrivileged = isPrivileged;
         }

Ryujinx.HLE/HOS/Services/Vi/IApplicationRootService.cs

@@ -5,7 +5,7 @@ namespace Ryujinx.HLE.HOS.Services.Vi
     [Service("vi:u")]
     class IApplicationRootService : IpcService
     {
-        public IApplicationRootService(ServiceCtx context) { }
+        public IApplicationRootService(ServiceCtx context) : base(new ServerBase("ViServer")) { }
 
         [Command(0)]
         // GetDisplayService(u32) -> object<nn::visrv::sf::IApplicationDisplayService>