Add on translation call counting

ARMeilleure/Instructions/InstEmitFlowHelper.cs

@@ -200,7 +200,7 @@ namespace ARMeilleure.Instructions
             }
         }
 
-        public static void EmitTailContinue(ArmEmitterContext context, Operand address, bool allowRejit)
+        public static void EmitTailContinue(ArmEmitterContext context, Operand address)
         {
             // Left option here as it may be useful if we need to return to managed rather than tail call in future.
             // (eg. for debug)
@@ -218,9 +218,8 @@ namespace ARMeilleure.Instructions
                 {
                     context.StoreToContext();
 
-                    Operand fallbackAddr = context.Call(typeof(NativeInterface).GetMethod(allowRejit
+                    Operand fallbackAddr = context.Call(
-                        ? nameof(NativeInterface.GetFunctionAddress)
+                        typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress)), address);
-                        : nameof(NativeInterface.GetFunctionAddressWithoutRejit)), address);
 
                     EmitNativeCall(context, fallbackAddr, isJump: true);
                 }

ARMeilleure/Instructions/NativeInterface.cs

@@ -220,6 +220,11 @@ namespace ARMeilleure.Instructions
         }
         #endregion
 
+        public static void EnqueueForRejit(ulong address)
+        {
+            Context.Translator.EnqueueForRejit(address, GetContext().ExecutionMode);
+        }
+
         public static void SignalMemoryTracking(ulong address, ulong size, bool write)
         {
             GetMemoryManager().SignalMemoryTracking(address, size, write);
@@ -232,24 +237,14 @@ namespace ARMeilleure.Instructions
 
         public static ulong GetFunctionAddress(ulong address)
         {
-            return GetFunctionAddressWithHint(address, true);
+            TranslatedFunction function = Context.Translator.GetOrTranslate(address, GetContext().ExecutionMode);
-        }
-
-        public static ulong GetFunctionAddressWithoutRejit(ulong address)
-        {
-            return GetFunctionAddressWithHint(address, false);
-        }
-
-        private static ulong GetFunctionAddressWithHint(ulong address, bool hintRejit)
-        {
-            TranslatedFunction function = Context.Translator.GetOrTranslate(address, GetContext().ExecutionMode, hintRejit);
 
             return (ulong)function.FuncPtr.ToInt64();
         }
 
         public static ulong GetIndirectFunctionAddress(ulong address, ulong entryAddress)
         {
-            TranslatedFunction function = Context.Translator.GetOrTranslate(address, GetContext().ExecutionMode, hintRejit: true);
+            TranslatedFunction function = Context.Translator.GetOrTranslate(address, GetContext().ExecutionMode);
 
             ulong ptr = (ulong)function.FuncPtr.ToInt64();
 

ARMeilleure/IntermediateRepresentation/OperandHelper.cs

@@ -1,4 +1,5 @@
 using ARMeilleure.Common;
+using System.Runtime.CompilerServices;
 
 namespace ARMeilleure.IntermediateRepresentation
 {
@@ -34,6 +35,11 @@ namespace ARMeilleure.IntermediateRepresentation
             return Operand().With(value);
         }
 
+        public static unsafe Operand Const<T>(ref T reference)
+        {
+            return Operand().With((ulong)Unsafe.AsPointer(ref reference));
+        }
+
         public static Operand ConstF(float value)
         {
             return Operand().With(value);

ARMeilleure/Translation/ArmEmitterContext.cs

@@ -1,3 +1,4 @@
+using ARMeilleure.Common;
 using ARMeilleure.Decoders;
 using ARMeilleure.Instructions;
 using ARMeilleure.IntermediateRepresentation;
@@ -41,15 +42,23 @@ namespace ARMeilleure.Translation
         public IMemoryManager Memory { get; }
 
         public JumpTable JumpTable { get; }
+        public EntryTable<byte> CountTable { get; }
 
         public ulong EntryAddress { get; }
         public bool HighCq { get; }
         public Aarch32Mode Mode { get; }
 
-        public ArmEmitterContext(IMemoryManager memory, JumpTable jumpTable, ulong entryAddress, bool highCq, Aarch32Mode mode)
+        public ArmEmitterContext(
+            IMemoryManager memory,
+            JumpTable jumpTable,
+            EntryTable<byte> countTable,
+            ulong entryAddress,
+            bool highCq,
+            Aarch32Mode mode)
         {
             Memory = memory;
             JumpTable = jumpTable;
+            CountTable = countTable;
             EntryAddress = entryAddress;
             HighCq = highCq;
             Mode = mode;

ARMeilleure/Translation/Delegates.cs

@@ -103,6 +103,7 @@ namespace ARMeilleure.Translation
 
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.Break)));
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.CheckSynchronization)));
+            SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.EnqueueForRejit)));
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetCntfrqEl0)));
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetCntpctEl0)));
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetCntvctEl0)));
@@ -113,7 +114,6 @@ namespace ARMeilleure.Translation
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFpscr))); // A32 only.
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFpsr)));
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress)));
-            SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddressWithoutRejit)));
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetIndirectFunctionAddress)));
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetTpidr)));
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetTpidr32))); // A32 only.

ARMeilleure/Translation/PTC/Ptc.cs

@@ -1,6 +1,7 @@
 using ARMeilleure.CodeGen;
 using ARMeilleure.CodeGen.Unwinding;
 using ARMeilleure.CodeGen.X86;
+using ARMeilleure.Common;
 using ARMeilleure.Memory;
 using ARMeilleure.Translation.Cache;
 using Ryujinx.Common;
@@ -771,7 +772,7 @@ namespace ARMeilleure.Translation.PTC
             }
         }
 
-        internal static void MakeAndSaveTranslations(ConcurrentDictionary<ulong, TranslatedFunction> funcs, IMemoryManager memory, JumpTable jumpTable)
+        internal static void MakeAndSaveTranslations(ConcurrentDictionary<ulong, TranslatedFunction> funcs, IMemoryManager memory, JumpTable jumpTable, EntryTable<byte> countTable)
         {
             var profiledFuncsToTranslate = PtcProfiler.GetProfiledFuncsToTranslate(funcs);
 
@@ -813,7 +814,7 @@ namespace ARMeilleure.Translation.PTC
 
                     Debug.Assert(PtcProfiler.IsAddressInStaticCodeRange(address));
 
-                    TranslatedFunction func = Translator.Translate(memory, jumpTable, address, item.mode, item.highCq);
+                    TranslatedFunction func = Translator.Translate(memory, jumpTable, countTable, address, item.mode, item.highCq);
 
                     bool isAddressUnique = funcs.TryAdd(address, func);
 

ARMeilleure/Translation/Translator.cs

@@ -1,3 +1,4 @@
+using ARMeilleure.Common;
 using ARMeilleure.Decoders;
 using ARMeilleure.Diagnostics;
 using ARMeilleure.Instructions;
@@ -10,7 +11,6 @@ using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Linq;
 using System.Runtime;
 using System.Threading;
 
@@ -22,36 +22,45 @@ namespace ARMeilleure.Translation
 {
     public class Translator
     {
+        private long _nextUpdate;
+
         private readonly IJitMemoryAllocator _allocator;
         private readonly IMemoryManager _memory;
 
         private readonly ConcurrentDictionary<ulong, TranslatedFunction> _funcs;
         private readonly ConcurrentQueue<KeyValuePair<ulong, IntPtr>> _oldFuncs;
 
+        private readonly ConcurrentDictionary<ulong, object> _backgroundSet;
         private readonly ConcurrentStack<RejitRequest> _backgroundStack;
         private readonly AutoResetEvent _backgroundTranslatorEvent;
         private readonly ReaderWriterLock _backgroundTranslatorLock;
 
         private JumpTable _jumpTable;
         internal JumpTable JumpTable => _jumpTable;
+        internal EntryTable<byte> CountTable { get; }
 
         private volatile int _threadCount;
 
         // FIXME: Remove this once the init logic of the emulator will be redone.
-        public static ManualResetEvent IsReadyForTranslation = new ManualResetEvent(false);
+        public static readonly ManualResetEvent IsReadyForTranslation = new(false);
 
         public Translator(IJitMemoryAllocator allocator, IMemoryManager memory)
         {
+            _nextUpdate = Stopwatch.GetTimestamp();
+
             _allocator = allocator;
             _memory = memory;
 
             _funcs = new ConcurrentDictionary<ulong, TranslatedFunction>();
             _oldFuncs = new ConcurrentQueue<KeyValuePair<ulong, IntPtr>>();
 
+            _backgroundSet = new ConcurrentDictionary<ulong, object>();
             _backgroundStack = new ConcurrentStack<RejitRequest>();
             _backgroundTranslatorEvent = new AutoResetEvent(false);
             _backgroundTranslatorLock = new ReaderWriterLock();
 
+            CountTable = new EntryTable<byte>(capacity: 16 * 1024 * 1024);
+
             JitCache.Initialize(allocator);
 
             DirectCallStubs.InitializeStubs();
@@ -63,9 +72,16 @@ namespace ARMeilleure.Translation
             {
                 _backgroundTranslatorLock.AcquireReaderLock(Timeout.Infinite);
 
-                if (_backgroundStack.TryPop(out RejitRequest request))
+                if (_backgroundStack.TryPop(out RejitRequest request) && 
+                    _backgroundSet.TryRemove(request.Address, out _))
                 {
-                    TranslatedFunction func = Translate(_memory, _jumpTable, request.Address, request.Mode, highCq: true);
+                    TranslatedFunction func = Translate(
+                        _memory,
+                        _jumpTable,
+                        CountTable,
+                        request.Address,
+                        request.Mode,
+                        highCq: true);
 
                     _funcs.AddOrUpdate(request.Address, func, (key, oldFunc) =>
                     {
@@ -80,6 +96,26 @@ namespace ARMeilleure.Translation
                         PtcProfiler.UpdateEntry(request.Address, request.Mode, highCq: true);
                     }
 
+                    var nextUpdate = Interlocked.Exchange(ref _nextUpdate, 0);
+
+                    if (nextUpdate != 0)
+                    {
+                        var now = Stopwatch.GetTimestamp();
+
+                        if (now < nextUpdate)
+                        {
+                            _nextUpdate = nextUpdate;
+                        }
+                        else
+                        {
+                            Ryujinx.Common.Logging.Logger.Info?.Print(
+                                Ryujinx.Common.Logging.LogClass.Cpu,
+                                $"{_backgroundStack.Count} rejit requests remaining");
+
+                            _nextUpdate = now + Stopwatch.Frequency * 30;
+                        }
+                    }
+
                     _backgroundTranslatorLock.ReleaseReaderLock();
                 }
                 else
@@ -89,7 +125,8 @@ namespace ARMeilleure.Translation
                 }
             }
 
-            _backgroundTranslatorEvent.Set(); // Wake up any other background translator threads, to encourage them to exit.
+             // Wake up any other background translator threads, to encourage them to exit.
+            _backgroundTranslatorEvent.Set();
         }
 
         public void Execute(State.ExecutionContext context, ulong address)
@@ -105,17 +142,20 @@ namespace ARMeilleure.Translation
                 {
                     Debug.Assert(_funcs.Count == 0);
                     Ptc.LoadTranslations(_funcs, _memory, _jumpTable);
-                    Ptc.MakeAndSaveTranslations(_funcs, _memory, _jumpTable);
+                    Ptc.MakeAndSaveTranslations(_funcs, _memory, _jumpTable, CountTable);
                 }
 
                 PtcProfiler.Start();
 
                 Ptc.Disable();
 
-                // Simple heuristic, should be user configurable in future. (1 for 4 core/ht or less, 2 for 6 core+ht etc).
+                // Simple heuristic, should be user configurable in future. (1 for 4 core/ht or less, 2 for 6 core + ht
-                // All threads are normal priority except from the last, which just fills as much of the last core as the os lets it with a low priority.
+                // etc). All threads are normal priority except from the last, which just fills as much of the last core
-                // If we only have one rejit thread, it should be normal priority as highCq code is performance critical.
+                // as the os lets it with a low priority. If we only have one rejit thread, it should be normal priority
-                // TODO: Use physical cores rather than logical. This only really makes sense for processors with hyperthreading. Requires OS specific code.
+                // as highCq code is performance critical.
+                //
+                // TODO: Use physical cores rather than logical. This only really makes sense for processors with
+                // hyperthreading. Requires OS specific code.
                 int unboundedThreadCount = Math.Max(1, (Environment.ProcessorCount - 6) / 3);
                 int threadCount          = Math.Min(4, unboundedThreadCount);
 
@@ -173,11 +213,11 @@ namespace ARMeilleure.Translation
             return nextAddr;
         }
 
-        internal TranslatedFunction GetOrTranslate(ulong address, ExecutionMode mode, bool hintRejit = false)
+        internal TranslatedFunction GetOrTranslate(ulong address, ExecutionMode mode)
         {
             if (!_funcs.TryGetValue(address, out TranslatedFunction func))
             {
-                func = Translate(_memory, _jumpTable, address, mode, highCq: false);
+                func = Translate(_memory, _jumpTable, CountTable, address, mode, highCq: false);
 
                 TranslatedFunction getFunc = _funcs.GetOrAdd(address, func);
 
@@ -193,18 +233,18 @@ namespace ARMeilleure.Translation
                 }
             }
 
-            if (hintRejit && func.ShouldRejit())
-            {
-                _backgroundStack.Push(new RejitRequest(address, mode));
-                _backgroundTranslatorEvent.Set();
-            }
-
             return func;
         }
 
-        internal static TranslatedFunction Translate(IMemoryManager memory, JumpTable jumpTable, ulong address, ExecutionMode mode, bool highCq)
+        internal static TranslatedFunction Translate(
+            IMemoryManager memory,
+            JumpTable jumpTable,
+            EntryTable<byte> countTable,
+            ulong address,
+            ExecutionMode mode,
+            bool highCq)
         {
-            ArmEmitterContext context = new ArmEmitterContext(memory, jumpTable, address, highCq, Aarch32Mode.User);
+            var context = new ArmEmitterContext(memory, jumpTable, countTable, address, highCq, Aarch32Mode.User);
 
             Logger.StartPass(PassName.Decoding);
 
@@ -216,6 +256,11 @@ namespace ARMeilleure.Translation
 
             Logger.StartPass(PassName.Translation);
 
+            if (!context.HighCq)
+            {
+                EmitRejitCheck(context);
+            }
+
             EmitSynchronization(context);
 
             if (blocks[0].Address != address)
@@ -320,7 +365,7 @@ namespace ARMeilleure.Translation
 
                 if (block.Exit)
                 {
-                    InstEmitFlowHelper.EmitTailContinue(context, Const(block.Address), block.TailCall);
+                    InstEmitFlowHelper.EmitTailContinue(context, Const(block.Address));
                 }
                 else
                 {
@@ -368,29 +413,51 @@ namespace ARMeilleure.Translation
             return context.GetControlFlowGraph();
         }
 
+        internal static void EmitRejitCheck(ArmEmitterContext context)
+        {
+            if (!context.CountTable.TryAllocate(out int index))
+            {
+                return;
+            }
+
+            Operand lblRejit = Label();
+            Operand lblAdd = Label();
+            Operand lblEnd = Label();
+
+            // TODO: PPTC.
+            Operand address = Const(ref context.CountTable.GetValue(index));
+            Operand count = context.Load8(address);
+            context.BranchIf(lblAdd, count, Const(100), Comparison.LessUI);
+            context.BranchIf(lblRejit, count, Const(100), Comparison.Equal);
+            context.Branch(lblEnd);
+
+            context.MarkLabel(lblRejit, BasicBlockFrequency.Cold);
+            context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.EnqueueForRejit)), Const(context.EntryAddress));
+
+            context.MarkLabel(lblAdd, BasicBlockFrequency.Cold);
+            context.Store8(address, context.Add(count, Const(1)));
+
+            context.MarkLabel(lblEnd);
+        }
+
         internal static void EmitSynchronization(EmitterContext context)
         {
             long countOffs = NativeContext.GetCounterOffset();
 
-            Operand countAddr = context.Add(context.LoadArgument(OperandType.I64, 0), Const(countOffs));
-
-            Operand count = context.Load(OperandType.I32, countAddr);
-
             Operand lblNonZero = Label();
             Operand lblExit = Label();
 
+            Operand countAddr = context.Add(context.LoadArgument(OperandType.I64, 0), Const(countOffs));
+            Operand count = context.Load(OperandType.I32, countAddr);
             context.BranchIfTrue(lblNonZero, count, BasicBlockFrequency.Cold);
 
             Operand running = context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.CheckSynchronization)));
-
             context.BranchIfTrue(lblExit, running, BasicBlockFrequency.Cold);
 
             context.Return(Const(0L));
 
             context.MarkLabel(lblNonZero);
-
             count = context.Subtract(count, Const(1));
-
             context.Store(countAddr, count);
 
             context.MarkLabel(lblExit);
@@ -404,6 +471,15 @@ namespace ARMeilleure.Translation
             // TODO: Completely remove functions overlapping the specified range from the cache.
         }
 
+        internal void EnqueueForRejit(ulong guestAddress, ExecutionMode mode)
+        {
+            if (_backgroundSet.TryAdd(guestAddress, null))
+            {
+                _backgroundStack.Push(new RejitRequest(guestAddress, mode));
+                _backgroundTranslatorEvent.Set();
+            }
+        }
+
         private void EnqueueForDeletion(ulong guestAddress, TranslatedFunction func)
         {
             _oldFuncs.Enqueue(new KeyValuePair<ulong, IntPtr>(guestAddress, func.FuncPtr));
@@ -439,6 +515,8 @@ namespace ARMeilleure.Translation
                     {
                         func.ResetCallCount();
                     }
+
+                    _backgroundSet.TryRemove(request.Address, out _);
                 }
             }
             else