From 733b6b340b1e941cddc721a7ee4f41459e2713a5 Mon Sep 17 00:00:00 2001 From: LDj3SNuD Date: Mon, 26 Sep 2022 13:29:30 +0200 Subject: [PATCH] RO optimization. Allows the same dynamic module (NRO) to always be remapped to the same base address, so that the Translator can reuse the same dynamic functions in it, without having to retranslate them and thus without having to add them back into the Jit Cache. --- ARMeilleure/Memory/IMemoryManager.cs | 2 +- ARMeilleure/Translation/Translator.cs | 16 +- Ryujinx.Cpu/Jit/JitCpuContext.cs | 4 +- Ryujinx.Cpu/Jit/MemoryManager.cs | 6 +- Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs | 6 +- Ryujinx.HLE/HOS/Kernel/Memory/KPageTable.cs | 4 +- .../HOS/Kernel/Memory/KPageTableBase.cs | 6 +- Ryujinx.HLE/HOS/Services/Ro/IRoInterface.cs | 157 ++++++++++++++---- Ryujinx.Memory/AddressSpaceManager.cs | 2 +- Ryujinx.Memory/IVirtualMemoryManager.cs | 2 +- Ryujinx.Tests/Cpu/CpuContext.cs | 4 +- Ryujinx.Tests/Memory/MockMemoryManager.cs | 2 +- 12 files changed, 155 insertions(+), 56 deletions(-) diff --git a/ARMeilleure/Memory/IMemoryManager.cs b/ARMeilleure/Memory/IMemoryManager.cs index c4ea70d17..f263d8335 100644 --- a/ARMeilleure/Memory/IMemoryManager.cs +++ b/ARMeilleure/Memory/IMemoryManager.cs @@ -10,7 +10,7 @@ namespace ARMeilleure.Memory MemoryManagerType Type { get; } - event Action UnmapEvent; + event Action UnmapEvent; /// /// Reads data from CPU mapped memory. diff --git a/ARMeilleure/Translation/Translator.cs b/ARMeilleure/Translation/Translator.cs index ee8e3e8b5..6b96180b9 100644 --- a/ARMeilleure/Translation/Translator.cs +++ b/ARMeilleure/Translation/Translator.cs @@ -453,15 +453,23 @@ namespace ARMeilleure.Translation context.MarkLabel(lblExit); } - public void InvalidateJitCacheRegion(ulong address, ulong size) + public void InvalidateJitCacheRegion(ulong address, ulong size, bool clearRejitQueueOnly = false) { - // If rejit is running, stop it as it may be trying to rejit a function on the invalidated region. - ClearRejitQueue(allowRequeue: true); - ulong[] overlapAddresses = Array.Empty(); int overlapsCount = Functions.GetOverlaps(address, size, ref overlapAddresses); + if (overlapsCount != 0) + { + // If rejit is running, stop it as it may be trying to rejit a function on the invalidated region. + ClearRejitQueue(allowRequeue: true); + } + + if (clearRejitQueueOnly) + { + return; + } + for (int index = 0; index < overlapsCount; index++) { ulong overlapAddress = overlapAddresses[index]; diff --git a/Ryujinx.Cpu/Jit/JitCpuContext.cs b/Ryujinx.Cpu/Jit/JitCpuContext.cs index d6892ea75..02c360b26 100644 --- a/Ryujinx.Cpu/Jit/JitCpuContext.cs +++ b/Ryujinx.Cpu/Jit/JitCpuContext.cs @@ -15,9 +15,9 @@ namespace Ryujinx.Cpu.Jit memory.UnmapEvent += UnmapHandler; } - private void UnmapHandler(ulong address, ulong size) + private void UnmapHandler(ulong address, ulong size, bool clearRejitQueueOnly = false) { - _translator.InvalidateJitCacheRegion(address, size); + _translator.InvalidateJitCacheRegion(address, size, clearRejitQueueOnly); } /// diff --git a/Ryujinx.Cpu/Jit/MemoryManager.cs b/Ryujinx.Cpu/Jit/MemoryManager.cs index 86c69431d..e57cb8df4 100644 --- a/Ryujinx.Cpu/Jit/MemoryManager.cs +++ b/Ryujinx.Cpu/Jit/MemoryManager.cs @@ -46,7 +46,7 @@ namespace Ryujinx.Cpu.Jit public MemoryTracking Tracking { get; } - public event Action UnmapEvent; + public event Action UnmapEvent; /// /// Creates a new instance of the memory manager. @@ -95,7 +95,7 @@ namespace Ryujinx.Cpu.Jit } /// - public void Unmap(ulong va, ulong size) + public void Unmap(ulong va, ulong size, bool clearRejitQueueOnly = false) { // If size is 0, there's nothing to unmap, just exit early. if (size == 0) @@ -105,7 +105,7 @@ namespace Ryujinx.Cpu.Jit AssertValidAddressAndSize(va, size); - UnmapEvent?.Invoke(va, size); + UnmapEvent?.Invoke(va, size, clearRejitQueueOnly); Tracking.Unmap(va, size); ulong remainingSize = size; diff --git a/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs b/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs index 8994e9c0e..388821857 100644 --- a/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs +++ b/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs @@ -56,7 +56,7 @@ namespace Ryujinx.Cpu.Jit public MemoryTracking Tracking { get; } - public event Action UnmapEvent; + public event Action UnmapEvent; /// /// Creates a new instance of the host mapped memory manager. @@ -158,11 +158,11 @@ namespace Ryujinx.Cpu.Jit } /// - public void Unmap(ulong va, ulong size) + public void Unmap(ulong va, ulong size, bool clearRejitQueueOnly = false) { AssertValidAddressAndSize(va, size); - UnmapEvent?.Invoke(va, size); + UnmapEvent?.Invoke(va, size, clearRejitQueueOnly); Tracking.Unmap(va, size); RemoveMapping(va, size); diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KPageTable.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KPageTable.cs index 9d5212315..0f6369967 100644 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KPageTable.cs +++ b/Ryujinx.HLE/HOS/Kernel/Memory/KPageTable.cs @@ -140,7 +140,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory } /// - protected override KernelResult Unmap(ulong address, ulong pagesCount) + protected override KernelResult Unmap(ulong address, ulong pagesCount, bool clearRejitQueueOnly = false) { KPageList pagesToClose = new KPageList(); @@ -155,7 +155,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory } } - _cpuMemory.Unmap(address, pagesCount * PageSize); + _cpuMemory.Unmap(address, pagesCount * PageSize, clearRejitQueueOnly); pagesToClose.DecrementPagesReferenceCount(Context.MemoryManager); diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KPageTableBase.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KPageTableBase.cs index 857be7a65..8c87b8614 100644 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KPageTableBase.cs +++ b/Ryujinx.HLE/HOS/Kernel/Memory/KPageTableBase.cs @@ -610,7 +610,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory } } - public KernelResult UnmapProcessCodeMemory(ulong dst, ulong src, ulong size) + public KernelResult UnmapProcessCodeMemory(ulong dst, ulong src, ulong size, bool clearRejitQueueOnly = false) { lock (_blockManager) { @@ -656,7 +656,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory { ulong pagesCount = size / PageSize; - KernelResult result = Unmap(dst, pagesCount); + KernelResult result = Unmap(dst, pagesCount, clearRejitQueueOnly); if (result != KernelResult.Success) { @@ -2957,7 +2957,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory /// Virtual address of the region to unmap /// Number of pages to unmap /// Result of the unmapping operation - protected abstract KernelResult Unmap(ulong address, ulong pagesCount); + protected abstract KernelResult Unmap(ulong address, ulong pagesCount, bool clearRejitQueueOnly = false); /// /// Changes the permissions of a given virtual memory region. diff --git a/Ryujinx.HLE/HOS/Services/Ro/IRoInterface.cs b/Ryujinx.HLE/HOS/Services/Ro/IRoInterface.cs index 5590bfddf..d42c35233 100644 --- a/Ryujinx.HLE/HOS/Services/Ro/IRoInterface.cs +++ b/Ryujinx.HLE/HOS/Services/Ro/IRoInterface.cs @@ -18,6 +18,8 @@ namespace Ryujinx.HLE.HOS.Services.Ro [Service("ro:1")] // 7.0.0+ class IRoInterface : DisposableIpcService { + private const bool EnableOpt = true; + private const int MaxNrr = 0x40; private const int MaxNro = 0x40; private const int MaxMapRetries = 0x200; @@ -28,18 +30,21 @@ namespace Ryujinx.HLE.HOS.Services.Ro private List _nrrInfos; private List _nroInfos; + private List _nroInfosUnloaded; private KProcess _owner; private IVirtualMemoryManager _ownerMm; - private static Random _random = new Random(); + private Random _random; public IRoInterface(ServiceCtx context) { _nrrInfos = new List(MaxNrr); _nroInfos = new List(MaxNro); - _owner = null; - _ownerMm = null; + _nroInfosUnloaded = new List(MaxNro); + _owner = null; + _ownerMm = null; + _random = new Random(); } private ResultCode ParseNrr(out NrrInfo nrrInfo, ServiceCtx context, ulong nrrAddress, ulong nrrSize) @@ -205,17 +210,18 @@ namespace Ryujinx.HLE.HOS.Services.Ro return ResultCode.Success; } - private ResultCode MapNro(KProcess process, NroInfo info, out ulong nroMappedAddress) + private ResultCode MapNro(KProcess process, NroInfo info, MapType mapType, out ulong nroMappedAddress) { KPageTableBase memMgr = process.MemoryManager; int retryCount = 0; + int maxMapRetries = mapType != MapType.Within ? MaxMapRetries : 1; nroMappedAddress = 0; - while (retryCount++ < MaxMapRetries) + while (retryCount++ < maxMapRetries) { - ResultCode result = MapCodeMemoryInProcess(process, info.NroAddress, info.NroSize, out nroMappedAddress); + ResultCode result = MapCodeMemoryInProcess(process, info, mapType, out nroMappedAddress); if (result != ResultCode.Success) { @@ -269,31 +275,73 @@ namespace Ryujinx.HLE.HOS.Services.Ro return false; } - private ResultCode MapCodeMemoryInProcess(KProcess process, ulong baseAddress, ulong size, out ulong targetAddress) + private static bool NotOverlapsWith(ulong newAddress, ulong newSize, ulong oldAddress, ulong oldSize) + { + return newAddress + newSize <= oldAddress || oldAddress + oldSize <= newAddress; + } + + private ResultCode MapCodeMemoryInProcess(KProcess process, NroInfo info, MapType mapType, out ulong targetAddress) { KPageTableBase memMgr = process.MemoryManager; + int retryCount = 0; + int maxMapRetries = mapType != MapType.Within ? MaxMapRetries : 1; + targetAddress = 0; - int retryCount; - - ulong addressSpacePageLimit = (memMgr.GetAddrSpaceSize() - size) >> 12; - - for (retryCount = 0; retryCount < MaxMapRetries; retryCount++) + while (retryCount++ < maxMapRetries) { - while (true) + if (mapType == MapType.Default) { - ulong randomOffset = (ulong)(uint)_random.Next(0, (int)addressSpacePageLimit) << 12; - - targetAddress = memMgr.GetAddrSpaceBaseAddr() + randomOffset; - - if (memMgr.InsideAddrSpace(targetAddress, size) && !memMgr.InsideHeapRegion(targetAddress, size) && !memMgr.InsideAliasRegion(targetAddress, size)) + while (true) { - break; + ulong addressSpacePageLimit = (memMgr.GetAddrSpaceSize() - info.NroSize) >> 12; + ulong randomOffset = (ulong)(uint)_random.Next(0, (int)addressSpacePageLimit) << 12; + + targetAddress = memMgr.GetAddrSpaceBaseAddr() + randomOffset; + + if (memMgr.InsideAddrSpace(targetAddress, info.NroSize) && + !memMgr.InsideHeapRegion(targetAddress, info.NroSize) && + !memMgr.InsideAliasRegion(targetAddress, info.NroSize)) + { + break; + } } } + else if (mapType == MapType.Between) + { + while (true) + { + while (true) + { + ulong addressSpacePageLimit = (memMgr.GetAddrSpaceSize() - info.NroSize) >> 12; + ulong randomOffset = (ulong)(uint)_random.Next(0, (int)addressSpacePageLimit) << 12; - KernelResult result = memMgr.MapProcessCodeMemory(targetAddress, baseAddress, size); + targetAddress = memMgr.GetAddrSpaceBaseAddr() + randomOffset; + + if (memMgr.InsideAddrSpace(targetAddress, info.NroSize) && + !memMgr.InsideHeapRegion(targetAddress, info.NroSize) && + !memMgr.InsideAliasRegion(targetAddress, info.NroSize)) + { + break; + } + } + + ulong targetAddressTmp = targetAddress; // CS1628. + if (_nroInfosUnloaded.TrueForAll((infoU) => NotOverlapsWith(targetAddressTmp, info.NroSize + info.BssSize, infoU.NroMappedAddress, infoU.NroSize + infoU.BssSize))) + { + break; + } + } + } + else /* if (mapType == MapType.Within) */ + { + NroInfo infoU = _nroInfosUnloaded.Find((infoU) => infoU.Hash.SequenceEqual(info.Hash)); + + targetAddress = infoU.NroMappedAddress; + } + + KernelResult result = memMgr.MapProcessCodeMemory(targetAddress, info.NroAddress, info.NroSize); if (result == KernelResult.InvalidMemState) { @@ -304,7 +352,7 @@ namespace Ryujinx.HLE.HOS.Services.Ro return (ResultCode)result; } - if (!CanAddGuardRegionsInProcess(process, targetAddress, size)) + if (!CanAddGuardRegionsInProcess(process, targetAddress, info.NroSize)) { continue; } @@ -312,12 +360,7 @@ namespace Ryujinx.HLE.HOS.Services.Ro return ResultCode.Success; } - if (retryCount == MaxMapRetries) - { - return ResultCode.InsufficientAddressSpace; - } - - return ResultCode.Success; + return ResultCode.InsufficientAddressSpace; } private KernelResult SetNroMemoryPermissions(KProcess process, IExecutable relocatableObject, ulong baseAddress) @@ -376,16 +419,20 @@ namespace Ryujinx.HLE.HOS.Services.Ro { if (info.NroMappedAddress == nroMappedAddress) { + if (EnableOpt) + { + _nroInfosUnloaded.Add(info); + } _nroInfos.Remove(info); - return UnmapNroFromInfo(info); + return UnmapNroFromInfo(info, clearRejitQueueOnly: EnableOpt); } } return ResultCode.NotLoaded; } - private ResultCode UnmapNroFromInfo(NroInfo info) + private ResultCode UnmapNroFromInfo(NroInfo info, bool clearRejitQueueOnly = false) { ulong textSize = (ulong)info.Executable.Text.Length; ulong roSize = (ulong)info.Executable.Ro.Length; @@ -399,7 +446,7 @@ namespace Ryujinx.HLE.HOS.Services.Ro result = _owner.MemoryManager.UnmapProcessCodeMemory( info.NroMappedAddress + textSize + roSize + dataSize, info.Executable.BssAddress, - bssSize); + bssSize, clearRejitQueueOnly); } if (result == KernelResult.Success) @@ -407,14 +454,14 @@ namespace Ryujinx.HLE.HOS.Services.Ro result = _owner.MemoryManager.UnmapProcessCodeMemory( info.NroMappedAddress + textSize + roSize, info.Executable.SourceAddress + textSize + roSize, - dataSize); + dataSize, clearRejitQueueOnly); if (result == KernelResult.Success) { result = _owner.MemoryManager.UnmapProcessCodeMemory( info.NroMappedAddress, info.Executable.SourceAddress, - textSize + roSize); + textSize + roSize, clearRejitQueueOnly); } } @@ -431,6 +478,8 @@ namespace Ryujinx.HLE.HOS.Services.Ro return ResultCode.InvalidProcess; } + private enum MapType { Default, Between, Within } + [CommandHipc(0)] // LoadNro(u64, u64, u64, u64, u64, pid) -> u64 public ResultCode LoadNro(ServiceCtx context) @@ -455,7 +504,48 @@ namespace Ryujinx.HLE.HOS.Services.Ro if (result == ResultCode.Success) { - result = MapNro(_owner, info, out nroMappedAddress); + MapType mapType = MapType.Default; + + if (_nroInfosUnloaded.Count != 0) + { + mapType = _nroInfosUnloaded.Exists((infoU) => infoU.Hash.SequenceEqual(info.Hash)) // One/zero match. + ? MapType.Within + : MapType.Between; + } + + result = MapNro(_owner, info, mapType, out nroMappedAddress); + + if (mapType == MapType.Between) + { + if (result != ResultCode.Success) + { + _nroInfosUnloaded.Clear(); + + mapType = MapType.Default; + + result = MapNro(_owner, info, mapType, out nroMappedAddress); + } + } + else if (mapType == MapType.Within) + { + _nroInfosUnloaded.RemoveAll((infoU) => infoU.Hash.SequenceEqual(info.Hash)); // One match. + + if (result != ResultCode.Success) + { + mapType = MapType.Between; + + result = MapNro(_owner, info, mapType, out nroMappedAddress); + + if (result != ResultCode.Success) + { + _nroInfosUnloaded.Clear(); + + mapType = MapType.Default; + + result = MapNro(_owner, info, mapType, out nroMappedAddress); + } + } + } if (result == ResultCode.Success) { @@ -598,6 +688,7 @@ namespace Ryujinx.HLE.HOS.Services.Ro } _nroInfos.Clear(); + _nroInfosUnloaded.Clear(); if (_ownerMm is IRefCounted rc) { diff --git a/Ryujinx.Memory/AddressSpaceManager.cs b/Ryujinx.Memory/AddressSpaceManager.cs index 45f3225e1..417a956b9 100644 --- a/Ryujinx.Memory/AddressSpaceManager.cs +++ b/Ryujinx.Memory/AddressSpaceManager.cs @@ -65,7 +65,7 @@ namespace Ryujinx.Memory } /// - public void Unmap(ulong va, ulong size) + public void Unmap(ulong va, ulong size, bool clearRejitQueueOnly = false) { AssertValidAddressAndSize(va, size); diff --git a/Ryujinx.Memory/IVirtualMemoryManager.cs b/Ryujinx.Memory/IVirtualMemoryManager.cs index f97cb0b57..ca61c6bcc 100644 --- a/Ryujinx.Memory/IVirtualMemoryManager.cs +++ b/Ryujinx.Memory/IVirtualMemoryManager.cs @@ -22,7 +22,7 @@ namespace Ryujinx.Memory /// /// Virtual address of the range to be unmapped /// Size of the range to be unmapped - void Unmap(ulong va, ulong size); + void Unmap(ulong va, ulong size, bool clearRejitQueueOnly = false); /// /// Reads data from CPU mapped memory. diff --git a/Ryujinx.Tests/Cpu/CpuContext.cs b/Ryujinx.Tests/Cpu/CpuContext.cs index 96b4965a2..c894b7535 100644 --- a/Ryujinx.Tests/Cpu/CpuContext.cs +++ b/Ryujinx.Tests/Cpu/CpuContext.cs @@ -16,9 +16,9 @@ namespace Ryujinx.Tests.Cpu memory.UnmapEvent += UnmapHandler; } - private void UnmapHandler(ulong address, ulong size) + private void UnmapHandler(ulong address, ulong size, bool clearRejitQueueOnly = false) { - _translator.InvalidateJitCacheRegion(address, size); + _translator.InvalidateJitCacheRegion(address, size, clearRejitQueueOnly); } public static ExecutionContext CreateExecutionContext() diff --git a/Ryujinx.Tests/Memory/MockMemoryManager.cs b/Ryujinx.Tests/Memory/MockMemoryManager.cs index 3f7692636..ffb5991a6 100644 --- a/Ryujinx.Tests/Memory/MockMemoryManager.cs +++ b/Ryujinx.Tests/Memory/MockMemoryManager.cs @@ -12,7 +12,7 @@ namespace Ryujinx.Tests.Memory public MemoryManagerType Type => MemoryManagerType.HostMappedUnsafe; #pragma warning disable CS0067 - public event Action UnmapEvent; + public event Action UnmapEvent; #pragma warning restore CS0067 public ref T GetRef(ulong va) where T : unmanaged