Device local mapping for all buffers

May avoid issues with drivers with NVIDIA on linux/older gpus on windows when using large buffers (?)
Also some performance things and fixes issues with opengl games loading textures weird.

Ryujinx.Graphics.Vulkan/Auto.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Diagnostics;
+using System.Threading;
 
 namespace Ryujinx.Graphics.Vulkan
 {
@@ -66,6 +67,16 @@ namespace Ryujinx.Graphics.Vulkan
             return _value;
         }
 
+        public bool HasCommandBufferDependency(CommandBufferScoped cbs)
+        {
+            return _cbOwnership.IsSet(cbs.CommandBufferIndex);
+        }
+
+        public bool HasRentedCommandBufferDependency(CommandBufferPool cbp)
+        {
+            return _cbOwnership.AnySet();
+        }
+
         public void AddCommandBufferDependencies(CommandBufferScoped cbs)
         {
             // We don't want to add a reference to this object to the command buffer
@@ -94,11 +105,11 @@ namespace Ryujinx.Graphics.Vulkan
 
         public void IncrementReferenceCount()
         {
-            if (_referenceCount == 0)
+            if (Interlocked.Increment(ref _referenceCount) == 1)
             {
+                Interlocked.Decrement(ref _referenceCount);
                 throw new Exception("Attempted to inc ref of dead object.");
             }
-            _referenceCount++;
         }
 
         public void DecrementReferenceCount(int cbIndex)
@@ -109,7 +120,7 @@ namespace Ryujinx.Graphics.Vulkan
 
         public void DecrementReferenceCount()
         {
-            if (--_referenceCount == 0)
+            if (Interlocked.Decrement(ref _referenceCount) == 0)
             {
                 _value.Dispose();
                 _value = default;

Ryujinx.Graphics.Vulkan/BackgroundResources.cs

@@ -0,0 +1,107 @@
+using System.Threading;
+using System.Collections.Generic;
+using System;
+using Silk.NET.Vulkan;
+
+namespace Ryujinx.Graphics.Vulkan
+{
+    class BackgroundResource : IDisposable
+    {
+        private VulkanGraphicsDevice _gd;
+        private Device _device;
+
+        private CommandBufferPool _pool;
+        private PersistentFlushBuffer _flushBuffer;
+
+        public BackgroundResource(VulkanGraphicsDevice gd, Device device)
+        {
+            _gd = gd;
+            _device = device;
+        }
+
+        public CommandBufferPool GetPool()
+        {
+            if (_pool == null)
+            {
+                _pool = new CommandBufferPool(_gd.Api, _device, _gd.BackgroundQueue, _gd.QueueFamilyIndex, isLight: true);
+            }
+
+            return _pool;
+        }
+
+        public PersistentFlushBuffer GetFlushBuffer()
+        {
+            if (_flushBuffer == null)
+            {
+                _flushBuffer = new PersistentFlushBuffer(_gd);
+            }
+
+            return _flushBuffer;
+        }
+
+        public void Dispose()
+        {
+            _pool?.Dispose();
+            _flushBuffer?.Dispose();
+        }
+    }
+
+    class BackgroundResources : IDisposable
+    {
+        private VulkanGraphicsDevice _gd;
+        private Device _device;
+
+        private Dictionary<Thread, BackgroundResource> _resources;
+
+        public BackgroundResources(VulkanGraphicsDevice gd, Device device)
+        {
+            _gd = gd;
+            _device = device;
+
+            _resources = new Dictionary<Thread, BackgroundResource>();
+        }
+
+        private void Cleanup()
+        {
+            foreach (KeyValuePair<Thread, BackgroundResource> tuple in _resources)
+            {
+                if (!tuple.Key.IsAlive)
+                {
+                    tuple.Value.Dispose();
+                    _resources.Remove(tuple.Key);
+                }
+            }
+        }
+
+        public BackgroundResource Get()
+        {
+            Thread thread = Thread.CurrentThread;
+
+            lock (_resources)
+            {
+                BackgroundResource resource;
+                if (!_resources.TryGetValue(thread, out resource))
+                {
+                    Cleanup();
+
+                    resource = new BackgroundResource(_gd, _device);
+
+                    _resources[thread] = resource;
+                }
+
+                return resource;
+            }
+        }
+
+        public void Dispose()
+        {
+            lock (_resources)
+            {
+                foreach (var resource in _resources.Values)
+                {
+                    resource.Dispose();
+                }
+            }
+        }
+    }
+}

Ryujinx.Graphics.Vulkan/BitMap.cs

@@ -12,6 +12,29 @@
             _masks = new long[(count + IntMask) / IntSize];
         }
 
+        public bool AnySet()
+        {
+            for (int i = 0; i < _masks.Length; i++)
+            {
+                if (_masks[i] != 0)
+                {
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
+        public bool IsSet(int bit)
+        {
+            int wordIndex = bit / IntSize;
+            int wordBit = bit & IntMask;
+
+            long wordMask = 1L << wordBit;
+
+            return (_masks[wordIndex] & wordMask) != 0;
+        }
+
         public bool Set(int bit)
         {
             int wordIndex = bit / IntSize;

Ryujinx.Graphics.Vulkan/BufferHolder.cs

@@ -129,7 +129,32 @@ namespace Ryujinx.Graphics.Vulkan
 
         public unsafe ReadOnlySpan<byte> GetData(int offset, int size)
         {
-            return GetDataStorage(offset, size);
+            if (_map != IntPtr.Zero)
+            {
+                return GetDataStorage(offset, size);
+            }
+            else
+            {
+                BackgroundResource resource = _gd.BackgroundResources.Get();
+
+                if (_gd.CommandBufferPool.OwnedByCurrentThread)
+                {
+                    _gd.FlushAllCommands();
+
+                    return resource.GetFlushBuffer().GetBufferData(_gd.CommandBufferPool, this, offset, size);
+                }
+                else if (_gd.BackgroundQueue.Handle != 0)
+                {
+                    lock (_gd.BackgroundQueueLock)
+                    {
+                        return resource.GetFlushBuffer().GetBufferData(resource.GetPool(), this, offset, size);
+                    }
+                }
+                else
+                {
+                    return new byte[size];
+                }
+            }
         }
 
         public unsafe Span<byte> GetDataStorage(int offset, int size)
@@ -152,28 +177,37 @@ namespace Ryujinx.Graphics.Vulkan
                 return;
             }
 
-            bool needsFlush = _gd.CommandBufferPool.HasWaitableOnRentedCommandBuffer(_waitable, offset, dataSize);
+            if (_map != IntPtr.Zero)
-            bool needsWait = needsFlush || MayWait(offset, dataSize);
+            {
+                // If persistently mapped, set the data directly if the buffer is not currently in use.
+                //bool needsFlush = _gd.CommandBufferPool.HasWaitableOnRentedCommandBuffer(_waitable, offset, dataSize);
+                bool needsFlush = _buffer.HasRentedCommandBufferDependency(_gd.CommandBufferPool);// (_waitable, offset, dataSize);
+
+                if (!needsFlush)
+                {
+                    WaitForFences(offset, dataSize);
+
+                    data.Slice(0, dataSize).CopyTo(new Span<byte>((void*)(_map + offset), dataSize));
+
+                    return;
+                }
+            }
+
+            if (cbs != null && !_buffer.HasCommandBufferDependency(cbs.Value))
+            {
+                // If the buffer hasn't been used on the command buffer yet, try to preload the data.
+                // This avoids ending and beginning render passes on each buffer data upload.
+
+                cbs = _gd.PipelineInternal.GetPreloadCommandBuffer();
+                endRenderPass = null;
+            }
 
             if (cbs == null ||
-                !needsWait ||
                 !VulkanConfiguration.UseFastBufferUpdates ||
+                data.Length > MaxUpdateBufferSize ||
                 !TryPushData(cbs.Value, endRenderPass, offset, data))
             {
-                // Some pending command might access the buffer,
+                _gd.BufferManager.StagingBuffer.PushData(_gd.CommandBufferPool, cbs, endRenderPass, this, offset, data);
-                // so we need to ensure they are all submited to the GPU before waiting.
-                // The flush below forces the submission of all pending commands.
-                if (needsFlush)
-                {
-                    _gd.FlushAllCommands();
-                }
-
-                WaitForFences(offset, dataSize);
-
-                if (_map != IntPtr.Zero)
-                {
-                    data.Slice(0, dataSize).CopyTo(new Span<byte>((void*)(_map + offset), dataSize));
-                }
             }
         }
 
@@ -189,6 +223,10 @@ namespace Ryujinx.Graphics.Vulkan
             {
                 data.Slice(0, dataSize).CopyTo(new Span<byte>((void*)(_map + offset), dataSize));
             }
+            else
+            {
+                _gd.BufferManager.StagingBuffer.PushData(_gd.CommandBufferPool, null, null, this, offset, data);
+            }
         }
 
         public void SetDataInline(CommandBufferScoped cbs, Action endRenderPass, int dstOffset, ReadOnlySpan<byte> data)
@@ -206,7 +244,7 @@ namespace Ryujinx.Graphics.Vulkan
                 return false;
             }
 
-            endRenderPass();
+            endRenderPass?.Invoke();
 
             var dstBuffer = GetBuffer(cbs.CommandBuffer, true).Get(cbs, dstOffset, data.Length).Value;
 

Ryujinx.Graphics.Vulkan/BufferManager.cs

@@ -15,6 +15,14 @@ namespace Ryujinx.Graphics.Vulkan
             MemoryPropertyFlags.MemoryPropertyHostCoherentBit |
             MemoryPropertyFlags.MemoryPropertyHostCachedBit;
 
+        private const MemoryPropertyFlags DeviceLocalBufferMemoryFlags =
+            MemoryPropertyFlags.MemoryPropertyDeviceLocalBit;
+
+        private const MemoryPropertyFlags FlushableDeviceLocalBufferMemoryFlags =
+            MemoryPropertyFlags.MemoryPropertyHostVisibleBit |
+            MemoryPropertyFlags.MemoryPropertyHostCoherentBit |
+            MemoryPropertyFlags.MemoryPropertyDeviceLocalBit;
+
         private const BufferUsageFlags DefaultBufferUsageFlags =
             BufferUsageFlags.BufferUsageTransferSrcBit |
             BufferUsageFlags.BufferUsageTransferDstBit |
@@ -41,9 +49,9 @@ namespace Ryujinx.Graphics.Vulkan
             StagingBuffer = new StagingBuffer(gd, this);
         }
 
-        public BufferHandle CreateWithHandle(VulkanGraphicsDevice gd, int size)
+        public BufferHandle CreateWithHandle(VulkanGraphicsDevice gd, int size, bool deviceLocal)
         {
-            var holder = Create(gd, size);
+            var holder = Create(gd, size, deviceLocal: deviceLocal);
             if (holder == null)
             {
                 return BufferHandle.Null;
@@ -58,7 +66,7 @@ namespace Ryujinx.Graphics.Vulkan
             return handle;
         }
 
-        public unsafe BufferHolder Create(VulkanGraphicsDevice gd, int size, bool forConditionalRendering = false)
+        public unsafe BufferHolder Create(VulkanGraphicsDevice gd, int size, bool forConditionalRendering = false, bool deviceLocal = false)
         {
             var usage = DefaultBufferUsageFlags;
 
@@ -81,7 +89,10 @@ namespace Ryujinx.Graphics.Vulkan
 
             gd.Api.CreateBuffer(_device, in bufferCreateInfo, null, out var buffer).ThrowOnError();
             gd.Api.GetBufferMemoryRequirements(_device, buffer, out var requirements);
-            var allocation = gd.MemoryAllocator.AllocateDeviceMemory(_physicalDevice, requirements, DefaultBufferMemoryFlags);
+
+            var allocateFlags = deviceLocal ? DeviceLocalBufferMemoryFlags : DefaultBufferMemoryFlags;
+
+            var allocation = gd.MemoryAllocator.AllocateDeviceMemory(_physicalDevice, requirements, allocateFlags);
 
             if (allocation.Memory.Handle == 0UL)
             {

Ryujinx.Graphics.Vulkan/CommandBufferPool.cs

@@ -8,7 +8,7 @@ namespace Ryujinx.Graphics.Vulkan
 {
     class CommandBufferPool : IDisposable
     {
-        public const int MaxCommandBuffers = 8;
+        public const int MaxCommandBuffers = 16;
 
         private int _totalCommandBuffers;
         private int _totalCommandBuffersMask;
@@ -19,12 +19,15 @@ namespace Ryujinx.Graphics.Vulkan
         private readonly CommandPool _pool;
         private readonly Thread _owner;
 
+        public int PerFrame = 0;
+
         public bool OwnedByCurrentThread => _owner == Thread.CurrentThread;
 
         private struct ReservedCommandBuffer
         {
             public bool InUse;
             public bool InConsumption;
+            public bool Active;
             public CommandBuffer CommandBuffer;
             public FenceHolder Fence;
             public SemaphoreHolder Semaphore;
@@ -48,6 +51,8 @@ namespace Ryujinx.Graphics.Vulkan
                 Dependants = new List<IAuto>();
                 Waitables = new HashSet<MultiFenceHolder>();
                 Dependencies = new HashSet<SemaphoreHolder>();
+
+                Active = true; // Fence should be refreshed.
             }
         }
 
@@ -120,6 +125,12 @@ namespace Ryujinx.Graphics.Vulkan
             entry.Waitables.Add(waitable);
         }
 
+        public bool HasWaitable(int cbIndex, MultiFenceHolder waitable)
+        {
+            ref var entry = ref _commandBuffers[cbIndex];
+            return entry.Waitables.Contains(waitable);
+        }
+
         public bool HasWaitableOnRentedCommandBuffer(MultiFenceHolder waitable, int offset, int size)
         {
             lock (_commandBuffers)
@@ -140,11 +151,47 @@ namespace Ryujinx.Graphics.Vulkan
             return false;
         }
 
+        public bool IsFenceOnRentedCommandBuffer(FenceHolder fence)
+        {
+            lock (_commandBuffers)
+            {
+                for (int i = 0; i < _totalCommandBuffers; i++)
+                {
+                    ref var entry = ref _commandBuffers[i];
+
+                    if (entry.InUse && entry.Fence == fence)
+                    {
+                        return true;
+                    }
+                }
+            }
+
+            return false;
+        }
+
         public FenceHolder GetFence(int cbIndex)
         {
             return _commandBuffers[cbIndex].Fence;
         }
 
+        private void CheckConsumption(int startAt)
+        {
+            int wrapBefore = _totalCommandBuffers + 1;
+            int index = (startAt + wrapBefore) % _totalCommandBuffers;
+
+            for (int i = 0; i < _totalCommandBuffers - 1; i++)
+            {
+                ref var entry = ref _commandBuffers[index];
+
+                if (!entry.InUse && entry.InConsumption && entry.Fence.IsSignaled())
+                {
+                    WaitAndDecrementRef(index);
+                }
+
+                index = (index + wrapBefore) % _totalCommandBuffers;
+            }
+        }
+
         public CommandBufferScoped ReturnAndRent(CommandBufferScoped cbs)
         {
             Return(cbs);
@@ -153,8 +200,11 @@ namespace Ryujinx.Graphics.Vulkan
 
         public CommandBufferScoped Rent()
         {
+            PerFrame++;
             lock (_commandBuffers)
             {
+                CheckConsumption(_cursor);
+
                 for (int i = 0; i < _totalCommandBuffers; i++)
                 {
                     int currentIndex = _cursor;
@@ -165,6 +215,7 @@ namespace Ryujinx.Graphics.Vulkan
                     {
                         WaitAndDecrementRef(currentIndex);
                         entry.InUse = true;
+                        entry.Active = true;
 
                         var commandBufferBeginInfo = new CommandBufferBeginInfo()
                         {
@@ -241,6 +292,13 @@ namespace Ryujinx.Graphics.Vulkan
         {
             ref var entry = ref _commandBuffers[cbIndex];
 
+            if (!entry.Active)
+            {
+                return;
+            }
+
+            entry.Active = false;
+
             if (entry.InConsumption)
             {
                 entry.Fence.Wait();

Ryujinx.Graphics.Vulkan/CommandBufferScoped.cs

@@ -31,6 +31,11 @@ namespace Ryujinx.Graphics.Vulkan
             _pool.AddDependency(CommandBufferIndex, dependencyCbs);
         }
 
+        public bool HasWaitable(MultiFenceHolder waitable)
+        {
+            return _pool.HasWaitable(CommandBufferIndex, waitable);
+        }
+
         public FenceHolder GetFence()
         {
             return _pool.GetFence(CommandBufferIndex);

Ryujinx.Graphics.Vulkan/HelperShader.cs

@@ -169,7 +169,7 @@ void main()
                 region[3] = temp;
             }
 
-            var bufferHandle = gd.BufferManager.CreateWithHandle(gd, RegionBufferSize);
+            var bufferHandle = gd.BufferManager.CreateWithHandle(gd, RegionBufferSize, false);
 
             gd.BufferManager.SetData<float>(bufferHandle, 0, region);
 
@@ -251,7 +251,7 @@ void main()
                 region[3] = temp;
             }
 
-            var bufferHandle = gd.BufferManager.CreateWithHandle(gd, RegionBufferSize);
+            var bufferHandle = gd.BufferManager.CreateWithHandle(gd, RegionBufferSize, false);
 
             gd.BufferManager.SetData<float>(bufferHandle, 0, region);
 

Ryujinx.Graphics.Vulkan/PersistentFlushBuffer.cs

@@ -0,0 +1,73 @@
+using System;
+
+namespace Ryujinx.Graphics.Vulkan
+{
+    internal class PersistentFlushBuffer : IDisposable
+    {
+        private VulkanGraphicsDevice _gd;
+
+        private BufferHolder _flushStorage;
+
+        public PersistentFlushBuffer(VulkanGraphicsDevice gd)
+        {
+            _gd = gd;
+        }
+
+        private BufferHolder ResizeIfNeeded(int size)
+        {
+            var flushStorage = _flushStorage;
+
+            if (flushStorage == null || size > _flushStorage.Size)
+            {
+                if (flushStorage != null)
+                {
+                    flushStorage.Dispose();
+                }
+
+                flushStorage = _gd.BufferManager.Create(_gd, size);
+                _flushStorage = flushStorage;
+            }
+
+            return flushStorage;
+        }
+
+        public Span<byte> GetBufferData(CommandBufferPool cbp, BufferHolder buffer, int offset, int size)
+        {
+            var flushStorage = ResizeIfNeeded(size);
+
+            using (var cbs = cbp.Rent())
+            {
+                var srcBuffer = buffer.GetBuffer(cbs.CommandBuffer);
+                var dstBuffer = flushStorage.GetBuffer(cbs.CommandBuffer);
+
+                BufferHolder.Copy(_gd, cbs, srcBuffer, dstBuffer, offset, 0, size);
+            }
+
+            flushStorage.WaitForFences();
+            return flushStorage.GetDataStorage(0, size);
+        }
+
+        public Span<byte> GetTextureData(CommandBufferPool cbp, TextureView view, int size)
+        {
+            GAL.TextureCreateInfo info = view.Info;
+
+            var flushStorage = ResizeIfNeeded(size);
+
+            using (var cbs = cbp.Rent())
+            {
+                var buffer = flushStorage.GetBuffer(cbs.CommandBuffer).Get(cbs).Value;
+                var image = view.GetImage().Get(cbs).Value;
+
+                view.CopyFromOrToBuffer(cbs.CommandBuffer, buffer, image, size, true, 0, 0, info.GetLayers(), info.Levels, singleSlice: false);
+            }
+
+            flushStorage.WaitForFences();
+            return flushStorage.GetDataStorage(0, size);
+        }
+
+        public void Dispose()
+        {
+            _flushStorage.Dispose();
+        }
+    }
+}

Ryujinx.Graphics.Vulkan/PipelineBase.cs

@@ -34,6 +34,7 @@ namespace Ryujinx.Graphics.Vulkan
         private PipelineCache _pipelineCache;
 
         protected CommandBufferScoped Cbs;
+        protected CommandBufferScoped? PreloadCbs;
         protected CommandBuffer CommandBuffer;
 
         public CommandBufferScoped CurrentCommandBuffer => Cbs;

Ryujinx.Graphics.Vulkan/PipelineFull.cs

@@ -225,6 +225,16 @@ namespace Ryujinx.Graphics.Vulkan
             }
         }
 
+        public CommandBufferScoped GetPreloadCommandBuffer()
+        {
+            if (PreloadCbs == null)
+            {
+                PreloadCbs = Gd.CommandBufferPool.Rent();
+            }
+
+            return PreloadCbs.Value;
+        }
+
         public void FlushCommandsImpl([System.Runtime.CompilerServices.CallerMemberName] string caller = "")
         {
             // System.Console.WriteLine("flush by " + caller);
@@ -237,6 +247,12 @@ namespace Ryujinx.Graphics.Vulkan
                 Gd.Api.CmdEndQuery(CommandBuffer, queryPool, 0);
             }
 
+            if (PreloadCbs != null)
+            {
+                PreloadCbs.Value.Dispose();
+                PreloadCbs = null;
+            }
+
             CommandBuffer = (Cbs = Gd.CommandBufferPool.ReturnAndRent(Cbs)).CommandBuffer;
 
             // Restore per-command buffer state.

Ryujinx.Graphics.Vulkan/StagingBuffer.cs

@@ -37,25 +37,55 @@ namespace Ryujinx.Graphics.Vulkan
             _freeSize = BufferSize;
         }
 
-        public unsafe bool TryPushData(CommandBufferScoped cbs, Action endRenderPass, BufferHolder dst, int dstOffset, ReadOnlySpan<byte> data)
+        public unsafe void PushData(CommandBufferPool cbp, CommandBufferScoped? cbs, Action endRenderPass, BufferHolder dst, int dstOffset, ReadOnlySpan<byte> data)
         {
-            if (data.Length > BufferSize)
+            bool isRender = cbs != null;
-            {
+            CommandBufferScoped scoped = cbs ?? cbp.Rent();
-                return false;
-            }
 
-            if (_freeSize < data.Length)
+            // Must push all data to the buffer. If it can't fit, split it up.
-            {
-                FreeCompleted();
 
+            endRenderPass?.Invoke();
+
+            while (data.Length > 0)
+            {
                 if (_freeSize < data.Length)
                 {
-                    return false;
+                    FreeCompleted();
                 }
+
+                while (_freeSize == 0)
+                {
+                    if (!WaitFreeCompleted(cbp))
+                    {
+                        if (isRender)
+                        {
+                            _gd.FlushAllCommands();
+                            scoped = cbp.Rent();
+                            isRender = false;
+                        }
+                        else
+                        {
+                            scoped = cbp.ReturnAndRent(scoped);
+                        }
+                    }
+                }
+
+                int chunkSize = Math.Min(_freeSize, data.Length);
+
+                PushDataImpl(scoped, dst, dstOffset, data.Slice(0, chunkSize));
+
+                dstOffset += chunkSize;
+                data = data.Slice(chunkSize);
             }
 
-            endRenderPass();
+            if (!isRender)
+            {
+                scoped.Dispose();
+            }
+        }
 
+        private void PushDataImpl(CommandBufferScoped cbs, BufferHolder dst, int dstOffset, ReadOnlySpan<byte> data)
+        {
             var srcBuffer = _buffer.GetBuffer();
             var dstBuffer = dst.GetBuffer();
 
@@ -81,14 +111,61 @@ namespace Ryujinx.Graphics.Vulkan
             Debug.Assert(_freeSize >= 0);
 
             _pendingCopies.Enqueue(new PendingCopy(cbs.GetFence(), data.Length));
+        }
+
+        public unsafe bool TryPushData(CommandBufferScoped cbs, Action endRenderPass, BufferHolder dst, int dstOffset, ReadOnlySpan<byte> data)
+        {
+            if (data.Length > BufferSize)
+            {
+                return false;
+            }
+
+            if (_freeSize < data.Length)
+            {
+                FreeCompleted();
+
+                if (_freeSize < data.Length)
+                {
+                    return false;
+                }
+            }
+
+            endRenderPass();
+
+            PushDataImpl(cbs, dst, dstOffset, data);
+
+            return true;
+        }
+
+        private bool WaitFreeCompleted(CommandBufferPool cbp)
+        {
+            if (_pendingCopies.TryPeek(out var pc))
+            {
+                if (!pc.Fence.IsSignaled())
+                {
+                    if (cbp.IsFenceOnRentedCommandBuffer(pc.Fence))
+                    {
+                        return false;
+                    }
+
+                    pc.Fence.Wait();
+                }
+
+                var dequeued = _pendingCopies.Dequeue();
+                Debug.Assert(dequeued.Fence == pc.Fence);
+                _freeSize += pc.Size;
+                pc.Fence.Put();
+            }
 
             return true;
         }
 
         private void FreeCompleted()
         {
-            while (_pendingCopies.TryPeek(out var pc) && pc.Fence.IsSignaled())
+            FenceHolder signalledFence = null;
+            while (_pendingCopies.TryPeek(out var pc) && (pc.Fence == signalledFence || pc.Fence.IsSignaled()))
             {
+                signalledFence = pc.Fence; // Already checked - don't need to do it again.
                 var dequeued = _pendingCopies.Dequeue();
                 Debug.Assert(dequeued.Fence == pc.Fence);
                 _freeSize += pc.Size;

Ryujinx.Graphics.Vulkan/TextureStorage.cs

@@ -228,9 +228,41 @@ namespace Ryujinx.Graphics.Vulkan
             return _image;
         }
 
+        public bool HasCommandBufferDependency(CommandBufferScoped cbs)
+        {
+            if (_foreignAllocationAuto != null)
+            {
+                return _foreignAllocationAuto.HasCommandBufferDependency(cbs);
+            }
+            else if (_allocationAuto != null)
+            {
+                return _allocationAuto.HasCommandBufferDependency(cbs);
+            }
+
+            return false;
+        }
+
         private unsafe void InitialTransition(ImageLayout srcLayout, ImageLayout dstLayout)
         {
-            using var cbs = _gd.CommandBufferPool.Rent();
+            CommandBufferScoped cbs;
+            bool useTempCbs = !_gd.CommandBufferPool.OwnedByCurrentThread;
+
+            if (useTempCbs)
+            {
+                cbs = _gd.BackgroundResources.Get().GetPool().Rent();
+            }
+            else
+            {
+                if (_gd.PipelineInternal != null)
+                {
+                    cbs = _gd.PipelineInternal.GetPreloadCommandBuffer();
+                }
+                else
+                {
+                    cbs = _gd.CommandBufferPool.Rent();
+                    useTempCbs = true;
+                }
+            }
 
             var aspectFlags = _info.Format.ConvertAspectFlags();
 
@@ -260,6 +292,11 @@ namespace Ryujinx.Graphics.Vulkan
                 null,
                 1,
                 barrier);
+
+            if (useTempCbs)
+            {
+                cbs.Dispose();
+            }
         }
 
         private static SampleCountFlags ConvertToSampleCountFlags(uint samples)

Ryujinx.Graphics.Vulkan/TextureView.cs

@@ -237,12 +237,7 @@ namespace Ryujinx.Graphics.Vulkan
             {
                 lock (_gd.BackgroundQueueLock)
                 {
-                    using var cbp = new CommandBufferPool(
+                    var cbp = _gd.BackgroundResources.Get().GetPool();
-                        _gd.Api,
-                        _device,
-                        _gd.BackgroundQueue,
-                        _gd.QueueFamilyIndex,
-                        isLight: true);
 
                     using var cbs = cbp.Rent();
 
@@ -426,8 +421,8 @@ namespace Ryujinx.Graphics.Vulkan
                     dstSize += dstTemp.Info.GetMipSize2D(l);
                 }
 
-                using var srcTempBuffer = gd.BufferManager.Create(gd, srcSize);
+                using var srcTempBuffer = gd.BufferManager.Create(gd, srcSize, deviceLocal: true);
-                using var dstTempBuffer = gd.BufferManager.Create(gd, dstSize);
+                using var dstTempBuffer = gd.BufferManager.Create(gd, dstSize, deviceLocal: true);
 
                 src.Storage.CopyFromOrToBuffer(
                     cbs.CommandBuffer,
@@ -669,30 +664,25 @@ namespace Ryujinx.Graphics.Vulkan
 
             bufferHolder.WaitForFences();
             byte[] bitmap = new byte[size];
-            GetDataFromBuffer(bufferHolder.GetDataStorage(0, size)).CopyTo(bitmap);
+            GetDataFromBuffer(bufferHolder.GetDataStorage(0, size), size, Span<byte>.Empty).CopyTo(bitmap);
             return bitmap;
         }
 
         public ReadOnlySpan<byte> GetData()
         {
+            BackgroundResource resources = _gd.BackgroundResources.Get();
+
             if (_gd.CommandBufferPool.OwnedByCurrentThread)
             {
                 _gd.FlushAllCommands();
 
-                return GetData(_gd.CommandBufferPool);
+                return GetData(_gd.CommandBufferPool, resources.GetFlushBuffer());
             }
             else if (_gd.BackgroundQueue.Handle != 0)
             {
                 lock (_gd.BackgroundQueueLock)
                 {
-                    using var cbp = new CommandBufferPool(
+                    return GetData(resources.GetPool(), resources.GetFlushBuffer());
-                        _gd.Api,
-                        _device,
-                        _gd.BackgroundQueue,
-                        _gd.QueueFamilyIndex,
-                        isLight: true);
-
-                    return GetData(cbp);
                 }
             }
             else
@@ -714,46 +704,19 @@ namespace Ryujinx.Graphics.Vulkan
             throw new NotImplementedException();
         }
 
-        private ReadOnlySpan<byte> GetData(CommandBufferPool cbp)
+        private ReadOnlySpan<byte> GetData(CommandBufferPool cbp, PersistentFlushBuffer flushBuffer)
         {
-            int size;
+            int size = 0;
-            var bufferHolder = _flushStorage;
 
-            if (bufferHolder == null)
+            for (int level = 0; level < Info.Levels; level++)
             {
-                size = 0;
+                size += Info.GetMipSize(level);
-
-                for (int level = 0; level < Info.Levels; level++)
-                {
-                    size += Info.GetMipSize(level);
-                }
-
-                size = GetBufferDataLength(size);
-
-                bufferHolder = _gd.BufferManager.Create(_gd, size);
-
-                var existingStorage = Interlocked.CompareExchange(ref _flushStorage, bufferHolder, null);
-                if (existingStorage != null)
-                {
-                    bufferHolder.Dispose();
-                    bufferHolder = existingStorage;
-                }
-            }
-            else
-            {
-                size = bufferHolder.Size;
             }
 
-            using (var cbs = cbp.Rent())
+            size = GetBufferDataLength(size);
-            {
-                var buffer = bufferHolder.GetBuffer(cbs.CommandBuffer).Get(cbs).Value;
-                var image = GetImage().Get(cbs).Value;
 
-                CopyFromOrToBuffer(cbs.CommandBuffer, buffer, image, size, true, 0, 0, Info.GetLayers(), Info.Levels, singleSlice: false);
+            Span<byte> result = flushBuffer.GetTextureData(cbp, this, size);
-            }
+            return GetDataFromBuffer(result, size, result);
-
-            bufferHolder.WaitForFences();
-            return GetDataFromBuffer(bufferHolder.GetDataStorage(0, size));
         }
 
         public void SetData(ReadOnlySpan<byte> data)
@@ -772,12 +735,24 @@ namespace Ryujinx.Graphics.Vulkan
 
             using var bufferHolder = _gd.BufferManager.Create(_gd, bufferDataLength);
 
-            using var cbs = _gd.CommandBufferPool.Rent();
+            Auto<DisposableImage> imageAuto = GetImage();
+
+            // Load texture data inline if the texture has been used on the current command buffer.
+
+            bool loadInline = Storage.HasCommandBufferDependency(_gd.PipelineInternal.CurrentCommandBuffer);
+
+            var cbs = loadInline ? _gd.PipelineInternal.CurrentCommandBuffer : _gd.PipelineInternal.GetPreloadCommandBuffer();
+
+            if (loadInline)
+            {
+                _gd.PipelineInternal.EndRenderPass();
+                Common.Logging.Logger.Error?.PrintMsg(Common.Logging.LogClass.Gpu, "Loaded inline!");
+            }
 
             CopyDataToBuffer(bufferHolder.GetDataStorage(0, bufferDataLength), data);
 
             var buffer = bufferHolder.GetBuffer(cbs.CommandBuffer).Get(cbs).Value;
-            var image = GetImage().Get(cbs).Value;
+            var image = imageAuto.Get(cbs).Value;
 
             CopyFromOrToBuffer(cbs.CommandBuffer, buffer, image, bufferDataLength, false, layer, level, layers, levels, singleSlice);
         }
@@ -813,11 +788,15 @@ namespace Ryujinx.Graphics.Vulkan
             input.CopyTo(storage);
         }
 
-        private ReadOnlySpan<byte> GetDataFromBuffer(ReadOnlySpan<byte> storage)
+        private ReadOnlySpan<byte> GetDataFromBuffer(ReadOnlySpan<byte> storage, int size, Span<byte> output)
         {
             if (NeedsD24S8Conversion())
             {
-                byte[] output = new byte[GetBufferDataLength(storage.Length)];
+                if (output.IsEmpty)
+                {
+                    output = new byte[GetBufferDataLength(size)];
+                }
+
                 FormatConverter.ConvertD32FS8ToD24S8(output, storage);
                 return output;
             }
@@ -830,7 +809,7 @@ namespace Ryujinx.Graphics.Vulkan
             return Info.Format == GAL.Format.D24UnormS8Uint && VkFormat == VkFormat.D32SfloatS8Uint;
         }
 
-        private void CopyFromOrToBuffer(
+        public void CopyFromOrToBuffer(
             CommandBuffer commandBuffer,
             VkBuffer buffer,
             Image image,

Ryujinx.Graphics.Vulkan/VulkanGraphicsDevice.cs

@@ -49,6 +49,7 @@ namespace Ryujinx.Graphics.Vulkan
         internal CommandBufferPool CommandBufferPool { get; private set; }
         internal DescriptorSetManager DescriptorSetManager { get; private set; }
         internal PipelineLayoutCache PipelineLayoutCache { get; private set; }
+        internal BackgroundResources BackgroundResources { get; private set; }
 
         internal BufferManager BufferManager { get; private set; }
 
@@ -170,6 +171,8 @@ namespace Ryujinx.Graphics.Vulkan
 
             PipelineLayoutCache = new PipelineLayoutCache();
 
+            BackgroundResources = new BackgroundResources(this, _device);
+
             BufferManager = new BufferManager(this, _physicalDevice, _device);
 
             _syncManager = new SyncManager(this, _device);
@@ -194,7 +197,7 @@ namespace Ryujinx.Graphics.Vulkan
 
         public BufferHandle CreateBuffer(int size)
         {
-            return BufferManager.CreateWithHandle(this, size);
+            return BufferManager.CreateWithHandle(this, size, true);
         }
 
         public IProgram CreateProgram(IShader[] shaders, ShaderInfo info)
@@ -386,6 +389,7 @@ namespace Ryujinx.Graphics.Vulkan
         public unsafe void Dispose()
         {
             CommandBufferPool.Dispose();
+            BackgroundResources.Dispose();
             _counters.Dispose();
             _window.Dispose();
             HelperShader.Dispose();