diff --git a/src/Ryujinx.Graphics.Gpu/Image/Texture.cs b/src/Ryujinx.Graphics.Gpu/Image/Texture.cs
index dca6263aa..326272e7c 100644
--- a/src/Ryujinx.Graphics.Gpu/Image/Texture.cs
+++ b/src/Ryujinx.Graphics.Gpu/Image/Texture.cs
@@ -102,9 +102,9 @@ namespace Ryujinx.Graphics.Gpu.Image
         public bool AlwaysFlushOnOverlap { get; private set; }
 
         /// <summary>
-        /// Indicates that the texture was fully unmapped since the modified flag was set, and flushes should be ignored until it is modified again.
+        /// Indicates that the texture was modified since the last time it was flushed.
         /// </summary>
-        public bool FlushStale { get; private set; }
+        public bool ModifiedSinceLastFlush { get; set; }
 
         /// <summary>
         /// Increments when the host texture is swapped, or when the texture is removed from all pools.
@@ -1417,7 +1417,6 @@ namespace Ryujinx.Graphics.Gpu.Image
         /// </summary>
         public void SignalModified()
         {
-            FlushStale = false;
             _scaledSetScore = Math.Max(0, _scaledSetScore - 1);
 
             if (_modifiedStale || Group.HasCopyDependencies)
@@ -1438,14 +1437,17 @@ namespace Ryujinx.Graphics.Gpu.Image
         {
             if (bound)
             {
-                FlushStale = false;
                 _scaledSetScore = Math.Max(0, _scaledSetScore - 1);
             }
 
             if (_modifiedStale || Group.HasCopyDependencies || Group.HasFlushBuffer)
             {
                 _modifiedStale = false;
-                Group.SignalModifying(this, bound);
+
+                if (bound || ModifiedSinceLastFlush || Group.HasCopyDependencies || Group.HasFlushBuffer)
+                {
+                    Group.SignalModifying(this, bound);
+                }
             }
 
             _physicalMemory.TextureCache.Lift(this);
@@ -1703,12 +1705,6 @@ namespace Ryujinx.Graphics.Gpu.Image
         /// <param name="unmapRange">The range of memory being unmapped</param>
         public void Unmapped(MultiRange unmapRange)
         {
-            if (unmapRange.Contains(Range))
-            {
-                // If this is a full unmap, prevent flushes until the texture is mapped again.
-                FlushStale = true;
-            }
-
             ChangedMapping = true;
 
             if (Group.Storage == this)
diff --git a/src/Ryujinx.Graphics.Gpu/Image/TextureGroup.cs b/src/Ryujinx.Graphics.Gpu/Image/TextureGroup.cs
index 21d7939ad..d7de8a3cd 100644
--- a/src/Ryujinx.Graphics.Gpu/Image/TextureGroup.cs
+++ b/src/Ryujinx.Graphics.Gpu/Image/TextureGroup.cs
@@ -1660,13 +1660,13 @@ namespace Ryujinx.Graphics.Gpu.Image
             }
 
             // If size is zero, we have nothing to flush.
-            // If the flush is stale, we should ignore it because the texture was unmapped since the modified
-            // flag was set, and flushing it is not safe anymore as the GPU might no longer own the memory.
-            if (size == 0 || Storage.FlushStale)
+            if (size == 0)
             {
                 return;
             }
 
+            Storage.ModifiedSinceLastFlush = false;
+
             // There is a small gap here where the action is removed but _actionRegistered is still 1.
             // In this case it will skip registering the action, but here we are already handling it,
             // so there shouldn't be any issue as it's the same handler for all actions.
diff --git a/src/Ryujinx.Graphics.Gpu/Image/TextureManager.cs b/src/Ryujinx.Graphics.Gpu/Image/TextureManager.cs
index ed181640a..e9f58314f 100644
--- a/src/Ryujinx.Graphics.Gpu/Image/TextureManager.cs
+++ b/src/Ryujinx.Graphics.Gpu/Image/TextureManager.cs
@@ -413,21 +413,35 @@ namespace Ryujinx.Graphics.Gpu.Image
         {
             bool anyChanged = false;
 
-            if (_rtHostDs != _rtDepthStencil?.HostTexture)
-            {
-                _rtHostDs = _rtDepthStencil?.HostTexture;
+            Texture dsTexture = _rtDepthStencil;
+            ITexture hostDsTexture = null;
 
+            if (dsTexture != null)
+            {
+                hostDsTexture = dsTexture.HostTexture;
+                dsTexture.ModifiedSinceLastFlush = true;
+            }
+
+            if (_rtHostDs != hostDsTexture)
+            {
+                _rtHostDs = hostDsTexture;
                 anyChanged = true;
             }
 
             for (int index = 0; index < _rtColors.Length; index++)
             {
-                ITexture hostTexture = _rtColors[index]?.HostTexture;
+                Texture texture = _rtColors[index];
+                ITexture hostTexture = null;
+
+                if (texture != null)
+                {
+                    hostTexture = texture.HostTexture;
+                    texture.ModifiedSinceLastFlush = true;
+                }
 
                 if (_rtHostColors[index] != hostTexture)
                 {
                     _rtHostColors[index] = hostTexture;
-
                     anyChanged = true;
                 }
             }