Only make render target 2D textures layered if needed (#2646)

* Only make render target 2D textures layered if needed

* Shader cache version bump

* Ensure topology is updated on channel swap

Ryujinx.Graphics.Gpu/Engine/Threed/DrawManager.cs

@@ -13,6 +13,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
         private readonly GpuChannel _channel;
         private readonly DeviceStateWithShadow<ThreedClassState> _state;
         private readonly DrawState _drawState;
+        private bool _topologySet;
 
         private bool _instancedDrawPending;
         private bool _instancedIndexed;
@@ -43,6 +44,14 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
             _drawState = drawState;
         }
 
+        /// <summary>
+        /// Marks the entire state as dirty, forcing a full host state update before the next draw.
+        /// </summary>
+        public void ForceStateDirty()
+        {
+            _topologySet = false;
+        }
+
         /// <summary>
         /// Pushes four 8-bit index buffer elements.
         /// </summary>
@@ -224,11 +233,11 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
                 _instanceIndex = 0;
             }
 
-            if (_drawState.Topology != topology)
+            if (_drawState.Topology != topology || !_topologySet)
             {
                 _context.Renderer.Pipeline.SetPrimitiveTopology(topology);
-
                 _drawState.Topology = topology;
+                _topologySet = true;
             }
         }
 
@@ -331,6 +340,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
 
             _context.Renderer.Pipeline.SetPrimitiveTopology(topology);
             _drawState.Topology = topology;
+            _topologySet = true;
 
             ConditionalRenderEnabled renderEnable = ConditionalRendering.GetRenderEnable(
                 _context,

Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs

@@ -6,7 +6,6 @@ using Ryujinx.Graphics.Gpu.Shader;
 using Ryujinx.Graphics.Shader;
 using Ryujinx.Graphics.Texture;
 using System;
-using System.Linq;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 
@@ -31,6 +30,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
 
         private readonly ShaderProgramInfo[] _currentProgramInfo;
 
+        private bool _vtgWritesRtLayer;
         private byte _vsClipDistancesWritten;
 
         private bool _prevDrawIndexed;
@@ -334,6 +334,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
                 Image.Texture color = memoryManager.Physical.TextureCache.FindOrCreateTexture(
                     memoryManager,
                     colorState,
+                    _vtgWritesRtLayer,
                     samplesInX,
                     samplesInY,
                     sizeHint);
@@ -956,6 +957,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
 
             _drawState.VsUsesInstanceId = gs.Shaders[0]?.Info.UsesInstanceId ?? false;
             _vsClipDistancesWritten = gs.Shaders[0]?.Info.ClipDistancesWritten ?? 0;
+            _vtgWritesRtLayer = false;
 
             if (oldVsClipDistancesWritten != _vsClipDistancesWritten)
             {
@@ -979,6 +981,11 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
 
                 Span<TextureBindingInfo> textureBindings = _channel.TextureManager.RentGraphicsTextureBindings(stage, info.Textures.Count);
 
+                if (info.UsesRtLayer)
+                {
+                    _vtgWritesRtLayer = true;
+                }
+
                 for (int index = 0; index < info.Textures.Count; index++)
                 {
                     var descriptor = info.Textures[index];

Ryujinx.Graphics.Gpu/Engine/Threed/ThreedClass.cs

@@ -139,6 +139,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
         /// </summary>
         public void ForceStateDirty()
         {
+            _drawManager.ForceStateDirty();
             _stateUpdater.SetAllDirty();
         }
 

Ryujinx.Graphics.Gpu/Image/TextureCache.cs

@@ -244,11 +244,18 @@ namespace Ryujinx.Graphics.Gpu.Image
         /// </summary>
         /// <param name="memoryManager">GPU memory manager where the texture is mapped</param>
         /// <param name="colorState">Color buffer texture to find or create</param>
+        /// <param name="layered">Indicates if the texture might be accessed with a non-zero layer index</param>
         /// <param name="samplesInX">Number of samples in the X direction, for MSAA</param>
         /// <param name="samplesInY">Number of samples in the Y direction, for MSAA</param>
         /// <param name="sizeHint">A hint indicating the minimum used size for the texture</param>
         /// <returns>The texture</returns>
-        public Texture FindOrCreateTexture(MemoryManager memoryManager, RtColorState colorState, int samplesInX, int samplesInY, Size sizeHint)
+        public Texture FindOrCreateTexture(
+            MemoryManager memoryManager,
+            RtColorState colorState,
+            bool layered,
+            int samplesInX,
+            int samplesInY,
+            Size sizeHint)
         {
             bool isLinear = colorState.MemoryLayout.UnpackIsLinear();
 
@@ -263,13 +270,13 @@ namespace Ryujinx.Graphics.Gpu.Image
             }
             else if ((samplesInX | samplesInY) != 1)
             {
-                target = colorState.Depth > 1
+                target = colorState.Depth > 1 && layered
                     ? Target.Texture2DMultisampleArray
                     : Target.Texture2DMultisample;
             }
             else
             {
-                target = colorState.Depth > 1
+                target = colorState.Depth > 1 && layered
                     ? Target.Texture2DArray
                     : Target.Texture2D;
             }

Ryujinx.Graphics.Gpu/Shader/Cache/Definition/HostShaderCacheEntry.cs

@@ -76,6 +76,7 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache.Definition
                                                     programInfo.Textures.Count,
                                                     programInfo.Images.Count,
                                                     programInfo.UsesInstanceId,
+                                                    programInfo.UsesRtLayer,
                                                     programInfo.ClipDistancesWritten);
             CBuffers = programInfo.CBuffers.ToArray();
             SBuffers = programInfo.SBuffers.ToArray();
@@ -89,7 +90,14 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache.Definition
         /// <returns>A new <see cref="ShaderProgramInfo"/> from this instance</returns>
         internal ShaderProgramInfo ToShaderProgramInfo()
         {
-            return new ShaderProgramInfo(CBuffers, SBuffers, Textures, Images, Header.UsesInstanceId, Header.ClipDistancesWritten);
+            return new ShaderProgramInfo(
+                CBuffers,
+                SBuffers,
+                Textures,
+                Images,
+                Header.UseFlags.HasFlag(UseFlags.InstanceId),
+                Header.UseFlags.HasFlag(UseFlags.RtLayer),
+                Header.ClipDistancesWritten);
         }
 
         /// <summary>

Ryujinx.Graphics.Gpu/Shader/Cache/Definition/HostShaderCacheEntryHeader.cs

@@ -1,8 +1,28 @@
 using System.Runtime.InteropServices;
-using Ryujinx.Graphics.Shader;
 
 namespace Ryujinx.Graphics.Gpu.Shader.Cache.Definition
 {
+    /// <summary>
+    /// Flags indicating if the shader accesses certain built-ins, such as the instance ID.
+    /// </summary>
+    enum UseFlags : byte
+    {
+        /// <summary>
+        /// None of the built-ins are used.
+        /// </summary>
+        None = 0,
+
+        /// <summary>
+        /// Indicates whenever the vertex shader reads the gl_InstanceID built-in.
+        /// </summary>
+        InstanceId = 1 << 0,
+
+        /// <summary>
+        /// Indicates whenever any of the VTG stages writes to the gl_Layer built-in.
+        /// </summary>
+        RtLayer = 1 << 1
+    }
+
     /// <summary>
     /// Host shader entry header used for binding information.
     /// </summary>
@@ -30,10 +50,9 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache.Definition
         public int ImagesCount;
 
         /// <summary>
-        /// Set to true if the shader uses instance id.
+        /// Flags indicating if the shader accesses certain built-ins, such as the instance ID.
         /// </summary>
-        [MarshalAs(UnmanagedType.I1)]
+        public UseFlags UseFlags;
-        public bool UsesInstanceId;
 
         /// <summary>
         /// Set to true if this entry is in use.
@@ -65,15 +84,22 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache.Definition
             int texturesCount,
             int imagesCount,
             bool usesInstanceId,
+            bool usesRtLayer,
             byte clipDistancesWritten) : this()
         {
             CBuffersCount        = cBuffersCount;
             SBuffersCount        = sBuffersCount;
             TexturesCount        = texturesCount;
             ImagesCount          = imagesCount;
-            UsesInstanceId       = usesInstanceId;
             ClipDistancesWritten = clipDistancesWritten;
             InUse                = true;
+
+            UseFlags = usesInstanceId ? UseFlags.InstanceId : UseFlags.None;
+
+            if (usesRtLayer)
+            {
+                UseFlags |= UseFlags.RtLayer;
+            }
         }
     }
 }

Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs

@@ -40,7 +40,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
         /// <summary>
         /// Version of the codegen (to be changed when codegen or guest format change).
         /// </summary>
-        private const ulong ShaderCodeGenVersion = 2627;
+        private const ulong ShaderCodeGenVersion = 2646;
 
         // Progress reporting helpers
         private volatile int _shaderCount;

Ryujinx.Graphics.Shader/ShaderProgramInfo.cs

@@ -11,6 +11,7 @@ namespace Ryujinx.Graphics.Shader
         public ReadOnlyCollection<TextureDescriptor> Images   { get; }
 
         public bool UsesInstanceId { get; }
+        public bool UsesRtLayer { get; }
         public byte ClipDistancesWritten { get; }
 
         public ShaderProgramInfo(
@@ -19,6 +20,7 @@ namespace Ryujinx.Graphics.Shader
             TextureDescriptor[] textures,
             TextureDescriptor[] images,
             bool                usesInstanceId,
+            bool                usesRtLayer,
             byte                clipDistancesWritten)
         {
             CBuffers = Array.AsReadOnly(cBuffers);
@@ -27,6 +29,7 @@ namespace Ryujinx.Graphics.Shader
             Images   = Array.AsReadOnly(images);
 
             UsesInstanceId = usesInstanceId;
+            UsesRtLayer = usesRtLayer;
             ClipDistancesWritten = clipDistancesWritten;
         }
     }

Ryujinx.Graphics.Shader/Translation/EmitterContext.cs

@@ -121,6 +121,11 @@ namespace Ryujinx.Graphics.Shader.Translation
                         break;
                 }
             }
+
+            if (Config.Stage != ShaderStage.Fragment && attribute == AttributeConsts.Layer)
+            {
+                Config.SetUsedFeature(FeatureFlags.RtLayer);
+            }
         }
 
         public void MarkLabel(Operand label)

Ryujinx.Graphics.Shader/Translation/FeatureFlags.cs

@@ -17,8 +17,9 @@ namespace Ryujinx.Graphics.Shader.Translation
 
         Bindless = 1 << 2,
         InstanceId = 1 << 3,
-        CbIndexing = 1 << 4,
+        RtLayer = 1 << 4,
-        IaIndexing = 1 << 5,
+        CbIndexing = 1 << 5,
-        OaIndexing = 1 << 6
+        IaIndexing = 1 << 6,
+        OaIndexing = 1 << 7
     }
 }

Ryujinx.Graphics.Shader/Translation/Translator.cs

@@ -106,6 +106,7 @@ namespace Ryujinx.Graphics.Shader.Translation
                 config.GetTextureDescriptors(),
                 config.GetImageDescriptors(),
                 config.UsedFeatures.HasFlag(FeatureFlags.InstanceId),
+                config.UsedFeatures.HasFlag(FeatureFlags.RtLayer),
                 config.ClipDistancesWritten);
 
             return program;