diff --git a/Ryujinx.Graphics.Gpu/Shader/Cache/CacheCollection.cs b/Ryujinx.Graphics.Gpu/Shader/Cache/CacheCollection.cs
index 0f4db4f35..9b61ef4a6 100644
--- a/Ryujinx.Graphics.Gpu/Shader/Cache/CacheCollection.cs
+++ b/Ryujinx.Graphics.Gpu/Shader/Cache/CacheCollection.cs
@@ -125,26 +125,26 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache
/// Get the temp path to the cache data directory.
///
/// The temp path to the cache data directory
- private string GetCacheTempDataPath() => Path.Combine(_cacheDirectory, "temp");
+ private string GetCacheTempDataPath() => CacheHelper.GetCacheTempDataPath(_cacheDirectory);
///
/// The path to the cache archive file.
///
/// The path to the cache archive file
- private string GetArchivePath() => Path.Combine(_cacheDirectory, "cache.zip");
+ private string GetArchivePath() => CacheHelper.GetArchivePath(_cacheDirectory);
///
/// The path to the cache manifest file.
///
/// The path to the cache manifest file
- private string GetManifestPath() => Path.Combine(_cacheDirectory, "cache.info");
+ private string GetManifestPath() => CacheHelper.GetManifestPath(_cacheDirectory);
///
/// Create a new temp path to the given cached file via its hash.
///
/// The hash of the cached data
/// New path to the given cached file
- private string GenCacheTempFilePath(Hash128 key) => Path.Combine(GetCacheTempDataPath(), key.ToString());
+ private string GenCacheTempFilePath(Hash128 key) => CacheHelper.GenCacheTempFilePath(_cacheDirectory, key);
///
/// Create a new cache collection.
@@ -162,7 +162,7 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache
throw new NotImplementedException($"{hashType}");
}
- _cacheDirectory = GenerateCachePath(baseCacheDirectory, graphicsApi, shaderProvider, cacheName);
+ _cacheDirectory = CacheHelper.GenerateCachePath(baseCacheDirectory, graphicsApi, shaderProvider, cacheName);
_graphicsApi = graphicsApi;
_hashType = hashType;
_version = version;
@@ -178,13 +178,9 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache
///
private void Load()
{
- bool isInvalid = false;
+ bool isValid = false;
- if (!Directory.Exists(_cacheDirectory))
- {
- isInvalid = true;
- }
- else
+ if (Directory.Exists(_cacheDirectory))
{
string manifestPath = GetManifestPath();
@@ -196,9 +192,9 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache
{
Memory hashTableRaw = rawManifest.Slice(Unsafe.SizeOf());
- isInvalid = !manifestHeader.IsValid(_version, _graphicsApi, _hashType, hashTableRaw.Span);
+ isValid = manifestHeader.IsValid(_graphicsApi, _hashType, hashTableRaw.Span) && _version == manifestHeader.Version;
- if (!isInvalid)
+ if (isValid)
{
ReadOnlySpan hashTable = MemoryMarshal.Cast(hashTableRaw.Span);
@@ -209,13 +205,9 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache
}
}
}
- else
- {
- isInvalid = true;
- }
}
- if (isInvalid)
+ if (!isValid)
{
Logger.Warning?.Print(LogClass.Gpu, $"Shader collection \"{_cacheDirectory}\" got invalidated, cache will need to be rebuilt.");
@@ -324,22 +316,7 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache
// Update the content of the zip.
lock (_hashTable)
{
- foreach (Hash128 hash in _hashTable)
- {
- string cacheTempFilePath = GenCacheTempFilePath(hash);
-
- if (File.Exists(cacheTempFilePath))
- {
- string cacheHash = $"{hash}";
-
- ZipArchiveEntry entry = _cacheArchive.GetEntry(cacheHash);
-
- entry?.Delete();
-
- _cacheArchive.CreateEntryFromFile(cacheTempFilePath, cacheHash);
- File.Delete(cacheTempFilePath);
- }
- }
+ CacheHelper.EnsureArchiveUpToDate(_cacheDirectory, _cacheArchive, _hashTable);
// Close the instance to force a flush.
_cacheArchive.Dispose();
@@ -362,56 +339,16 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache
///
private void SaveManifest()
{
- CacheManifestHeader manifestHeader = new CacheManifestHeader(_version, _graphicsApi, _hashType);
-
byte[] data;
lock (_hashTable)
{
- data = new byte[Unsafe.SizeOf() + _hashTable.Count * Unsafe.SizeOf()];
-
- // CacheManifestHeader has the same size as a Hash128.
- Span dataSpan = MemoryMarshal.Cast(data.AsSpan()).Slice(1);
-
- int i = 0;
-
- foreach (Hash128 hash in _hashTable)
- {
- dataSpan[i++] = hash;
- }
+ data = CacheHelper.ComputeManifest(_version, _graphicsApi, _hashType, _hashTable);
}
- manifestHeader.UpdateChecksum(data.AsSpan().Slice(Unsafe.SizeOf()));
-
- MemoryMarshal.Write(data, ref manifestHeader);
-
File.WriteAllBytes(GetManifestPath(), data);
}
- ///
- /// Generate the path to the cache directory.
- ///
- /// The base of the cache directory
- /// The graphics api in use
- /// The name of the shader provider in use
- /// The name of the cache
- /// The path to the cache directory
- private static string GenerateCachePath(string baseCacheDirectory, CacheGraphicsApi graphicsApi, string shaderProvider, string cacheName)
- {
- string graphicsApiName = graphicsApi switch
- {
- CacheGraphicsApi.OpenGL => "opengl",
- CacheGraphicsApi.OpenGLES => "opengles",
- CacheGraphicsApi.Vulkan => "vulkan",
- CacheGraphicsApi.DirectX => "directx",
- CacheGraphicsApi.Metal => "metal",
- CacheGraphicsApi.Guest => "guest",
- _ => throw new NotImplementedException(graphicsApi.ToString()),
- };
-
- return Path.Combine(baseCacheDirectory, graphicsApiName, shaderProvider, cacheName);
- }
-
///
/// Get a cached file with the given hash.
///
@@ -438,27 +375,7 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache
if (found)
{
- ZipArchiveEntry archiveEntry = _cacheArchive.GetEntry($"{keyHash}");
-
- if (archiveEntry != null)
- {
- try
- {
- byte[] result = new byte[archiveEntry.Length];
-
- using (Stream archiveStream = archiveEntry.Open())
- {
- archiveStream.Read(result);
-
- return result;
- }
- }
- catch (Exception e)
- {
- Logger.Error?.Print(LogClass.Gpu, $"Cannot load cache file {keyHash} from archive");
- Logger.Error?.Print(LogClass.Gpu, e.ToString());
- }
- }
+ return CacheHelper.ReadFromArchive(_cacheArchive, keyHash);
}
return null;
@@ -480,17 +397,7 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache
if (found)
{
- string cacheTempFilePath = GenCacheTempFilePath(keyHash);
-
- try
- {
- return File.ReadAllBytes(GenCacheTempFilePath(keyHash));
- }
- catch (Exception e)
- {
- Logger.Error?.Print(LogClass.Gpu, $"Cannot load cache file at {cacheTempFilePath}");
- Logger.Error?.Print(LogClass.Gpu, e.ToString());
- }
+ return CacheHelper.ReadFromFile(GetCacheTempDataPath(), keyHash);
}
return null;
diff --git a/Ryujinx.Graphics.Gpu/Shader/Cache/CacheHelper.cs b/Ryujinx.Graphics.Gpu/Shader/Cache/CacheHelper.cs
new file mode 100644
index 000000000..1d492214e
--- /dev/null
+++ b/Ryujinx.Graphics.Gpu/Shader/Cache/CacheHelper.cs
@@ -0,0 +1,482 @@
+using Ryujinx.Common;
+using Ryujinx.Common.Configuration;
+using Ryujinx.Common.Logging;
+using Ryujinx.Graphics.GAL;
+using Ryujinx.Graphics.Gpu.Memory;
+using Ryujinx.Graphics.Gpu.Shader.Cache.Definition;
+using Ryujinx.Graphics.Shader;
+using Ryujinx.Graphics.Shader.Translation;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+namespace Ryujinx.Graphics.Gpu.Shader.Cache
+{
+ ///
+ /// Helper to manipulate the disk shader cache.
+ ///
+ static class CacheHelper
+ {
+ ///
+ /// Try to read the manifest header from a given file path.
+ ///
+ /// The path to the manifest file
+ /// The manifest header read
+ /// Return true if the manifest header was read
+ public static bool TryReadManifestHeader(string manifestPath, out CacheManifestHeader header)
+ {
+ header = default;
+
+ if (File.Exists(manifestPath))
+ {
+ Memory rawManifest = File.ReadAllBytes(manifestPath);
+
+ if (MemoryMarshal.TryRead(rawManifest.Span, out header))
+ {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ ///
+ /// Try to read the manifest from a given file path.
+ ///
+ /// The path to the manifest file
+ /// The graphics api used by the cache
+ /// The hash type of the cache
+ /// The manifest header read
+ /// The entries read from the cache manifest
+ /// Return true if the manifest was read
+ public static bool TryReadManifestFile(string manifestPath, CacheGraphicsApi graphicsApi, CacheHashType hashType, out CacheManifestHeader header, out HashSet entries)
+ {
+ header = default;
+ entries = new HashSet();
+
+ if (File.Exists(manifestPath))
+ {
+ Memory rawManifest = File.ReadAllBytes(manifestPath);
+
+ if (MemoryMarshal.TryRead(rawManifest.Span, out header))
+ {
+ Memory hashTableRaw = rawManifest.Slice(Unsafe.SizeOf());
+
+ bool isValid = header.IsValid(graphicsApi, hashType, hashTableRaw.Span);
+
+ if (isValid)
+ {
+ ReadOnlySpan hashTable = MemoryMarshal.Cast(hashTableRaw.Span);
+
+ foreach (Hash128 hash in hashTable)
+ {
+ entries.Add(hash);
+ }
+ }
+
+ return isValid;
+ }
+ }
+
+ return false;
+ }
+
+ ///
+ /// Compute a cache manifest from runtime data.
+ ///
+ /// The version of the cache
+ /// The graphics api used by the cache
+ /// The hash type of the cache
+ /// The entries in the cache
+ /// The cache manifest from runtime data
+ public static byte[] ComputeManifest(ulong version, CacheGraphicsApi graphicsApi, CacheHashType hashType, HashSet entries)
+ {
+ if (hashType != CacheHashType.XxHash128)
+ {
+ throw new NotImplementedException($"{hashType}");
+ }
+
+ CacheManifestHeader manifestHeader = new CacheManifestHeader(version, graphicsApi, hashType);
+
+ byte[] data = new byte[Unsafe.SizeOf() + entries.Count * Unsafe.SizeOf()];
+
+ // CacheManifestHeader has the same size as a Hash128.
+ Span dataSpan = MemoryMarshal.Cast(data.AsSpan()).Slice(1);
+
+ int i = 0;
+
+ foreach (Hash128 hash in entries)
+ {
+ dataSpan[i++] = hash;
+ }
+
+ manifestHeader.UpdateChecksum(data.AsSpan().Slice(Unsafe.SizeOf()));
+
+ MemoryMarshal.Write(data, ref manifestHeader);
+
+ return data;
+ }
+
+ ///
+ /// Get the base directory of the shader cache for a given title id.
+ ///
+ /// The title id of the target application
+ /// The base directory of the shader cache for a given title id
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static string GetBaseCacheDirectory(string titleId) => Path.Combine(AppDataManager.GamesDirPath, titleId, "cache", "shader");
+
+ ///
+ /// Get the temp path to the cache data directory.
+ ///
+ /// The cache directory
+ /// The temp path to the cache data directory
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static string GetCacheTempDataPath(string cacheDirectory) => Path.Combine(cacheDirectory, "temp");
+
+ ///
+ /// The path to the cache archive file.
+ ///
+ /// The cache directory
+ /// The path to the cache archive file
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static string GetArchivePath(string cacheDirectory) => Path.Combine(cacheDirectory, "cache.zip");
+
+ ///
+ /// The path to the cache manifest file.
+ ///
+ /// The cache directory
+ /// The path to the cache manifest file
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static string GetManifestPath(string cacheDirectory) => Path.Combine(cacheDirectory, "cache.info");
+
+ ///
+ /// Create a new temp path to the given cached file via its hash.
+ ///
+ /// The cache directory
+ /// The hash of the cached data
+ /// New path to the given cached file
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static string GenCacheTempFilePath(string cacheDirectory, Hash128 key) => Path.Combine(GetCacheTempDataPath(cacheDirectory), key.ToString());
+
+ ///
+ /// Generate the path to the cache directory.
+ ///
+ /// The base of the cache directory
+ /// The graphics api in use
+ /// The name of the shader provider in use
+ /// The name of the cache
+ /// The path to the cache directory
+ public static string GenerateCachePath(string baseCacheDirectory, CacheGraphicsApi graphicsApi, string shaderProvider, string cacheName)
+ {
+ string graphicsApiName = graphicsApi switch
+ {
+ CacheGraphicsApi.OpenGL => "opengl",
+ CacheGraphicsApi.OpenGLES => "opengles",
+ CacheGraphicsApi.Vulkan => "vulkan",
+ CacheGraphicsApi.DirectX => "directx",
+ CacheGraphicsApi.Metal => "metal",
+ CacheGraphicsApi.Guest => "guest",
+ _ => throw new NotImplementedException(graphicsApi.ToString()),
+ };
+
+ return Path.Combine(baseCacheDirectory, graphicsApiName, shaderProvider, cacheName);
+ }
+
+ ///
+ /// Read a cached file with the given hash that is present in the archive.
+ ///
+ /// The archive in use
+ /// The given hash
+ /// The cached file if present or null
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static byte[] ReadFromArchive(ZipArchive archive, Hash128 entry)
+ {
+ if (archive != null)
+ {
+ ZipArchiveEntry archiveEntry = archive.GetEntry($"{entry}");
+
+ if (archiveEntry != null)
+ {
+ try
+ {
+ byte[] result = new byte[archiveEntry.Length];
+
+ using (Stream archiveStream = archiveEntry.Open())
+ {
+ archiveStream.Read(result);
+
+ return result;
+ }
+ }
+ catch (Exception e)
+ {
+ Logger.Error?.Print(LogClass.Gpu, $"Cannot load cache file {entry} from archive");
+ Logger.Error?.Print(LogClass.Gpu, e.ToString());
+ }
+ }
+ }
+
+ return null;
+ }
+
+ ///
+ /// Read a cached file with the given hash that is not present in the archive.
+ ///
+ /// The cache directory
+ /// The given hash
+ /// The cached file if present or null
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static byte[] ReadFromFile(string cacheDirectory, Hash128 entry)
+ {
+ string cacheTempFilePath = GenCacheTempFilePath(cacheDirectory, entry);
+
+ try
+ {
+ return File.ReadAllBytes(cacheTempFilePath);
+ }
+ catch (Exception e)
+ {
+ Logger.Error?.Print(LogClass.Gpu, $"Cannot load cache file at {cacheTempFilePath}");
+ Logger.Error?.Print(LogClass.Gpu, e.ToString());
+ }
+
+ return null;
+ }
+
+ ///
+ /// Compute the guest program code for usage while dumping to disk or hash.
+ ///
+ /// The guest shader entries to use
+ /// The transform feedback descriptors
+ /// Used to determine if the guest program code is generated for hashing
+ /// The guest program code for usage while dumping to disk or hash
+ private static byte[] ComputeGuestProgramCode(ReadOnlySpan cachedShaderEntries, TransformFeedbackDescriptor[] tfd, bool forHashCompute = false)
+ {
+ using (MemoryStream stream = new MemoryStream())
+ {
+ BinaryWriter writer = new BinaryWriter(stream);
+
+ foreach (GuestShaderCacheEntry cachedShaderEntry in cachedShaderEntries)
+ {
+ if (cachedShaderEntry != null)
+ {
+ // Code (and Code A if present)
+ stream.Write(cachedShaderEntry.Code);
+
+ if (forHashCompute)
+ {
+ // Guest GPU accessor header (only write this for hashes, already present in the header for dumps)
+ writer.WriteStruct(cachedShaderEntry.Header.GpuAccessorHeader);
+ }
+
+ // Texture descriptors
+ foreach (GuestTextureDescriptor textureDescriptor in cachedShaderEntry.TextureDescriptors.Values)
+ {
+ writer.WriteStruct(textureDescriptor);
+ }
+ }
+ }
+
+ // Transformation feedback
+ if (tfd != null)
+ {
+ foreach (TransformFeedbackDescriptor transform in tfd)
+ {
+ writer.WriteStruct(new GuestShaderCacheTransformFeedbackHeader(transform.BufferIndex, transform.Stride, transform.VaryingLocations.Length));
+ writer.Write(transform.VaryingLocations);
+ }
+ }
+
+ return stream.ToArray();
+ }
+ }
+
+ ///
+ /// Compute a guest hash from shader entries.
+ ///
+ /// The guest shader entries to use
+ /// The optional transform feedback descriptors
+ /// A guest hash from shader entries
+ public static Hash128 ComputeGuestHashFromCache(ReadOnlySpan cachedShaderEntries, TransformFeedbackDescriptor[] tfd = null)
+ {
+ return XXHash128.ComputeHash(ComputeGuestProgramCode(cachedShaderEntries, tfd, true));
+ }
+
+ ///
+ /// Read transform feedback descriptors from guest.
+ ///
+ /// The raw guest transform feedback descriptors
+ /// The guest shader program header
+ /// The transform feedback descriptors read from guest
+ public static TransformFeedbackDescriptor[] ReadTransformationFeedbackInformations(ref ReadOnlySpan data, GuestShaderCacheHeader header)
+ {
+ if (header.TransformFeedbackCount != 0)
+ {
+ TransformFeedbackDescriptor[] result = new TransformFeedbackDescriptor[header.TransformFeedbackCount];
+
+ for (int i = 0; i < result.Length; i++)
+ {
+ GuestShaderCacheTransformFeedbackHeader feedbackHeader = MemoryMarshal.Read(data);
+
+ result[i] = new TransformFeedbackDescriptor(feedbackHeader.BufferIndex, feedbackHeader.Stride, data.Slice(Unsafe.SizeOf(), feedbackHeader.VaryingLocationsLength).ToArray());
+
+ data = data.Slice(Unsafe.SizeOf() + feedbackHeader.VaryingLocationsLength);
+ }
+
+ return result;
+ }
+
+ return null;
+ }
+
+ ///
+ /// Create a new instance of from an gpu accessor.
+ ///
+ /// The gpu accessor
+ /// A new instance of
+ public static GuestGpuAccessorHeader CreateGuestGpuAccessorCache(IGpuAccessor gpuAccessor)
+ {
+ return new GuestGpuAccessorHeader
+ {
+ ComputeLocalSizeX = gpuAccessor.QueryComputeLocalSizeX(),
+ ComputeLocalSizeY = gpuAccessor.QueryComputeLocalSizeY(),
+ ComputeLocalSizeZ = gpuAccessor.QueryComputeLocalSizeZ(),
+ ComputeLocalMemorySize = gpuAccessor.QueryComputeLocalMemorySize(),
+ ComputeSharedMemorySize = gpuAccessor.QueryComputeSharedMemorySize(),
+ PrimitiveTopology = gpuAccessor.QueryPrimitiveTopology(),
+ };
+ }
+
+ ///
+ /// Create guest shader cache entries from the runtime contexts.
+ ///
+ /// The GPU memory manager in use
+ /// The runtime contexts
+ /// Guest shader cahe entries from the runtime contexts
+ public static GuestShaderCacheEntry[] CreateShaderCacheEntries(MemoryManager memoryManager, ReadOnlySpan shaderContexts)
+ {
+ GuestShaderCacheEntry[] entries = new GuestShaderCacheEntry[shaderContexts.Length];
+
+ for (int i = 0; i < shaderContexts.Length; i++)
+ {
+ TranslatorContext context = shaderContexts[i];
+
+ if (context == null)
+ {
+ continue;
+ }
+
+ int sizeA = context.AddressA == 0 ? 0 : context.SizeA;
+
+ byte[] code = new byte[context.Size + sizeA];
+
+ memoryManager.GetSpan(context.Address, context.Size).CopyTo(code);
+
+ if (context.AddressA != 0)
+ {
+ memoryManager.GetSpan(context.AddressA, context.SizeA).CopyTo(code.AsSpan().Slice(context.Size, context.SizeA));
+ }
+
+ GuestGpuAccessorHeader gpuAccessorHeader = CreateGuestGpuAccessorCache(context.GpuAccessor);
+
+ if (context.GpuAccessor is GpuAccessor)
+ {
+ gpuAccessorHeader.TextureDescriptorCount = context.TextureHandlesForCache.Count;
+ }
+
+ GuestShaderCacheEntryHeader header = new GuestShaderCacheEntryHeader(context.Stage, context.Size, sizeA, gpuAccessorHeader);
+
+ GuestShaderCacheEntry entry = new GuestShaderCacheEntry(header, code);
+
+ if (context.GpuAccessor is GpuAccessor gpuAccessor)
+ {
+ foreach (int textureHandle in context.TextureHandlesForCache)
+ {
+ GuestTextureDescriptor textureDescriptor = ((Image.TextureDescriptor)gpuAccessor.GetTextureDescriptor(textureHandle)).ToCache();
+
+ textureDescriptor.Handle = (uint)textureHandle;
+
+ entry.TextureDescriptors.Add(textureHandle, textureDescriptor);
+ }
+ }
+
+ entries[i] = entry;
+ }
+
+ return entries;
+ }
+
+ ///
+ /// Create a guest shader program.
+ ///
+ /// The entries composing the guest program dump
+ /// The transform feedback descriptors in use
+ /// The resulting guest shader program
+ public static byte[] CreateGuestProgramDump(GuestShaderCacheEntry[] shaderCacheEntries, TransformFeedbackDescriptor[] tfd = null)
+ {
+ using (MemoryStream resultStream = new MemoryStream())
+ {
+ BinaryWriter resultStreamWriter = new BinaryWriter(resultStream);
+
+ byte transformFeedbackCount = 0;
+
+ if (tfd != null)
+ {
+ transformFeedbackCount = (byte)tfd.Length;
+ }
+
+ // Header
+ resultStreamWriter.WriteStruct(new GuestShaderCacheHeader((byte)shaderCacheEntries.Length, transformFeedbackCount));
+
+ // Write all entries header
+ foreach (GuestShaderCacheEntry entry in shaderCacheEntries)
+ {
+ if (entry == null)
+ {
+ resultStreamWriter.WriteStruct(new GuestShaderCacheEntryHeader());
+ }
+ else
+ {
+ resultStreamWriter.WriteStruct(entry.Header);
+ }
+ }
+
+ // Finally, write all program code and all transform feedback information.
+ resultStreamWriter.Write(ComputeGuestProgramCode(shaderCacheEntries, tfd));
+
+ return resultStream.ToArray();
+ }
+ }
+
+ ///
+ /// Save temporary files not in archive.
+ ///
+ /// The base of the cache directory
+ /// The archive to use
+ /// The entries in the cache
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static void EnsureArchiveUpToDate(string baseCacheDirectory, ZipArchive archive, HashSet entries)
+ {
+ foreach (Hash128 hash in entries)
+ {
+ string cacheTempFilePath = GenCacheTempFilePath(baseCacheDirectory, hash);
+
+ if (File.Exists(cacheTempFilePath))
+ {
+ string cacheHash = $"{hash}";
+
+ ZipArchiveEntry entry = archive.GetEntry(cacheHash);
+
+ entry?.Delete();
+
+ archive.CreateEntryFromFile(cacheTempFilePath, cacheHash);
+
+ File.Delete(cacheTempFilePath);
+ }
+ }
+ }
+ }
+}
diff --git a/Ryujinx.Graphics.Gpu/Shader/Cache/CacheManager.cs b/Ryujinx.Graphics.Gpu/Shader/Cache/CacheManager.cs
index f977e96bc..ca0070fdc 100644
--- a/Ryujinx.Graphics.Gpu/Shader/Cache/CacheManager.cs
+++ b/Ryujinx.Graphics.Gpu/Shader/Cache/CacheManager.cs
@@ -29,7 +29,7 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache
///
/// Version of the guest cache shader (to increment when guest cache structure change).
///
- private const ulong GuestCacheVersion = 1717;
+ private const ulong GuestCacheVersion = 1759;
///
/// Create a new cache manager instance
@@ -45,7 +45,9 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache
_hashType = hashType;
_shaderProvider = shaderProvider;
- string baseCacheDirectory = Path.Combine(AppDataManager.GamesDirPath, titleId, "cache", "shader");
+ string baseCacheDirectory = CacheHelper.GetBaseCacheDirectory(titleId);
+
+ CacheMigration.Run(baseCacheDirectory, graphicsApi, hashType, shaderProvider);
_guestProgramCache = new CacheCollection(baseCacheDirectory, _hashType, CacheGraphicsApi.Guest, "", "program", GuestCacheVersion);
_hostProgramCache = new CacheCollection(baseCacheDirectory, _hashType, _graphicsApi, _shaderProvider, "host", shaderCodeGenVersion);
@@ -80,16 +82,6 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache
_hostProgramCache.Synchronize();
}
- ///
- /// Computes the hash of some data using the current cache hashing algorithm.
- ///
- /// Some data to generate a hash for.
- /// The hash of some data using the current hashing algorithm of the cache
- public Hash128 ComputeHash(ReadOnlySpan data)
- {
- return XXHash128.ComputeHash(data);
- }
-
///
/// Save a shader program not present in the program cache.
///
diff --git a/Ryujinx.Graphics.Gpu/Shader/Cache/CacheMigration.cs b/Ryujinx.Graphics.Gpu/Shader/Cache/CacheMigration.cs
new file mode 100644
index 000000000..965287b54
--- /dev/null
+++ b/Ryujinx.Graphics.Gpu/Shader/Cache/CacheMigration.cs
@@ -0,0 +1,158 @@
+using Ryujinx.Common;
+using Ryujinx.Common.Logging;
+using Ryujinx.Graphics.GAL;
+using Ryujinx.Graphics.Gpu.Shader.Cache.Definition;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+
+namespace Ryujinx.Graphics.Gpu.Shader.Cache
+{
+ ///
+ /// Class handling shader cache migrations.
+ ///
+ static class CacheMigration
+ {
+ ///
+ /// Check if the given cache version need to recompute its hash.
+ ///
+ /// The version in use
+ /// The new version after migration
+ /// True if a hash recompute is needed
+ public static bool NeedHashRecompute(ulong version, out ulong newVersion)
+ {
+ const ulong TargetBrokenVersion = 1717;
+ const ulong TargetFixedVersion = 1759;
+
+ newVersion = TargetFixedVersion;
+
+ if (version == TargetBrokenVersion)
+ {
+ return true;
+ }
+
+ return false;
+ }
+
+ ///
+ /// Move a file with the name of a given hash to another in the cache archive.
+ ///
+ /// The archive in use
+ /// The old key
+ /// The new key
+ private static void MoveEntry(ZipArchive archive, Hash128 oldKey, Hash128 newKey)
+ {
+ ZipArchiveEntry oldGuestEntry = archive.GetEntry($"{oldKey}");
+
+ if (oldGuestEntry != null)
+ {
+ ZipArchiveEntry newGuestEntry = archive.CreateEntry($"{newKey}");
+
+ using (Stream oldStream = oldGuestEntry.Open())
+ using (Stream newStream = newGuestEntry.Open())
+ {
+ oldStream.CopyTo(newStream);
+ }
+
+ oldGuestEntry.Delete();
+ }
+ }
+
+ ///
+ /// Recompute all the hashes of a given cache.
+ ///
+ /// The guest cache directory path
+ /// The host cache directory path
+ /// The graphics api in use
+ /// The hash type in use
+ /// The version to write in the host and guest manifest after migration
+ private static void RecomputeHashes(string guestBaseCacheDirectory, string hostBaseCacheDirectory, CacheGraphicsApi graphicsApi, CacheHashType hashType, ulong newVersion)
+ {
+ string guestManifestPath = CacheHelper.GetManifestPath(guestBaseCacheDirectory);
+ string hostManifestPath = CacheHelper.GetManifestPath(hostBaseCacheDirectory);
+
+ if (CacheHelper.TryReadManifestFile(guestManifestPath, CacheGraphicsApi.Guest, hashType, out _, out HashSet guestEntries))
+ {
+ CacheHelper.TryReadManifestFile(hostManifestPath, graphicsApi, hashType, out _, out HashSet hostEntries);
+
+ Logger.Info?.Print(LogClass.Gpu, "Shader cache hashes need to be recomputed, performing migration...");
+
+ string guestArchivePath = CacheHelper.GetArchivePath(guestBaseCacheDirectory);
+ string hostArchivePath = CacheHelper.GetArchivePath(hostBaseCacheDirectory);
+
+ ZipArchive guestArchive = ZipFile.Open(guestArchivePath, ZipArchiveMode.Update);
+ ZipArchive hostArchive = ZipFile.Open(hostArchivePath, ZipArchiveMode.Update);
+
+ CacheHelper.EnsureArchiveUpToDate(guestBaseCacheDirectory, guestArchive, guestEntries);
+ CacheHelper.EnsureArchiveUpToDate(hostBaseCacheDirectory, hostArchive, hostEntries);
+
+ int programIndex = 0;
+
+ HashSet newEntries = new HashSet();
+
+ foreach (Hash128 oldHash in guestEntries)
+ {
+ byte[] guestProgram = CacheHelper.ReadFromArchive(guestArchive, oldHash);
+
+ Logger.Info?.Print(LogClass.Gpu, $"Migrating shader {oldHash} ({programIndex + 1} / {guestEntries.Count})");
+
+ if (guestProgram != null)
+ {
+ ReadOnlySpan guestProgramReadOnlySpan = guestProgram;
+
+ ReadOnlySpan cachedShaderEntries = GuestShaderCacheEntry.Parse(ref guestProgramReadOnlySpan, out GuestShaderCacheHeader fileHeader);
+
+ TransformFeedbackDescriptor[] tfd = CacheHelper.ReadTransformationFeedbackInformations(ref guestProgramReadOnlySpan, fileHeader);
+
+ Hash128 newHash = CacheHelper.ComputeGuestHashFromCache(cachedShaderEntries, tfd);
+
+ if (newHash != oldHash)
+ {
+ MoveEntry(guestArchive, oldHash, newHash);
+ MoveEntry(hostArchive, oldHash, newHash);
+ }
+ else
+ {
+ Logger.Warning?.Print(LogClass.Gpu, $"Same hashes for shader {oldHash}");
+ }
+
+ newEntries.Add(newHash);
+ }
+
+ programIndex++;
+ }
+
+ byte[] newGuestManifestContent = CacheHelper.ComputeManifest(newVersion, CacheGraphicsApi.Guest, hashType, newEntries);
+ byte[] newHostManifestContent = CacheHelper.ComputeManifest(newVersion, graphicsApi, hashType, newEntries);
+
+ File.WriteAllBytes(guestManifestPath, newGuestManifestContent);
+ File.WriteAllBytes(hostManifestPath, newHostManifestContent);
+
+ guestArchive.Dispose();
+ hostArchive.Dispose();
+ }
+ }
+
+ ///
+ /// Check and run cache migration if needed.
+ ///
+ /// The base path of the cache
+ /// The graphics api in use
+ /// The hash type in use
+ /// The shader provider name of the cache
+ public static void Run(string baseCacheDirectory, CacheGraphicsApi graphicsApi, CacheHashType hashType, string shaderProvider)
+ {
+ string guestBaseCacheDirectory = CacheHelper.GenerateCachePath(baseCacheDirectory, CacheGraphicsApi.Guest, "", "program");
+ string hostBaseCacheDirectory = CacheHelper.GenerateCachePath(baseCacheDirectory, graphicsApi, shaderProvider, "host");
+
+ if (CacheHelper.TryReadManifestHeader(CacheHelper.GetManifestPath(guestBaseCacheDirectory), out CacheManifestHeader header))
+ {
+ if (NeedHashRecompute(header.Version, out ulong newVersion))
+ {
+ RecomputeHashes(guestBaseCacheDirectory, hostBaseCacheDirectory, graphicsApi, hashType, newVersion);
+ }
+ }
+ }
+ }
+}
diff --git a/Ryujinx.Graphics.Gpu/Shader/Cache/Definition/CacheManifestHeader.cs b/Ryujinx.Graphics.Gpu/Shader/Cache/Definition/CacheManifestHeader.cs
index 3f198dca5..0601451dd 100644
--- a/Ryujinx.Graphics.Gpu/Shader/Cache/Definition/CacheManifestHeader.cs
+++ b/Ryujinx.Graphics.Gpu/Shader/Cache/Definition/CacheManifestHeader.cs
@@ -84,14 +84,14 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache.Definition
///
/// Check the validity of the header.
///
- /// The target version in use
/// The target graphics api in use
/// The target hash type in use
/// The data after this header
/// True if the header is valid
- public bool IsValid(ulong version, CacheGraphicsApi graphicsApi, CacheHashType hashType, ReadOnlySpan data)
+ /// This doesn't check that versions match
+ public bool IsValid(CacheGraphicsApi graphicsApi, CacheHashType hashType, ReadOnlySpan data)
{
- return Version == version && GraphicsApi == graphicsApi && HashType == hashType && TableChecksum == CalculateCrc16(data);
+ return GraphicsApi == graphicsApi && HashType == hashType && TableChecksum == CalculateCrc16(data);
}
}
}
diff --git a/Ryujinx.Graphics.Gpu/Shader/Cache/Definition/GuestShaderCacheEntry.cs b/Ryujinx.Graphics.Gpu/Shader/Cache/Definition/GuestShaderCacheEntry.cs
index 45a442e2b..373fa6c64 100644
--- a/Ryujinx.Graphics.Gpu/Shader/Cache/Definition/GuestShaderCacheEntry.cs
+++ b/Ryujinx.Graphics.Gpu/Shader/Cache/Definition/GuestShaderCacheEntry.cs
@@ -31,7 +31,7 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache.Definition
///
/// The header of the cached shader entry
/// The code of this shader
- private GuestShaderCacheEntry(GuestShaderCacheEntryHeader header, byte[] code)
+ public GuestShaderCacheEntry(GuestShaderCacheEntryHeader header, byte[] code)
{
Header = header;
Code = code;
diff --git a/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs b/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs
index b469aab52..a04affc20 100644
--- a/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs
+++ b/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs
@@ -9,9 +9,6 @@ using Ryujinx.Graphics.Shader.Translation;
using System;
using System.Collections.Generic;
using System.Diagnostics;
-using System.IO;
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
namespace Ryujinx.Graphics.Gpu.Shader
{
@@ -37,7 +34,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
///
/// Version of the codegen (to be changed when codegen or guest format change).
///
- private const ulong ShaderCodeGenVersion = 1717;
+ private const ulong ShaderCodeGenVersion = 1759;
///
/// Creates a new instance of the shader cache.
@@ -165,7 +162,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
ShaderCodeHolder[] shaders = new ShaderCodeHolder[cachedShaderEntries.Length];
List shaderPrograms = new List();
- TransformFeedbackDescriptor[] tfd = ReadTransformationFeedbackInformations(ref guestProgramReadOnlySpan, fileHeader);
+ TransformFeedbackDescriptor[] tfd = CacheHelper.ReadTransformationFeedbackInformations(ref guestProgramReadOnlySpan, fileHeader);
TranslationFlags flags = DefaultFlags;
@@ -347,14 +344,14 @@ namespace Ryujinx.Graphics.Gpu.Shader
bool isShaderCacheEnabled = _cacheManager != null;
- byte[] programCode = null;
Hash128 programCodeHash = default;
- GuestShaderCacheEntryHeader[] shaderCacheEntries = null;
+ GuestShaderCacheEntry[] shaderCacheEntries = null;
if (isShaderCacheEnabled)
{
// Compute hash and prepare data for shader disk cache comparison.
- GetProgramInformations(null, shaderContexts, out programCode, out programCodeHash, out shaderCacheEntries);
+ shaderCacheEntries = CacheHelper.CreateShaderCacheEntries(_context.MemoryManager, shaderContexts);
+ programCodeHash = CacheHelper.ComputeGuestHashFromCache(shaderCacheEntries);
}
ShaderBundle cpShader;
@@ -381,7 +378,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
if (isShaderCacheEnabled)
{
_cpProgramsDiskCache.Add(programCodeHash, cpShader);
- _cacheManager.SaveProgram(ref programCodeHash, CreateGuestProgramDump(programCode, shaderCacheEntries, null), hostProgramBinary);
+ _cacheManager.SaveProgram(ref programCodeHash, CacheHelper.CreateGuestProgramDump(shaderCacheEntries), hostProgramBinary);
}
}
@@ -451,14 +448,14 @@ namespace Ryujinx.Graphics.Gpu.Shader
bool isShaderCacheEnabled = _cacheManager != null;
- byte[] programCode = null;
Hash128 programCodeHash = default;
- GuestShaderCacheEntryHeader[] shaderCacheEntries = null;
+ GuestShaderCacheEntry[] shaderCacheEntries = null;
if (isShaderCacheEnabled)
{
// Compute hash and prepare data for shader disk cache comparison.
- GetProgramInformations(tfd, shaderContexts, out programCode, out programCodeHash, out shaderCacheEntries);
+ shaderCacheEntries = CacheHelper.CreateShaderCacheEntries(_context.MemoryManager, shaderContexts);
+ programCodeHash = CacheHelper.ComputeGuestHashFromCache(shaderCacheEntries, tfd);
}
ShaderBundle gpShaders;
@@ -507,7 +504,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
if (isShaderCacheEnabled)
{
_gpProgramsDiskCache.Add(programCodeHash, gpShaders);
- _cacheManager.SaveProgram(ref programCodeHash, CreateGuestProgramDump(programCode, shaderCacheEntries, tfd), hostProgramBinary);
+ _cacheManager.SaveProgram(ref programCodeHash, CacheHelper.CreateGuestProgramDump(shaderCacheEntries, tfd), hostProgramBinary);
}
}
@@ -766,191 +763,5 @@ namespace Ryujinx.Graphics.Gpu.Shader
_cacheManager?.Dispose();
}
-
- ///
- /// Create a guest shader program.
- ///
- /// The program code of the shader code
- /// The resulting guest shader entries header
- /// The transform feedback descriptors in use
- /// The resulting guest shader program
- private static byte[] CreateGuestProgramDump(ReadOnlySpan programCode, GuestShaderCacheEntryHeader[] shaderCacheEntries, TransformFeedbackDescriptor[] tfd)
- {
- using (MemoryStream resultStream = new MemoryStream())
- {
- BinaryWriter resultStreamWriter = new BinaryWriter(resultStream);
-
- byte transformFeedbackCount = 0;
-
- if (tfd != null)
- {
- transformFeedbackCount = (byte)tfd.Length;
- }
-
- // Header
- resultStreamWriter.WriteStruct(new GuestShaderCacheHeader((byte)shaderCacheEntries.Length, transformFeedbackCount));
-
- // Write all entries header
- foreach (GuestShaderCacheEntryHeader entry in shaderCacheEntries)
- {
- resultStreamWriter.WriteStruct(entry);
- }
-
- // Finally, write all program code and all transform feedback information.
- resultStreamWriter.Write(programCode);
-
- return resultStream.ToArray();
- }
- }
-
- ///
- /// Write transform feedback guest information to the given stream.
- ///
- /// The stream to write data to
- /// The current transform feedback descriptors used
- private static void WriteTransformationFeedbackInformation(Stream stream, TransformFeedbackDescriptor[] tfd)
- {
- if (tfd != null)
- {
- BinaryWriter writer = new BinaryWriter(stream);
-
- foreach (TransformFeedbackDescriptor transform in tfd)
- {
- writer.WriteStruct(new GuestShaderCacheTransformFeedbackHeader(transform.BufferIndex, transform.Stride, transform.VaryingLocations.Length));
- writer.Write(transform.VaryingLocations);
- }
- }
- }
-
- ///
- /// Read transform feedback descriptors from guest.
- ///
- /// The raw guest transform feedback descriptors
- /// The guest shader program header
- /// The transform feedback descriptors read from guest
- private static TransformFeedbackDescriptor[] ReadTransformationFeedbackInformations(ref ReadOnlySpan data, GuestShaderCacheHeader header)
- {
- if (header.TransformFeedbackCount != 0)
- {
- TransformFeedbackDescriptor[] result = new TransformFeedbackDescriptor[header.TransformFeedbackCount];
-
- for (int i = 0; i < result.Length; i++)
- {
- GuestShaderCacheTransformFeedbackHeader feedbackHeader = MemoryMarshal.Read(data);
-
- result[i] = new TransformFeedbackDescriptor(feedbackHeader.BufferIndex, feedbackHeader.Stride, data.Slice(Unsafe.SizeOf(), feedbackHeader.VaryingLocationsLength).ToArray());
-
- data = data.Slice(Unsafe.SizeOf() + feedbackHeader.VaryingLocationsLength);
- }
-
- return result;
- }
-
- return null;
- }
-
- ///
- /// Create a new instance of from an gpu accessor.
- ///
- /// The gpu accessor
- /// a new instance of
- private static GuestGpuAccessorHeader CreateGuestGpuAccessorCache(IGpuAccessor gpuAccessor)
- {
- return new GuestGpuAccessorHeader
- {
- ComputeLocalSizeX = gpuAccessor.QueryComputeLocalSizeX(),
- ComputeLocalSizeY = gpuAccessor.QueryComputeLocalSizeY(),
- ComputeLocalSizeZ = gpuAccessor.QueryComputeLocalSizeZ(),
- ComputeLocalMemorySize = gpuAccessor.QueryComputeLocalMemorySize(),
- ComputeSharedMemorySize = gpuAccessor.QueryComputeSharedMemorySize(),
- PrimitiveTopology = gpuAccessor.QueryPrimitiveTopology(),
- };
- }
-
- ///
- /// Write the guest GpuAccessor informations to the given stream.
- ///
- /// The stream to write the guest GpuAcessor
- /// The shader tranlator context in use
- /// The guest gpu accessor header
- private static GuestGpuAccessorHeader WriteGuestGpuAccessorCache(Stream stream, TranslatorContext shaderContext)
- {
- BinaryWriter writer = new BinaryWriter(stream);
-
- GuestGpuAccessorHeader header = CreateGuestGpuAccessorCache(shaderContext.GpuAccessor);
-
- // If we have a full gpu accessor, cache textures descriptors
- if (shaderContext.GpuAccessor is GpuAccessor gpuAccessor)
- {
- HashSet textureHandlesInUse = shaderContext.TextureHandlesForCache;
-
- header.TextureDescriptorCount = textureHandlesInUse.Count;
-
- foreach (int textureHandle in textureHandlesInUse)
- {
- GuestTextureDescriptor textureDescriptor = ((Image.TextureDescriptor)gpuAccessor.GetTextureDescriptor(textureHandle)).ToCache();
-
- textureDescriptor.Handle = (uint)textureHandle;
-
- writer.WriteStruct(textureDescriptor);
- }
- }
-
- return header;
- }
-
- ///
- /// Get the shader program information for use on the shader cache.
- ///
- /// The current transform feedback descriptors used
- /// The shader translators context in use
- /// The resulting raw shader program code
- /// The resulting raw shader program code hash
- /// The resulting guest shader entries header
- private void GetProgramInformations(TransformFeedbackDescriptor[] tfd, ReadOnlySpan shaderContexts, out byte[] programCode, out Hash128 programCodeHash, out GuestShaderCacheEntryHeader[] entries)
- {
- GuestShaderCacheEntryHeader ComputeStage(Stream stream, TranslatorContext context)
- {
- if (context == null)
- {
- return new GuestShaderCacheEntryHeader();
- }
-
- ReadOnlySpan data = _context.MemoryManager.GetSpan(context.Address, context.Size);
-
- stream.Write(data);
-
- int size = data.Length;
- int sizeA = 0;
-
- if (context.AddressA != 0)
- {
- data = _context.MemoryManager.GetSpan(context.AddressA, context.SizeA);
-
- sizeA = data.Length;
-
- stream.Write(data);
- }
-
- GuestGpuAccessorHeader gpuAccessorHeader = WriteGuestGpuAccessorCache(stream, context);
-
- return new GuestShaderCacheEntryHeader(context.Stage, size, sizeA, gpuAccessorHeader);
- }
-
- entries = new GuestShaderCacheEntryHeader[shaderContexts.Length];
-
- using (MemoryStream stream = new MemoryStream())
- {
- for (int i = 0; i < shaderContexts.Length; i++)
- {
- entries[i] = ComputeStage(stream, shaderContexts[i]);
- }
-
- WriteTransformationFeedbackInformation(stream, tfd);
-
- programCode = stream.ToArray();
- programCodeHash = _cacheManager.ComputeHash(programCode);
- }
- }
}
}
\ No newline at end of file