Download from Steam/GOG

src/NexusMods.Abstractions.GOG/IClient.cs

@@ -34,5 +34,5 @@ public interface IClient
     /// Given a depot, a build, and a path, return a stream to the file. This file is seekable, and will cache and
     /// stream in data as required from the CDN.
     /// </summary>
-    public Task<Stream> GetFileStream(Build build, DepotInfo depotInfo, RelativePath path, CancellationToken token);
+    public Task<Stream> GetFileStream(ProductId productId, DepotInfo depotInfo, RelativePath path, CancellationToken token);
 }

src/NexusMods.Abstractions.Games.FileHashes/Attributes/Steam/AppIdsAttribute.cs

@@ -0,0 +1,22 @@
+using NexusMods.Abstractions.Steam.Values;
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
+
+namespace NexusMods.Abstractions.Games.FileHashes.Attributes.Steam;
+
+/// <summary>
+/// An attribute for a Steam App ID.
+/// </summary>
+public class AppIdsAttribute(string ns, string name) : CollectionAttribute<AppId, uint, UInt32Serializer>(ns, name) 
+{
+    protected override uint ToLowLevel(AppId value)
+    {
+        return value.Value;
+    }
+
+    protected override AppId FromLowLevel(uint value, AttributeResolver resolver)
+    {
+        return AppId.From(value);
+    }
+}

src/NexusMods.Abstractions.Games.FileHashes/Attributes/Steam/PackageIdAttribute.cs

@@ -0,0 +1,16 @@
+using NexusMods.Abstractions.Steam.Values;
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
+
+namespace NexusMods.Abstractions.Games.FileHashes.Attributes.Steam;
+
+/// <summary>
+/// An attribute for a Steam App ID.
+/// </summary>
+public class PackageIdAttribute(string ns, string name) : ScalarAttribute<PackageId, uint, UInt32Serializer>(ns, name) 
+{
+    protected override uint ToLowLevel(PackageId value) => value.Value;
+
+    protected override PackageId FromLowLevel(uint value, AttributeResolver resolver) => PackageId.From(value);
+}

src/NexusMods.Abstractions.Games.FileHashes/IFileHashesService.cs

@@ -3,6 +3,7 @@
 using NexusMods.Abstractions.GameLocators;
 using NexusMods.Abstractions.Games.FileHashes.Models;
 using NexusMods.Abstractions.NexusWebApi.Types.V2;
+using NexusMods.Cascade;
 using NexusMods.Hashing.xxHash3;
 using NexusMods.MnemonicDB.Abstractions;
 

src/NexusMods.Abstractions.Games.FileHashes/Queries.cs

@@ -0,0 +1,45 @@
+using NexusMods.Abstractions.Games.FileHashes.Models;
+using NexusMods.Abstractions.GOG.Values;
+using NexusMods.Abstractions.Steam.Values;
+using NexusMods.Cascade;
+using NexusMods.Cascade.Patterns;
+using NexusMods.Hashing.xxHash3;
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.Cascade;
+using NexusMods.Paths;
+using NexusMods.Sdk.Hashes;
+
+namespace NexusMods.Abstractions.Games.FileHashes;
+
+public static class Queries
+{
+    /// <summary>
+    /// The currently loaded file hashes database.
+    /// </summary>
+    public static readonly Inlet<IDb> Db = new();
+
+    /// <summary>
+    /// A flow of all the hashes for a given steam app ID.
+    /// </summary>
+    public static readonly Flow<(AppId AppId, EntityId Manifest, Hash Hash)> HashesForAppId =
+        Pattern.Create()
+            .Db(Db, out var manifest, SteamManifest.AppId, out var appId)
+            .Db(Db, manifest, SteamManifest.Files, out var file)
+            .Db(Db, file, PathHashRelation.Hash, out var hashRelation)
+            .Db(Db, hashRelation, HashRelation.XxHash3, out var xxHash3)
+            .Return(appId, manifest, xxHash3);
+
+    /// <summary>
+    /// A flow of all the hashes for a given gog product ID.
+    /// </summary>
+    public static readonly Flow<(ProductId ProductId, BuildId BuildId, RelativePath Path, Hash Hash)> HashesForProductId =
+        Pattern.Create()
+            .Db(Db, out var manifest, GogBuild.ProductId, out var productId)
+            .Db(Db, manifest, GogBuild.BuildId, out var buildId)
+            .Db(Db, manifest, GogBuild.Files, out var file)
+            .Db(Db, file, PathHashRelation.Hash, out var hashRelation)
+            .Db(Db, file, PathHashRelation.Path, out var path)
+            .Db(Db, hashRelation, HashRelation.XxHash3, out var xxHash3)
+            .Return(productId, buildId, path, xxHash3);
+
+}

src/NexusMods.Abstractions.Steam/NexusMods.Abstractions.Steam.csproj

@@ -2,11 +2,16 @@
     <Import Project="$([MSBuild]::GetPathOfFileAbove('NuGet.Build.props', '$(MSBuildThisFileDirectory)../'))" />
 
     <ItemGroup>
+        <PackageReference Include="NexusMods.MnemonicDB.Abstractions" />
         <PackageReference Include="NexusMods.Paths" />
         <PackageReference Include="TransparentValueObjects" PrivateAssets="all" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
     </ItemGroup>
 
     <ItemGroup>
       <ProjectReference Include="..\NexusMods.Sdk\NexusMods.Sdk.csproj" />
     </ItemGroup>
+
+    <ItemGroup>
+      <Folder Include="Models\" />
+    </ItemGroup>
 </Project>

src/NexusMods.Abstractions.Steam/Values/PackageId.cs

@@ -0,0 +1,12 @@
+using TransparentValueObjects;
+
+namespace NexusMods.Abstractions.Steam.Values;
+
+/// <summary>
+/// A globally unique identifier for an application on Steam.
+/// </summary>
+[ValueObject<uint>]
+public readonly partial struct PackageId : IAugmentWith<JsonAugment>
+{
+
+}

src/NexusMods.DataModel/NexusMods.DataModel.csproj

@@ -31,6 +31,7 @@
         <PackageReference Include="NexusMods.Cascade" />
         <PackageReference Include="NexusMods.Cascade.SourceGenerator" />
         <PackageReference Include="NexusMods.MnemonicDB" />
+        <PackageReference Include="NexusMods.MnemonicDB.Abstractions" />
         <PackageReference Include="Polly.Core"/>
         <PackageReference Include="Microsoft.Extensions.Http.Resilience"/>
         <PackageReference Include="TransparentValueObjects" PrivateAssets="all" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />

src/NexusMods.DataModel/NxFileStore.cs

@@ -19,6 +19,7 @@
 using NexusMods.Sdk.Hashes;
 using NexusMods.Sdk.Threading;
 using System.Diagnostics;
+using NexusMods.Abstractions.IO;
 using NexusMods.Sdk.FileStore;
 using NexusMods.Sdk.IO;
 
@@ -33,6 +34,7 @@ public class NxFileStore : IFileStore
     private readonly AbsolutePath[] _archiveLocations;
     private readonly IConnection _conn;
     private readonly ILogger<NxFileStore> _logger;
+    private readonly IReadOnlyFileStore[] _alternativeStores;
 
     /// <summary>
     /// Constructor
@@ -41,10 +43,13 @@ public NxFileStore(
         ILogger<NxFileStore> logger,
         IConnection conn,
         ISettingsManager settingsManager,
-        IFileSystem fileSystem)
+        IFileSystem fileSystem,
+        IEnumerable<IReadOnlyFileStore>? alternativeFileStore = null)
     {
         var settings = settingsManager.Get<DataModelSettings>();
 
+        _alternativeStores = alternativeFileStore?.ToArray() ?? [];
+
         _archiveLocations = settings.ArchiveLocations.Select(f => f.ToPath(fileSystem)).ToArray();
         foreach (var location in _archiveLocations)
         {
@@ -57,12 +62,21 @@ public NxFileStore(
     }
 
     /// <inheritdoc />
-    public ValueTask<bool> HaveFile(Hash hash)
+    public async ValueTask<bool> HaveFile(Hash hash)
     {
         using var lck = _lock.ReadLock();
         var db = _conn.Db;
         var archivedFiles = ArchivedFile.FindByHash(db, hash).Any(x => x.IsValid());
-        return ValueTask.FromResult(archivedFiles);
+        if (archivedFiles)
+            return archivedFiles;
+
+        foreach (var alternativeStore in _alternativeStores)
+        {
+            if (await alternativeStore.HaveFile(hash))
+                return true;
+        }
+
+        return false;
     }
 
     /// <inheritdoc />
@@ -150,7 +164,7 @@ public async Task ExtractFiles(IEnumerable<(Hash Hash, AbsolutePath Dest)> files
         // Group the files by archive.
         // In almost all cases, everything will go in one archive, except for cases
         // of duplicate files between different mods.
-        var groupedFiles = new ConcurrentDictionary<AbsolutePath, List<(Hash Hash, FileEntry FileEntry, AbsolutePath Dest)>>(Environment.ProcessorCount, 1);
+        var groupedFiles = new ConcurrentDictionary<(AbsolutePath? Path, IReadOnlyFileStore? Store), List<(Hash Hash, FileEntry FileEntry, AbsolutePath Dest)>>(Environment.ProcessorCount, 1);
         var createdDirectories = new ConcurrentDictionary<AbsolutePath, byte>();
 
 #if DEBUG
@@ -159,63 +173,117 @@ public async Task ExtractFiles(IEnumerable<(Hash Hash, AbsolutePath Dest)> files
 
         // Capacity is set to 'expected archive count' + 1.
         var fileExistsCache = new ConcurrentDictionary<AbsolutePath, bool>(Environment.ProcessorCount, 2);
-        Parallel.ForEach(files, file =>
+        await Parallel.ForEachAsync(files, token, async (file, _) =>
         {
             if (TryGetLocation(_conn.Db, file.Hash, fileExistsCache,
                     out var archivePath, out var fileEntry))
             {
-                var group = groupedFiles.GetOrAdd(archivePath, _ => new List<(Hash, FileEntry, AbsolutePath)>());
+                var group = groupedFiles.GetOrAdd((archivePath, null), _ => []);
                 lock (group)
                 {
                     group.Add((file.Hash, fileEntry, file.Dest));
                 }
 
-                // Create the directory, this will speed up extraction in Nx
-                // down the road. Usually the difference is negligible, but in
-                // extra special with 100s of directories scenarios, it can
-                // save a second or two.
-                var containingDir = file.Dest.Parent;
-                if (createdDirectories.TryAdd(containingDir, 0))
-                    containingDir.CreateDirectory();
+                CreateDirectoryIfNeeded(file);
 #if DEBUG
                 Debug.Assert(destPaths.TryAdd(file.Dest, 0), $"Duplicate destination path: {file.Dest}. Should not happen.");
 #endif
             }
             else
             {
+                foreach (var alternativeStore in _alternativeStores)
+                {
+                    if (await alternativeStore.HaveFile(file.Hash))
+                    {
+                        var group = groupedFiles.GetOrAdd((null, alternativeStore), _ => []);
+                        lock (group)
+                        {
+                            group.Add((file.Hash, fileEntry, file.Dest));
+                        }
+                        CreateDirectoryIfNeeded(file);
+                        return;
+                    }
+                }
                 throw new FileNotFoundException($"Missing archive for {file.Hash.ToHex()}");
             }
         });
 
         // Extract from all source archives.
         foreach (var group in groupedFiles)
         {
-            await using var file = group.Key.Read();
-            var provider = new FromStreamProvider(file);
-            var unpacker = new NxUnpacker(provider);
-
-            // Make all output providers.
-            var toExtract = GC.AllocateUninitializedArray<IOutputDataProvider>(group.Value.Count);
-            Parallel.For(0, group.Value.Count, x =>
+            if (group.Key.Store == null)
             {
-                var entry = group.Value[x];
-                toExtract[x] = new OutputFileProvider(entry.Dest.Parent.GetFullPath(), entry.Dest.FileName, entry.FileEntry);
-            });
+                await using var file = group.Key.Path!.Value.Read();
+                var provider = new FromStreamProvider(file);
+                var unpacker = new NxUnpacker(provider);
+
+                // Make all output providers.
+                var toExtract = GC.AllocateUninitializedArray<IOutputDataProvider>(group.Value.Count);
+                Parallel.For(0, group.Value.Count, x =>
+                    {
+                        var entry = group.Value[x];
+                        toExtract[x] = new OutputFileProvider(entry.Dest.Parent.GetFullPath(), entry.Dest.FileName, entry.FileEntry);
+                    }
+                );
+
+                try
+                {
+                    unpacker.ExtractFiles(toExtract, new UnpackerSettings());
+                }
+                catch (Exception e)
+                {
+                    _logger.LogError(e, "Failed to extract files from {Path}", group.Key.Path);
+                    foreach (var entry in group.Value)
+                    {
+                        if (entry.Dest.FileExists)
+                            entry.Dest.Delete();
+                    }
+                    throw;
+                }
 
-            try
-            {
-                unpacker.ExtractFiles(toExtract, new UnpackerSettings());
+                foreach (var toDispose in toExtract)
+                {
+                    toDispose.Dispose();
+                }
             }
-            catch (Exception e)
+            else
             {
-                Console.WriteLine(e);
-                throw;
+                var store = group.Key.Store;
+                await Parallel.ForEachAsync(group.Value, token, async (entry, _) =>
+                    {
+                        try
+                        {
+                            // If we have an alternative store, use it.
+                            var stream = await store.GetFileStream(entry.Hash, token);
+                            if (stream == null)
+                                throw new FileNotFoundException($"Missing file {entry.Hash.ToHex()} in alternative store");
+
+                            // Write the file to disk.
+                            await using var fs = entry.Dest.Create();
+                            await stream.CopyToAsync(fs, token);
+                        }
+                        catch (Exception e)
+                        {
+                            _logger.LogError(e, "Failed to extract file {Hash} to {Dest}", entry.Hash.ToHex(), entry.Dest);
+                            // Delete the destination file if it exists, to avoid leaving a partial file.
+                            if (entry.Dest.FileExists) 
+                                entry.Dest.Delete();
+                            throw;
+                        }
+                    }
+                );
             }
+        }
 
-            foreach (var toDispose in toExtract)
-            {
-                toDispose.Dispose();
-            }
+        void CreateDirectoryIfNeeded((Hash Hash, AbsolutePath Dest) file)
+        {
+            // Create the directory, this will speed up extraction in Nx
+            // down the road. Usually the difference is negligible, but in
+            // extra special with 100s of directories scenarios, it can
+            // save a second or two.
+            var containingDir = file.Dest.Parent;
+            if (createdDirectories.TryAdd(containingDir, 0))
+                containingDir.CreateDirectory();
         }
     }
 
@@ -285,23 +353,34 @@ public Task<Dictionary<Hash, byte[]>> ExtractFiles(IEnumerable<Hash> files, Canc
     }
 
     /// <inheritdoc />
-    public Task<Stream> GetFileStream(Hash hash, CancellationToken token = default)
+    public async Task<Stream> GetFileStream(Hash hash, CancellationToken token = default)
     {
         if (hash == Hash.Zero)
             throw new ArgumentNullException(nameof(hash));
 
         using var lck = _lock.ReadLock();
         if (!TryGetLocation(_conn.Db, hash, null,
-                out var archivePath, out var entry))
+                out var archivePath, out var entry
+            ))
+        {
+            foreach (var alternativeStore in _alternativeStores)
+            {
+                var stream = await alternativeStore.GetFileStream(hash, token);
+                if (stream != null)
+                {
+                    // If we found a stream in an alternative store, return it.
+                    return stream;
+                }
+            }
             throw new Exception($"Missing archive for {hash.ToHex()}");
+        }
 
         var file = archivePath.Read();
 
         var provider = new FromStreamProvider(file);
         var header = HeaderParser.ParseHeader(provider);
 
-        return Task.FromResult<Stream>(
-            new ChunkedStream<ChunkedArchiveStream>(new ChunkedArchiveStream(entry, header, file)));
+        return new ChunkedStream<ChunkedArchiveStream>(new ChunkedArchiveStream(entry, header, file));
     }
 
     public Task<byte[]> Load(Hash hash, CancellationToken token = default)