diff --git a/src/NexusMods.Abstractions.GOG/IClient.cs b/src/NexusMods.Abstractions.GOG/IClient.cs index 50c412ff05..1568ba3fce 100644 --- a/src/NexusMods.Abstractions.GOG/IClient.cs +++ b/src/NexusMods.Abstractions.GOG/IClient.cs @@ -34,5 +34,5 @@ public interface IClient /// Given a depot, a build, and a path, return a stream to the file. This file is seekable, and will cache and /// stream in data as required from the CDN. /// - public Task GetFileStream(Build build, DepotInfo depotInfo, RelativePath path, CancellationToken token); + public Task GetFileStream(ProductId productId, DepotInfo depotInfo, RelativePath path, CancellationToken token); } diff --git a/src/NexusMods.Abstractions.Games.FileHashes/Attributes/Steam/AppIdsAttribute.cs b/src/NexusMods.Abstractions.Games.FileHashes/Attributes/Steam/AppIdsAttribute.cs new file mode 100644 index 0000000000..0338971a24 --- /dev/null +++ b/src/NexusMods.Abstractions.Games.FileHashes/Attributes/Steam/AppIdsAttribute.cs @@ -0,0 +1,22 @@ +using NexusMods.Abstractions.Steam.Values; +using NexusMods.MnemonicDB.Abstractions; +using NexusMods.MnemonicDB.Abstractions.Attributes; +using NexusMods.MnemonicDB.Abstractions.ValueSerializers; + +namespace NexusMods.Abstractions.Games.FileHashes.Attributes.Steam; + +/// +/// An attribute for a Steam App ID. +/// +public class AppIdsAttribute(string ns, string name) : CollectionAttribute(ns, name) +{ + protected override uint ToLowLevel(AppId value) + { + return value.Value; + } + + protected override AppId FromLowLevel(uint value, AttributeResolver resolver) + { + return AppId.From(value); + } +} diff --git a/src/NexusMods.Abstractions.Games.FileHashes/Attributes/Steam/PackageIdAttribute.cs b/src/NexusMods.Abstractions.Games.FileHashes/Attributes/Steam/PackageIdAttribute.cs new file mode 100644 index 0000000000..89751fe1d3 --- /dev/null +++ b/src/NexusMods.Abstractions.Games.FileHashes/Attributes/Steam/PackageIdAttribute.cs @@ -0,0 +1,16 @@ +using NexusMods.Abstractions.Steam.Values; +using NexusMods.MnemonicDB.Abstractions; +using NexusMods.MnemonicDB.Abstractions.Attributes; +using NexusMods.MnemonicDB.Abstractions.ValueSerializers; + +namespace NexusMods.Abstractions.Games.FileHashes.Attributes.Steam; + +/// +/// An attribute for a Steam App ID. +/// +public class PackageIdAttribute(string ns, string name) : ScalarAttribute(ns, name) +{ + protected override uint ToLowLevel(PackageId value) => value.Value; + + protected override PackageId FromLowLevel(uint value, AttributeResolver resolver) => PackageId.From(value); +} diff --git a/src/NexusMods.Abstractions.Games.FileHashes/IFileHashesService.cs b/src/NexusMods.Abstractions.Games.FileHashes/IFileHashesService.cs index 782f17c46f..3e81e711ee 100644 --- a/src/NexusMods.Abstractions.Games.FileHashes/IFileHashesService.cs +++ b/src/NexusMods.Abstractions.Games.FileHashes/IFileHashesService.cs @@ -3,6 +3,7 @@ using NexusMods.Abstractions.GameLocators; using NexusMods.Abstractions.Games.FileHashes.Models; using NexusMods.Abstractions.NexusWebApi.Types.V2; +using NexusMods.Cascade; using NexusMods.Hashing.xxHash3; using NexusMods.MnemonicDB.Abstractions; diff --git a/src/NexusMods.Abstractions.Games.FileHashes/Queries.cs b/src/NexusMods.Abstractions.Games.FileHashes/Queries.cs new file mode 100644 index 0000000000..c44262d229 --- /dev/null +++ b/src/NexusMods.Abstractions.Games.FileHashes/Queries.cs @@ -0,0 +1,45 @@ +using NexusMods.Abstractions.Games.FileHashes.Models; +using NexusMods.Abstractions.GOG.Values; +using NexusMods.Abstractions.Steam.Values; +using NexusMods.Cascade; +using NexusMods.Cascade.Patterns; +using NexusMods.Hashing.xxHash3; +using NexusMods.MnemonicDB.Abstractions; +using NexusMods.MnemonicDB.Abstractions.Cascade; +using NexusMods.Paths; +using NexusMods.Sdk.Hashes; + +namespace NexusMods.Abstractions.Games.FileHashes; + +public static class Queries +{ + /// + /// The currently loaded file hashes database. + /// + public static readonly Inlet Db = new(); + + /// + /// A flow of all the hashes for a given steam app ID. + /// + public static readonly Flow<(AppId AppId, EntityId Manifest, Hash Hash)> HashesForAppId = + Pattern.Create() + .Db(Db, out var manifest, SteamManifest.AppId, out var appId) + .Db(Db, manifest, SteamManifest.Files, out var file) + .Db(Db, file, PathHashRelation.Hash, out var hashRelation) + .Db(Db, hashRelation, HashRelation.XxHash3, out var xxHash3) + .Return(appId, manifest, xxHash3); + + /// + /// A flow of all the hashes for a given gog product ID. + /// + public static readonly Flow<(ProductId ProductId, BuildId BuildId, RelativePath Path, Hash Hash)> HashesForProductId = + Pattern.Create() + .Db(Db, out var manifest, GogBuild.ProductId, out var productId) + .Db(Db, manifest, GogBuild.BuildId, out var buildId) + .Db(Db, manifest, GogBuild.Files, out var file) + .Db(Db, file, PathHashRelation.Hash, out var hashRelation) + .Db(Db, file, PathHashRelation.Path, out var path) + .Db(Db, hashRelation, HashRelation.XxHash3, out var xxHash3) + .Return(productId, buildId, path, xxHash3); + +} diff --git a/src/NexusMods.Abstractions.Steam/NexusMods.Abstractions.Steam.csproj b/src/NexusMods.Abstractions.Steam/NexusMods.Abstractions.Steam.csproj index 69f9237b09..c3edd7925e 100644 --- a/src/NexusMods.Abstractions.Steam/NexusMods.Abstractions.Steam.csproj +++ b/src/NexusMods.Abstractions.Steam/NexusMods.Abstractions.Steam.csproj @@ -2,6 +2,7 @@ + @@ -9,4 +10,8 @@ + + + + diff --git a/src/NexusMods.Abstractions.Steam/Values/PackageId.cs b/src/NexusMods.Abstractions.Steam/Values/PackageId.cs new file mode 100644 index 0000000000..72c171a98a --- /dev/null +++ b/src/NexusMods.Abstractions.Steam/Values/PackageId.cs @@ -0,0 +1,12 @@ +using TransparentValueObjects; + +namespace NexusMods.Abstractions.Steam.Values; + +/// +/// A globally unique identifier for an application on Steam. +/// +[ValueObject] +public readonly partial struct PackageId : IAugmentWith +{ + +} diff --git a/src/NexusMods.DataModel/NexusMods.DataModel.csproj b/src/NexusMods.DataModel/NexusMods.DataModel.csproj index 8c7ca28426..33445efd85 100644 --- a/src/NexusMods.DataModel/NexusMods.DataModel.csproj +++ b/src/NexusMods.DataModel/NexusMods.DataModel.csproj @@ -31,6 +31,7 @@ + diff --git a/src/NexusMods.DataModel/NxFileStore.cs b/src/NexusMods.DataModel/NxFileStore.cs index 5f566b07e5..549a7f1e44 100644 --- a/src/NexusMods.DataModel/NxFileStore.cs +++ b/src/NexusMods.DataModel/NxFileStore.cs @@ -19,6 +19,7 @@ using NexusMods.Sdk.Hashes; using NexusMods.Sdk.Threading; using System.Diagnostics; +using NexusMods.Abstractions.IO; using NexusMods.Sdk.FileStore; using NexusMods.Sdk.IO; @@ -33,6 +34,7 @@ public class NxFileStore : IFileStore private readonly AbsolutePath[] _archiveLocations; private readonly IConnection _conn; private readonly ILogger _logger; + private readonly IReadOnlyFileStore[] _alternativeStores; /// /// Constructor @@ -41,10 +43,13 @@ public NxFileStore( ILogger logger, IConnection conn, ISettingsManager settingsManager, - IFileSystem fileSystem) + IFileSystem fileSystem, + IEnumerable? alternativeFileStore = null) { var settings = settingsManager.Get(); + _alternativeStores = alternativeFileStore?.ToArray() ?? []; + _archiveLocations = settings.ArchiveLocations.Select(f => f.ToPath(fileSystem)).ToArray(); foreach (var location in _archiveLocations) { @@ -57,12 +62,21 @@ public NxFileStore( } /// - public ValueTask HaveFile(Hash hash) + public async ValueTask HaveFile(Hash hash) { using var lck = _lock.ReadLock(); var db = _conn.Db; var archivedFiles = ArchivedFile.FindByHash(db, hash).Any(x => x.IsValid()); - return ValueTask.FromResult(archivedFiles); + if (archivedFiles) + return archivedFiles; + + foreach (var alternativeStore in _alternativeStores) + { + if (await alternativeStore.HaveFile(hash)) + return true; + } + + return false; } /// @@ -150,7 +164,7 @@ public async Task ExtractFiles(IEnumerable<(Hash Hash, AbsolutePath Dest)> files // Group the files by archive. // In almost all cases, everything will go in one archive, except for cases // of duplicate files between different mods. - var groupedFiles = new ConcurrentDictionary>(Environment.ProcessorCount, 1); + var groupedFiles = new ConcurrentDictionary<(AbsolutePath? Path, IReadOnlyFileStore? Store), List<(Hash Hash, FileEntry FileEntry, AbsolutePath Dest)>>(Environment.ProcessorCount, 1); var createdDirectories = new ConcurrentDictionary(); #if DEBUG @@ -159,30 +173,37 @@ public async Task ExtractFiles(IEnumerable<(Hash Hash, AbsolutePath Dest)> files // Capacity is set to 'expected archive count' + 1. var fileExistsCache = new ConcurrentDictionary(Environment.ProcessorCount, 2); - Parallel.ForEach(files, file => + await Parallel.ForEachAsync(files, token, async (file, _) => { if (TryGetLocation(_conn.Db, file.Hash, fileExistsCache, out var archivePath, out var fileEntry)) { - var group = groupedFiles.GetOrAdd(archivePath, _ => new List<(Hash, FileEntry, AbsolutePath)>()); + var group = groupedFiles.GetOrAdd((archivePath, null), _ => []); lock (group) { group.Add((file.Hash, fileEntry, file.Dest)); } - // Create the directory, this will speed up extraction in Nx - // down the road. Usually the difference is negligible, but in - // extra special with 100s of directories scenarios, it can - // save a second or two. - var containingDir = file.Dest.Parent; - if (createdDirectories.TryAdd(containingDir, 0)) - containingDir.CreateDirectory(); + CreateDirectoryIfNeeded(file); #if DEBUG Debug.Assert(destPaths.TryAdd(file.Dest, 0), $"Duplicate destination path: {file.Dest}. Should not happen."); #endif } else { + foreach (var alternativeStore in _alternativeStores) + { + if (await alternativeStore.HaveFile(file.Hash)) + { + var group = groupedFiles.GetOrAdd((null, alternativeStore), _ => []); + lock (group) + { + group.Add((file.Hash, fileEntry, file.Dest)); + } + CreateDirectoryIfNeeded(file); + return; + } + } throw new FileNotFoundException($"Missing archive for {file.Hash.ToHex()}"); } }); @@ -190,32 +211,79 @@ public async Task ExtractFiles(IEnumerable<(Hash Hash, AbsolutePath Dest)> files // Extract from all source archives. foreach (var group in groupedFiles) { - await using var file = group.Key.Read(); - var provider = new FromStreamProvider(file); - var unpacker = new NxUnpacker(provider); - - // Make all output providers. - var toExtract = GC.AllocateUninitializedArray(group.Value.Count); - Parallel.For(0, group.Value.Count, x => + if (group.Key.Store == null) { - var entry = group.Value[x]; - toExtract[x] = new OutputFileProvider(entry.Dest.Parent.GetFullPath(), entry.Dest.FileName, entry.FileEntry); - }); + await using var file = group.Key.Path!.Value.Read(); + var provider = new FromStreamProvider(file); + var unpacker = new NxUnpacker(provider); + + // Make all output providers. + var toExtract = GC.AllocateUninitializedArray(group.Value.Count); + Parallel.For(0, group.Value.Count, x => + { + var entry = group.Value[x]; + toExtract[x] = new OutputFileProvider(entry.Dest.Parent.GetFullPath(), entry.Dest.FileName, entry.FileEntry); + } + ); + + try + { + unpacker.ExtractFiles(toExtract, new UnpackerSettings()); + } + catch (Exception e) + { + _logger.LogError(e, "Failed to extract files from {Path}", group.Key.Path); + foreach (var entry in group.Value) + { + if (entry.Dest.FileExists) + entry.Dest.Delete(); + } + throw; + } - try - { - unpacker.ExtractFiles(toExtract, new UnpackerSettings()); + foreach (var toDispose in toExtract) + { + toDispose.Dispose(); + } } - catch (Exception e) + else { - Console.WriteLine(e); - throw; + var store = group.Key.Store; + await Parallel.ForEachAsync(group.Value, token, async (entry, _) => + { + try + { + // If we have an alternative store, use it. + var stream = await store.GetFileStream(entry.Hash, token); + if (stream == null) + throw new FileNotFoundException($"Missing file {entry.Hash.ToHex()} in alternative store"); + + // Write the file to disk. + await using var fs = entry.Dest.Create(); + await stream.CopyToAsync(fs, token); + } + catch (Exception e) + { + _logger.LogError(e, "Failed to extract file {Hash} to {Dest}", entry.Hash.ToHex(), entry.Dest); + // Delete the destination file if it exists, to avoid leaving a partial file. + if (entry.Dest.FileExists) + entry.Dest.Delete(); + throw; + } + } + ); } + } - foreach (var toDispose in toExtract) - { - toDispose.Dispose(); - } + void CreateDirectoryIfNeeded((Hash Hash, AbsolutePath Dest) file) + { + // Create the directory, this will speed up extraction in Nx + // down the road. Usually the difference is negligible, but in + // extra special with 100s of directories scenarios, it can + // save a second or two. + var containingDir = file.Dest.Parent; + if (createdDirectories.TryAdd(containingDir, 0)) + containingDir.CreateDirectory(); } } @@ -285,23 +353,34 @@ public Task> ExtractFiles(IEnumerable files, Canc } /// - public Task GetFileStream(Hash hash, CancellationToken token = default) + public async Task GetFileStream(Hash hash, CancellationToken token = default) { if (hash == Hash.Zero) throw new ArgumentNullException(nameof(hash)); using var lck = _lock.ReadLock(); if (!TryGetLocation(_conn.Db, hash, null, - out var archivePath, out var entry)) + out var archivePath, out var entry + )) + { + foreach (var alternativeStore in _alternativeStores) + { + var stream = await alternativeStore.GetFileStream(hash, token); + if (stream != null) + { + // If we found a stream in an alternative store, return it. + return stream; + } + } throw new Exception($"Missing archive for {hash.ToHex()}"); + } var file = archivePath.Read(); var provider = new FromStreamProvider(file); var header = HeaderParser.ParseHeader(provider); - return Task.FromResult( - new ChunkedStream(new ChunkedArchiveStream(entry, header, file))); + return new ChunkedStream(new ChunkedArchiveStream(entry, header, file)); } public Task Load(Hash hash, CancellationToken token = default) diff --git a/src/NexusMods.Games.FileHashes/FileHashesService.cs b/src/NexusMods.Games.FileHashes/FileHashesService.cs index b5a3e9e009..8e2b63f41b 100644 --- a/src/NexusMods.Games.FileHashes/FileHashesService.cs +++ b/src/NexusMods.Games.FileHashes/FileHashesService.cs @@ -12,6 +12,7 @@ using NexusMods.Abstractions.NexusWebApi.Types.V2; using NexusMods.Abstractions.Settings; using NexusMods.Abstractions.Steam.Values; +using NexusMods.Cascade; using NexusMods.Games.FileHashes.DTOs; using NexusMods.Hashing.xxHash3; using NexusMods.MnemonicDB; @@ -40,6 +41,7 @@ internal sealed class FileHashesService : IFileHashesService, IDisposable private ConnectedDb? _currentDb; private readonly ILogger _logger; + private readonly InletNode _globalInlet; private record ConnectedDb(IDb Db, Connection Connection, DatomStore Store, MnemonicDB.Storage.RocksDbBackend.Backend Backend, DateTimeOffset Timestamp, AbsolutePath Path); @@ -53,6 +55,9 @@ public FileHashesService(ILogger logger, ISettingsManager set _databases = new Dictionary(); _provider = provider; + var globalConnection = _provider.GetRequiredService(); + _globalInlet = globalConnection.Topology.Intern(Queries.Db); + _settings.HashDatabaseLocation.ToPath(_fileSystem).CreateDirectory(); // Delete any old databases that are not the latest @@ -68,6 +73,7 @@ public FileHashesService(ILogger logger, ISettingsManager set if (latest.Path != default(AbsolutePath)) { _currentDb = OpenDb(latest.PublishTime, latest.Path); + _globalInlet.Values = [_currentDb.Db]; } // Trigger an update @@ -163,6 +169,7 @@ private async Task CheckForUpdateCore(bool forceUpdate) if (latest.Path != default(AbsolutePath)) { _currentDb = OpenDb(latest.PublishTime, latest.Path); + _globalInlet.Values = [_currentDb.Db]; return; } } @@ -211,6 +218,7 @@ private async Task CheckForUpdateCore(bool forceUpdate) // open the new database _currentDb = OpenDb(release.CreatedAt, finalDir); + _globalInlet.Values = [_currentDb.Db]; } private AbsolutePath GameHashesReleaseFileName => _settings.HashDatabaseLocation.ToPath(_fileSystem) / _settings.ReleaseFilePath; diff --git a/src/NexusMods.Networking.GOG/CLI/Verbs.cs b/src/NexusMods.Networking.GOG/CLI/Verbs.cs index 7e8876d82b..4b23457d44 100644 --- a/src/NexusMods.Networking.GOG/CLI/Verbs.cs +++ b/src/NexusMods.Networking.GOG/CLI/Verbs.cs @@ -96,7 +96,7 @@ await Parallel.ForEachAsync(depot.Items, token, async (item, token) => var fullSize = item.Chunks.Sum(s => (double)s.Size.Value); { await using var task = await renderer.StartProgressTask($"Hashing {item.Path}", maxValue: fullSize); - await using var stream = await client.GetFileStream(build, depot, item.Path, + await using var stream = await client.GetFileStream(build.ProductId, depot, item.Path, token ); var multiHasher = new MultiHasher(); @@ -129,7 +129,7 @@ await JsonSerializer.SerializeAsync(outputStream, multiHash, indentedOptions, if (verify) { var offset = Size.Zero; - await using var stream = await client.GetFileStream(build, depot, item.Path, + await using var stream = await client.GetFileStream(build.ProductId, depot, item.Path, token ); await foreach (var chunk in item.Chunks.WithProgress(renderer, $"Verifying {item.Path}")) diff --git a/src/NexusMods.Networking.GOG/Client.cs b/src/NexusMods.Networking.GOG/Client.cs index d3394a3897..d47d6e2032 100644 --- a/src/NexusMods.Networking.GOG/Client.cs +++ b/src/NexusMods.Networking.GOG/Client.cs @@ -154,8 +154,70 @@ public async Task Login(CancellationToken token) tx.Add(e, AuthInfo.UserId, ulong.Parse(tokenResponse.UserId)); await tx.Commit(); _logger.LogInformation("Logged in successfully to GOG."); + _ = RefreshLicenses(); + } + + private async Task RefreshLicenses() + { + _logger.LogInformation("Refreshing GOG licenses."); + var ownedGames = await GetOwnedGames(CancellationToken.None); + if (ownedGames == null) + { + return; + } + + var existingLicenses = GOGLicense.All(_connection.Db).ToDictionary(l => l.ProductId, l => l.Id); + var changes = false; + using var tx = _connection.BeginTransaction(); + + foreach (var license in ownedGames.Owned) + { + if (existingLicenses.ContainsKey(license)) + continue; + + var e = tx.TempId(); + tx.Add(e, GOGLicense.ProductId, license); + changes = true; + } + + foreach (var (existing, eid) in existingLicenses) + { + if (ownedGames.Owned.Contains(existing)) + continue; + + // If the license is not in the owned games, we can remove it + tx.Delete(eid, false); + changes = true; + } + + if (changes) + await tx.Commit(); + } + + private async Task GetOwnedGames(CancellationToken token) + { + return await _pipeline.ExecuteAsync(async token => + { + var msg = await CreateMessage(new Uri($"https://embed.gog.com/user/data/games"), CancellationToken.None); + using var response = await _client.SendAsync(msg, token); + if (!response.IsSuccessStatusCode) + { + if (response.StatusCode == System.Net.HttpStatusCode.NotFound) + return null; + else + throw new Exception($"Failed to get license list"); + } + + + await using var responseStream = await response.Content.ReadAsStreamAsync(token); + var content = await JsonSerializer.DeserializeAsync(responseStream, _jsonSerializerOptions, token); + + if (content == null) + throw new Exception("Failed to deserialize the builds response."); + + return content; + }, token); } - /// /// Create a new HttpRequestMessage with the OAuth token. @@ -329,7 +391,7 @@ public async Task GetDepot(Build build, CancellationToken token) /// /// Given a depot, a build, and a path, return a stream to the file. /// - public async Task GetFileStream(Build build, DepotInfo depotInfo, RelativePath path, CancellationToken token) + public async Task GetFileStream(ProductId productId, DepotInfo depotInfo, RelativePath path, CancellationToken token) { return await _pipeline.ExecuteAsync(async token => { @@ -337,7 +399,7 @@ public async Task GetFileStream(Build build, DepotInfo depotInfo, Relati if (itemInfo == null) throw new KeyNotFoundException($"The path {path} was not found in the depot."); - var secureUrl = await GetSecureUrl(build.ProductId, token); + var secureUrl = await GetSecureUrl(productId, token); if (itemInfo.SfcRef == null) { diff --git a/src/NexusMods.Networking.GOG/DTOs/OwnedGamesList.cs b/src/NexusMods.Networking.GOG/DTOs/OwnedGamesList.cs new file mode 100644 index 0000000000..d34700cc8a --- /dev/null +++ b/src/NexusMods.Networking.GOG/DTOs/OwnedGamesList.cs @@ -0,0 +1,13 @@ +using System.Text.Json.Serialization; +using NexusMods.Abstractions.GOG.Values; + +namespace NexusMods.Networking.GOG.DTOs; + +/// +/// Response from /user/data/games +/// +public class OwnedGamesList +{ + [JsonPropertyName("owned")] + public List Owned { get; set; } = []; +} diff --git a/src/NexusMods.Networking.GOG/Models/GOGLicenses.cs b/src/NexusMods.Networking.GOG/Models/GOGLicenses.cs new file mode 100644 index 0000000000..812821653f --- /dev/null +++ b/src/NexusMods.Networking.GOG/Models/GOGLicenses.cs @@ -0,0 +1,14 @@ +using NexusMods.Abstractions.Games.FileHashes.Attributes.Gog; +using NexusMods.MnemonicDB.Abstractions.Models; + +namespace NexusMods.Networking.GOG.Models; + +public partial class GOGLicense : IModelDefinition +{ + private const string Namespace = "NexusMods.Networking.GOG.GOGLicenses"; + + /// + /// The product ID of the GOG license. + /// + public static readonly ProductIdAttribute ProductId = new(Namespace, nameof(ProductId)); +} diff --git a/src/NexusMods.Networking.GOG/NexusMods.Networking.GOG.csproj b/src/NexusMods.Networking.GOG/NexusMods.Networking.GOG.csproj index 0d2e6e5e25..653149e11c 100644 --- a/src/NexusMods.Networking.GOG/NexusMods.Networking.GOG.csproj +++ b/src/NexusMods.Networking.GOG/NexusMods.Networking.GOG.csproj @@ -20,6 +20,7 @@ + diff --git a/src/NexusMods.Networking.GOG/Queries.cs b/src/NexusMods.Networking.GOG/Queries.cs new file mode 100644 index 0000000000..538263984c --- /dev/null +++ b/src/NexusMods.Networking.GOG/Queries.cs @@ -0,0 +1,18 @@ +using NexusMods.Abstractions.GOG.Values; +using NexusMods.Cascade; +using NexusMods.Cascade.Patterns; +using NexusMods.Hashing.xxHash3; +using NexusMods.MnemonicDB.Abstractions.Cascade; +using NexusMods.Networking.GOG.Models; +using NexusMods.Paths; + +namespace NexusMods.Networking.GOG; + +public class Queries +{ + public static readonly Flow<(ProductId ProductId, BuildId BuildId, RelativePath Path, Hash Hash)> AvailableHashes = + Pattern.Create() + .Db(out _, GOGLicense.ProductId, out var productId) + .Match(NexusMods.Abstractions.Games.FileHashes.Queries.HashesForProductId, productId, out var buildId, out var path, out var hash) + .Return(productId, buildId, path, hash); +} diff --git a/src/NexusMods.Networking.GOG/ReadOnlyFileStore.cs b/src/NexusMods.Networking.GOG/ReadOnlyFileStore.cs new file mode 100644 index 0000000000..456a5d9bd8 --- /dev/null +++ b/src/NexusMods.Networking.GOG/ReadOnlyFileStore.cs @@ -0,0 +1,46 @@ +using NexusMods.Abstractions.Games.FileHashes; +using NexusMods.Abstractions.GOG; +using NexusMods.Abstractions.GOG.Values; +using NexusMods.Abstractions.IO; +using NexusMods.Abstractions.Steam.Values; +using NexusMods.Cascade; +using NexusMods.Hashing.xxHash3; +using NexusMods.MnemonicDB.Abstractions; +using NexusMods.Paths; +using NexusMods.Sdk; + +namespace NexusMods.Networking.GOG; + +public class ReadOnlyFileStore : IReadOnlyFileStore +{ + private readonly IClient _client; + private readonly IFileHashesService _hashesService; + private readonly IQueryResult _knownHashes; + private readonly IQueryResult<(ProductId ProductId, BuildId BuildId, RelativePath Path, Hash Hash)> _availableFiles; + + public ReadOnlyFileStore(IClient client, IFileHashesService fileHashesService, IConnection connection) + { + _client = client; + _hashesService = fileHashesService; + _ = _client.Login(CancellationToken.None); + _knownHashes = connection.Topology.Query(Queries.AvailableHashes.Select(r => r.Hash)); + _availableFiles = connection.Topology.Query(Queries.AvailableHashes); + } + public ValueTask HaveFile(Hash hash) + { + return ValueTask.FromResult(_knownHashes.Contains(hash)); + } + + public async Task GetFileStream(Hash hash, CancellationToken token = default) + { + if (!_availableFiles.TryGetFirst(f => f.Hash == hash, out var row)) + return null; + + var builds = await _client.GetBuilds(row.ProductId, OS.windows, token); + if (!builds.TryGetFirst(b => b.BuildId == row.BuildId, out var buildInfo)) + return null; + + var depot = await _client.GetDepot(buildInfo, token); + return await _client.GetFileStream(row.ProductId, depot, row.Path, token); + } +} diff --git a/src/NexusMods.Networking.Steam/Models/SteamLicenses.cs b/src/NexusMods.Networking.Steam/Models/SteamLicenses.cs new file mode 100644 index 0000000000..da9a43e417 --- /dev/null +++ b/src/NexusMods.Networking.Steam/Models/SteamLicenses.cs @@ -0,0 +1,22 @@ +using NexusMods.Abstractions.Games.FileHashes.Attributes.Steam; +using NexusMods.MnemonicDB.Abstractions.Models; + +namespace NexusMods.Abstractions.Steam.Models; + +/// +/// Steam entitlements +/// +public partial class SteamLicenses : IModelDefinition +{ + private const string Namespace = "NexusMods.Networking.Steam.Models"; + + /// + /// The app id of the game this license is for. + /// + public static readonly AppIdsAttribute AppIds = new(nameof(Namespace), nameof(AppIds)) { IsIndexed = true }; + + /// + /// The package id of the game + /// + public static readonly PackageIdAttribute PackageId = new(nameof(Namespace), nameof(PackageId)) { IsIndexed = true, IsUnique = true }; +} diff --git a/src/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj b/src/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj index 15d05e8a19..42390cd96d 100644 --- a/src/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj +++ b/src/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj @@ -7,9 +7,11 @@ + + diff --git a/src/NexusMods.Networking.Steam/Queries.cs b/src/NexusMods.Networking.Steam/Queries.cs new file mode 100644 index 0000000000..1513199ef2 --- /dev/null +++ b/src/NexusMods.Networking.Steam/Queries.cs @@ -0,0 +1,22 @@ +using NexusMods.Abstractions.Steam.Models; +using NexusMods.Abstractions.Steam.Values; +using NexusMods.Cascade; +using NexusMods.Cascade.Patterns; +using NexusMods.Hashing.xxHash3; +using NexusMods.MnemonicDB.Abstractions; +using NexusMods.MnemonicDB.Abstractions.Cascade; +using NexusMods.Sdk.Hashes; + +namespace NexusMods.Networking.Steam; + +public class Queries +{ + public static readonly Flow<(AppId, EntityId, Hash)> AvailableFiles = + Pattern.Create() + .Db(out var license, SteamLicenses.AppIds, out var appId) + .Match(NexusMods.Abstractions.Games.FileHashes.Queries.HashesForAppId, appId, out var manifest, out var hash) + .Return(appId, manifest, hash); + + public static readonly Flow AvailableHashes = AvailableFiles.Select(f => f.Item3); + +} diff --git a/src/NexusMods.Networking.Steam/ReadOnlyFileStore.cs b/src/NexusMods.Networking.Steam/ReadOnlyFileStore.cs new file mode 100644 index 0000000000..c3a451d1a9 --- /dev/null +++ b/src/NexusMods.Networking.Steam/ReadOnlyFileStore.cs @@ -0,0 +1,47 @@ +using NexusMods.Abstractions.Games.FileHashes; +using NexusMods.Abstractions.Games.FileHashes.Models; +using NexusMods.Abstractions.IO; +using NexusMods.Abstractions.Steam; +using NexusMods.Abstractions.Steam.DTOs; +using NexusMods.Abstractions.Steam.Values; +using NexusMods.Cascade; +using NexusMods.Hashing.xxHash3; +using NexusMods.MnemonicDB.Abstractions; +using NexusMods.Paths; +using NexusMods.Sdk.Hashes; + +namespace NexusMods.Networking.Steam; + +public class ReadOnlyFileStore : IReadOnlyFileStore +{ + private readonly ISteamSession _session; + private readonly IFileHashesService _hashesService; + private readonly IQueryResult _knownHashes; + private readonly IQueryResult<(AppId, EntityId, Hash)> _availableFiles; + + public ReadOnlyFileStore(ISteamSession session, IFileHashesService fileHashesService, IConnection connection) + { + _session = session; + _hashesService = fileHashesService; + _ = _session.Connect(CancellationToken.None); + _knownHashes = connection.Topology.Query(Queries.AvailableHashes); + _availableFiles = connection.Topology.Query(Queries.AvailableFiles); + } + + public async ValueTask HaveFile(Hash hash) + { + return _knownHashes.Contains(hash); + } + + public async Task GetFileStream(Hash hash, CancellationToken token = default) + { + var entry = _availableFiles.First(f => f.Item3 == hash); + + var hashesDb = await _hashesService.GetFileHashesDb(); + var manifestMetadata = SteamManifest.Load(hashesDb, entry.Item2); + var file = manifestMetadata.Files.First(f => f.Hash.XxHash3 == hash); + + var info = await _session.GetManifestContents(manifestMetadata.AppId, manifestMetadata.DepotId, manifestMetadata.ManifestId, manifestMetadata.Name, token); + return _session.GetFileStream(manifestMetadata.AppId, info, file.Path); + } +} diff --git a/src/NexusMods.Networking.Steam/Services.cs b/src/NexusMods.Networking.Steam/Services.cs index 8f6369fdb6..055fb8d193 100644 --- a/src/NexusMods.Networking.Steam/Services.cs +++ b/src/NexusMods.Networking.Steam/Services.cs @@ -1,5 +1,7 @@ using Microsoft.Extensions.DependencyInjection; +using NexusMods.Abstractions.IO; using NexusMods.Abstractions.Steam; +using NexusMods.Abstractions.Steam.Models; using NexusMods.Abstractions.Steam.Values; using NexusMods.Networking.Steam.CLI; using NexusMods.Sdk.ProxyConsole; @@ -14,6 +16,7 @@ public static class Services public static IServiceCollection AddSteam(this IServiceCollection services) { services.AddSingleton(); + services.AddSteamLicensesModel(); return services; } @@ -48,6 +51,7 @@ public static IServiceCollection AddSteamCli(this IServiceCollection services) services.AddSingleton(); services.AddLocalAuthFileStorage(); services.AddSteamVerbs(); + services.AddSingleton(); return services; } diff --git a/src/NexusMods.Networking.Steam/Session.cs b/src/NexusMods.Networking.Steam/Session.cs index 48e9586d0f..385d4d42af 100644 --- a/src/NexusMods.Networking.Steam/Session.cs +++ b/src/NexusMods.Networking.Steam/Session.cs @@ -1,8 +1,12 @@ using System.Collections.Concurrent; using Microsoft.Extensions.Logging; +using NexusMods.Abstractions.GameLocators; +using NexusMods.Abstractions.GameLocators.Stores.Steam; using NexusMods.Abstractions.Steam; using NexusMods.Abstractions.Steam.DTOs; +using NexusMods.Abstractions.Steam.Models; using NexusMods.Abstractions.Steam.Values; +using NexusMods.MnemonicDB.Abstractions; using NexusMods.Networking.Steam.DTOs; using NexusMods.Paths; using NexusMods.Sdk.IO; @@ -60,17 +64,30 @@ public class Session : ISteamSession private ConcurrentDictionary<(AppId, DepotId), byte[]> _depotKeys = new(); private ConcurrentDictionary<(AppId, DepotId, ManifestId, string Branch), ulong> _manifestRequestCodes = new(); internal readonly ResiliencePipeline _pipeline; + private readonly IConnection _connection; + private readonly ISteamGame[] _steamGames; - public Session(ILogger logger, IAuthInterventionHandler handler, IAuthStorage storage) + public Session(ILogger logger, IAuthInterventionHandler handler, IAuthStorage storage, IConnection connection, IEnumerable games) { _logger = logger; + _connection = connection; _handler = handler; _authStorage = storage; + _steamGames = games.OfType().ToArray(); + var steamConfiguration = SteamConfiguration.Create(configurator => { // The client will dispose of these on its own - configurator.WithHttpClientFactory(() => new HttpClient()); + configurator.WithHttpClientFactory(() => + { + var client = new HttpClient(); + client.DefaultRequestHeaders.UserAgent.Add( + new System.Net.Http.Headers.ProductInfoHeaderValue("NexusMods.Networking.Steam", "1.0") + ); + return client; + } + ); }); _steamClient = new SteamClient(steamConfiguration); _steamUser = _steamClient.GetHandler()!; @@ -108,15 +125,78 @@ public Session(ILogger logger, IAuthInterventionHandler handler, IAuthS /// internal CDNPool CDNPool => _cdnPool; - private Task LicenseListCallback(SteamApps.LicenseListCallback arg) + private async Task LicenseListCallback(SteamApps.LicenseListCallback arg) { - return Task.CompletedTask; + _logger.LogInformation("Got {LicenseCount} licenses from Steam", arg.LicenseList.Count); + var appInfos = await GetAppIdsForPackages(arg.LicenseList.Select(r => r.PackageID)); + + var licenses = (from info in appInfos + from package in info.Packages.Values + from keyvalue in package.KeyValues.Children + where keyvalue.Name == "appids" + from appId in keyvalue.Children + let parsedAppId = AppId.From(uint.Parse(appId.Value ?? "0")) + group parsedAppId by PackageId.From(package.ID) + into grouped + select grouped) + .ToArray(); + + _logger.LogInformation("Got details {LicenseCount} licenses from Steam, caching the data", licenses.Length); + + var db = _connection.Db; + using var tx = _connection.BeginTransaction(); + bool changes = false; + foreach (var grouping in licenses) + { + var existing = SteamLicenses.FindByPackageId(db, grouping.Key).FirstOrDefault(); + if (existing.IsValid()) + { + foreach (var appId in grouping) + { + if (!existing.AppIds.Contains(appId)) + { + tx.Add(existing.Id, SteamLicenses.AppIds, appId); + changes = true; + } + } + foreach (var appId in existing.AppIds) + { + if (!grouping.Contains(appId)) + { + tx.Retract(existing.Id, SteamLicenses.AppIds, appId); + changes = true; + } + } + } + else + { + _ = new SteamLicenses.New(tx) + { + PackageId = grouping.Key, + AppIds = grouping.ToList(), + }; + changes = true; + } + } + + if (changes) + await tx.Commit(); } private async Task LoggedOnCallback(SteamUser.LoggedOnCallback callback) { + if (callback.Result != EResult.OK) + { + _logger.LogError("Failed to log on to Steam network: {Result}", callback.Result); + _isLoggedOn = false; + return; + } + _isLoggedOn = true; _logger.LogInformation("Logged on to Steam network."); + + return; + } private async Task DisconnectedCallback(SteamClient.DisconnectedCallback callback) @@ -215,6 +295,18 @@ public async Task GetProductInfoAsync(AppId appId, CancellationToke return ProductInfoParser.Parse(results[0]); } + private async Task GetAppIdsForPackages(IEnumerable packageIds, CancellationToken cancellationToken = default) + { + await ConnectedAsync(cancellationToken); + + var jobs = await _steamApps.PICSGetProductInfo([], packageIds.Select(id => new SteamApps.PICSRequest(id))); + + if (jobs.Failed) + throw new Exception("Failed to get app ids for packages"); + + return jobs.Results!.ToArray(); + } + /// /// Performs a login if required, returns once the login is complete /// @@ -240,11 +332,11 @@ public async Task GetManifestRequestCodeAsync(AppId appId, DepotId depotI var requestCodeResult = await _steamContent.GetManifestRequestCode(depotId.Value, appId.Value, manifestId.Value, branch); if (requestCodeResult == 0) { - _logger.LogWarning("Failed to get request code for depot {0} manifest {1}", depotId.Value, manifestId.Value); + _logger.LogWarning("Failed to get request code for depot {DepotId} manifest {ManifestId}", depotId.Value, manifestId.Value); throw new Exception("Failed to get request code for depot " + depotId.Value + " manifest " + manifestId.Value); } - _logger.LogInformation("Got request code depot {1} manifest {2}", depotId.Value, manifestId.Value); + _logger.LogInformation("Got request code depot {DepotId} manifest {ManifestId}", depotId.Value, manifestId.Value); _manifestRequestCodes.TryAdd((appId, depotId, manifestId, branch), requestCodeResult); return requestCodeResult; diff --git a/src/NexusMods.Sdk/IO/IReadOnlyFileStore.cs b/src/NexusMods.Sdk/IO/IReadOnlyFileStore.cs new file mode 100644 index 0000000000..b5f9b59ebf --- /dev/null +++ b/src/NexusMods.Sdk/IO/IReadOnlyFileStore.cs @@ -0,0 +1,19 @@ +using NexusMods.Hashing.xxHash3; + +namespace NexusMods.Abstractions.IO; + +/// +/// An alternative read-only source of files could be a game store (e.g. Steam, GOG) or a some mod hostig service (e.g. Nexus Mods). +/// +public interface IReadOnlyFileStore +{ + /// + /// Returns true if the file with the given hash exists in the store. + /// + ValueTask HaveFile(Hash hash); + + /// + /// Get a filestream for the given file hash. Returns null if the file does not exist in the store. + /// + Task GetFileStream(Hash hash, CancellationToken token = default); +} diff --git a/tests/NexusMods.DataModel.SchemaVersions.Tests/Schema.verified.md b/tests/NexusMods.DataModel.SchemaVersions.Tests/Schema.verified.md index 05ef1770e1..190ee23a3d 100644 --- a/tests/NexusMods.DataModel.SchemaVersions.Tests/Schema.verified.md +++ b/tests/NexusMods.DataModel.SchemaVersions.Tests/Schema.verified.md @@ -3,13 +3,15 @@ This schema is written to a markdown file for both documentation and validation models in the app, then validate the tests to update this file. ## Statistics - - Fingerprint: 0xD7B048D3EFE21816 - - Total attributes: 219 - - Total namespaces: 73 + - Fingerprint: 0xB5B51B6BF98A736B + - Total attributes: 221 + - Total namespaces: 74 ## Attributes | AttributeId | Type | Indexed | Many | NoHistory | | ---------------------------------------------------------------------------------- | ----------------------- | ------- | ----- | --------- | +| Namespace/AppIds | UInt32 | True | True | False | +| Namespace/PackageId | UInt32 | True | False | False | | NexusMods.Abstractions.Games.FileHashes.GogBuild/BuildId | UInt64 | True | False | False | | NexusMods.Abstractions.Games.FileHashes.GogBuild/Files | Reference | False | True | False | | NexusMods.Abstractions.Games.FileHashes.GogBuild/OperatingSystem | UInt8 | False | False | False |