Intrinsic files (#3894)

* Add intrinsic file support for Fallout 4 and Skyrim SE synchronizers (#3830)

* Add intrinsic file handling for Skyrim SE and Fallout 4 synchronizers, refactor `Write` method to remove unused transaction parameter, and extend plugin file handling logic.

* Refactor `KnownPaths` to use target-typed `new` and add missing `using` directive in `PluginsFile`.

* Add stream source architecture with prioritization and improve plugin file handling

- Introduced `IReadOnlyStreamSource`, `IStreamSourceDispatcher`, and `SourcePriority` to enable prioritized, read-only stream sources.
- Implemented `NxFileStore`, `GameFileStreamSource`, and `StreamSourceDispatcher` to handle stream retrieval by source type.
- Refactored `PluginsFile` to include sorting logic and enhanced plugin dependency handling.
- Updated `ACreationEngineSynchronizer` constructor to support the new stream architecture.

* Refactor stream handling and remove `IFileStore` from SkyrimSE, Fallout 4, and PluginFile implementations

- Replaced `IFileStore` usage with `IStreamSourceDispatcher` for streamlined and prioritized stream access.
- Updated synchronizers and constructors for SkyrimSE and Fallout 4 to reflect the new stream architecture.
- Enhanced `PluginsFile.Write` logic to generate dynamic plugin lists, removed unused `transaction` parameter, and added intrinsic file support.
- Refactored error handling and conditions in `NxFileStore` and `GameFileStreamSource`.

* WIP updating the syncronizer code to support file types

* Expand ActionMapping to support additional shorthand variants

* Add handling for intrinsic file actions in synchronizer rules

- Introduced `WriteIntrinsic` and `AdaptLoadout` actions for intrinsic file types.
- Updated test cases to validate intrinsic file scenarios.
- Refactored `ActionMapping` to correctly map actions for intrinsic files.
- Adjusted flag ordering in `Actions` enum to ensure proper action sequence.

* Add `AdaptLoadout` and `WriteIntrinsic` actions for intrinsic files

- Implemented handling for `AdaptLoadout` and `WriteIntrinsic` actions in synchronizers.
- Updated tests to cover intrinsic file scenarios and associated validations.
- Enhanced `PluginsFile` with intrinsic file handling and logging.
- Adjusted `ActionMapping` to correctly map shorthand to new actions.

* Ensure directories exist before writing resolved paths and add `WriteIntrinsic` action validation

- Added directory creation for resolved paths in synchronizers to avoid file write failures.
- Implemented validation for `WriteIntrinsic` action in debug mode.
- Prevented empty plugin lists from being sorted in `PluginsFile` logic.

* Remove redundant signature shorthand cases
- Reduced signature count from 276 to 256 in verified output.

* Refactor `IntrinsicFiles` to use `Dictionary` for improved lookup efficiency and update related logic.

* Refactor `IntrinsicFiles` to a method and update synchronization logic for improved flexibility and context handling.

Author: halgari

src/NexusMods.Abstractions.GameLocators/GamePath.cs

@@ -52,6 +52,16 @@ public GamePath(LocationId locationId, RelativePath path)
 
     /// <summary/>
     public static bool operator !=(GamePath a, GamePath b) => a.LocationId != b.LocationId || a.Path != b.Path;
+    
+    /// <summary>
+    /// Joins the current path with a relative path.
+    /// </summary>
+    public static GamePath operator /(GamePath a, RelativePath b) => new(a.LocationId, a.Path / b);
+    
+    /// <summary>
+    /// Joins the current path with a string.
+    /// </summary>
+    public static GamePath operator /(GamePath a, string b) => new(a.LocationId, a.Path / b);
 
     /// <inheritdoc />
     public override bool Equals(object? obj) => obj is GamePath other && Equals(other);

src/NexusMods.Abstractions.Loadouts.Synchronizers/ALoadoutSynchronizer.cs

@@ -333,6 +333,28 @@ public Dictionary<GamePath, SyncNode> BuildSyncTree<T>(T latestDiskState, T prev
                 }
             }
         }
+
+        // Add in the intrinsic files
+        foreach (var file in IntrinsicFiles(loadout).Values)
+        {
+            ref var found = ref CollectionsMarshal.GetValueRefOrAddDefault(syncTree, file.Path, out var exists);
+            if (exists)
+            {
+                Logger.LogWarning("Found duplicate intrinsic file `{Path}` in Loadout {LoadoutName} for game {Game}", file.Path, loadout.Name, loadout.InstallationInstance.Game.Name);
+            }
+            found.SourceItemType = LoadoutSourceItemType.Intrinsic;
+            var stream = new MemoryStream();
+            file.Write(stream, loadout, syncTree);
+            stream.Position = 0;
+            var span = stream.GetBuffer().AsSpan(0, (int)stream.Length);
+            found.Loadout = new SyncNodePart()
+            {
+                Hash = span.xxHash3(),
+                Size = Size.From((ulong)span.Length),
+                LastModifiedTicks = 0,
+            };
+            found.SourceItemType = LoadoutSourceItemType.Intrinsic;
+        }
 
         // Remove deleted files. I'm not super happy with this requiring a full scan of
         // the loadout, but we have to somehow mark the deleted files and then delete them. 
@@ -526,7 +548,8 @@ public void ProcessSyncTree(Dictionary<GamePath, SyncNode> tree)
                 diskArchived: item.HaveDisk && HaveArchive(item.Disk.Hash),
                 prevArchived: item.HavePrevious && HaveArchive(item.Previous.Hash),
                 loadoutArchived: item.Loadout.Hash != Hash.Zero && HaveArchive(item.Loadout.Hash),
-                pathIsIgnored: IsIgnoredBackupPath(path));
+                pathIsIgnored: IsIgnoredBackupPath(path),
+                item.SourceItemType);
 
 
             item.Signature = signature;
@@ -560,6 +583,11 @@ public void ProcessSyncTree(Dictionary<GamePath, SyncNode> tree)
                     job?.SetStatus("Adding external changes");
                     ActionIngestFromDisk(syncTree, loadout, tx, ref overridesGroup);
                     break;
+                
+                case Actions.AdaptLoadout:
+                    job?.SetStatus("Updating loadout");
+                    await AdaptLoadout(syncTree, register, loadout, tx);
+                    break;
 
                 case Actions.DeleteFromDisk:
                     job?.SetStatus("Deleting files");
@@ -570,6 +598,11 @@ public void ProcessSyncTree(Dictionary<GamePath, SyncNode> tree)
                     job?.SetStatus("Extracting files");
                     await ActionExtractToDisk(syncTree, register, tx, gameMetadataId, job);
                     break;
+                
+                case Actions.WriteIntrinsic:
+                    job?.SetStatus("Writing intrinsic files");
+                    await ActionWriteIntrinsics(syncTree, register, tx, loadout, job);
+                    break;
 
                 case Actions.AddReifiedDelete:
                     job?.SetStatus("Updating deleted files");
@@ -583,7 +616,7 @@ public void ProcessSyncTree(Dictionary<GamePath, SyncNode> tree)
                 case Actions.WarnOfConflict:
                     WarnOfConflict(syncTree);
                     break;
-
+                
                 default:
                     throw new InvalidOperationException($"Unknown action: {action}");
             }
@@ -615,6 +648,45 @@ public void ProcessSyncTree(Dictionary<GamePath, SyncNode> tree)
         return loadout;
     }
 
+    private async Task ActionWriteIntrinsics(Dictionary<GamePath, SyncNode> syncTree, IGameLocationsRegister register, IMainTransaction tx, Loadout.ReadOnly loadout, SynchronizeLoadoutJob? job)
+    {
+        var intrinsicFiles = IntrinsicFiles(loadout);
+        foreach (var (path, node) in syncTree)
+        {
+            if (!node.Actions.HasFlag(Actions.WriteIntrinsic))
+                continue;
+            
+            if (node.SourceItemType != LoadoutSourceItemType.Intrinsic)
+                throw new Exception("WriteIntrinsic should only be called on intrinsic files");
+
+            var instance = intrinsicFiles[path];
+            var resolvedPath = register.GetResolvedPath(path);
+            resolvedPath.Parent.CreateDirectory();
+            await using var stream = resolvedPath.Create();
+            stream.SetLength(0);
+            await instance.Write(stream, loadout, syncTree);
+        }
+    }
+
+    private async Task AdaptLoadout(Dictionary<GamePath, SyncNode> syncTree, IGameLocationsRegister register, Loadout.ReadOnly loadout, IMainTransaction tx)
+    {
+        var intrinsicFiles = IntrinsicFiles(loadout);
+        foreach (var (path, node) in syncTree)
+        {
+            if (!node.Actions.HasFlag(Actions.AdaptLoadout))
+                continue;
+            
+            if (node.SourceItemType != LoadoutSourceItemType.Intrinsic)
+                throw new Exception("AdaptLoadout should only be called on intrinsic files");
+
+            var instance = intrinsicFiles[path];
+            var resolvedPath = register.GetResolvedPath(path);
+            await using var stream = resolvedPath.Read();
+            await instance.Ingest(stream, loadout, syncTree, tx);
+        }
+
+    }
+
     private async ValueTask<Loadout.ReadOnly> ReprocessOverrides(Loadout.ReadOnly loadout)
     {
         loadout = await ReprocessGameUpdates(loadout);
@@ -739,6 +811,16 @@ public async Task RunActions(Dictionary<GamePath, SyncNode> syncTree, GameInstal
                 case Actions.BackupFile:
                     await ActionBackupNewFiles(gameInstallation, syncTree);
                     break;
+                
+                case Actions.AdaptLoadout:
+                    if (ApplicationConstants.IsDebug && syncTree.Any(n => n.Value.Actions.HasFlag(Actions.AdaptLoadout)))
+                        throw new InvalidOperationException("Cannot adapt loadout when not in a loadout context");
+                    break;
+                
+                case Actions.WriteIntrinsic:
+                    if (ApplicationConstants.IsDebug && syncTree.Any(n => n.Value.Actions.HasFlag(Actions.AdaptLoadout)))
+                        throw new InvalidOperationException("Cannot adapt loadout when not in a loadout context");
+                    break;
 
                 case Actions.IngestFromDisk:
                     if (ApplicationConstants.IsDebug && syncTree.Any(n => n.Value.Actions.HasFlag(Actions.IngestFromDisk)))
@@ -1849,6 +1931,16 @@ EntityId RemapFn(EntityId entityId)
         }
     }
 
+    /// <summary>
+    /// Gets a set of files intrinsic to this game. Such as mod order files, preference files, etc.
+    /// These files will not be backed up and will not be included in the loadout directly. Instead, they are
+    /// generated at sync time by calling the implementations of the files themselves. 
+    /// </summary>
+    protected virtual Dictionary<GamePath, IIntrinsicFile> IntrinsicFiles(Loadout.ReadOnly loadout)
+    {
+        return new();
+    }
+
     private static string GetNewShortName(IDb db, GameInstallMetadataId installationId)
     {
         var existingShortNames = Loadout.All(db)

src/NexusMods.Abstractions.Loadouts.Synchronizers/IIntrinsicFile.cs

@@ -0,0 +1,22 @@
+using NexusMods.Abstractions.GameLocators;
+using NexusMods.MnemonicDB.Abstractions;
+
+namespace NexusMods.Abstractions.Loadouts.Synchronizers;
+
+public interface IIntrinsicFile
+{
+    /// <summary>
+    /// The game path of this file.
+    /// </summary>
+    public GamePath Path { get; }
+
+    /// <summary>
+    /// Write the contents of this file to the stream.
+    /// </summary>
+    public Task Write(Stream stream, Loadout.ReadOnly loadout, Dictionary<GamePath, SyncNode> syncTree);
+
+    /// <summary>
+    /// Ingest the contents of the stream into the loadout
+    /// </summary>
+    public Task Ingest(Stream stream, Loadout.ReadOnly loadout, Dictionary<GamePath, SyncNode> syncTree, ITransaction tx);
+}

src/NexusMods.Abstractions.Loadouts.Synchronizers/LoadoutSourceItem.cs

@@ -26,6 +26,13 @@ public enum LoadoutSourceItemType : byte
     /// This is a deleted file
     /// </summary>
     Deleted,
+    
+    /// <summary>
+    /// A file intrinsic to the game. This file cannot be deleted, and will not show
+    /// up in the loadout as a file that can be manipluated. Instead the game provides
+    /// special handling for this file.
+    /// </summary>
+    Intrinsic,
 }
 
 public readonly record struct LoadoutSourceItem