Fix development for CK3 bookmark dates other than the save's date (#2740) #patch

closes #2684

Author: IhateTrains

ImperatorToCK3.UnitTests/CK3/Titles/LandedTitlesTests.cs

@@ -558,17 +558,15 @@ public void HistoryIsLoadedFromDatedBlocks() {
 		var date = new Date(867, 1, 1);
 		var config = new Configuration {
 			CK3BookmarkDate = date,
-			CK3Path = "TestFiles/LandedTitlesTests/CK3"
 		};
-		var ck3ModFS = new ModFilesystem(Path.Combine(config.CK3Path, "game"), new List<Mod>());
 
 		var titles = new Title.LandedTitles();
 		var title = titles.Add("k_greece");
 
 		titles.LoadHistory(config, ck3ModFS);
 
 		Assert.Equal("420", title.GetHolderId(date));
-		Assert.Equal(20, title.GetDevelopmentLevel(date));
+		Assert.Equal("e_persia", title.GetLiegeId(date));
 	}
 
 	[Fact]

ImperatorToCK3.UnitTests/CK3/Titles/TitleTests.cs

@@ -237,6 +237,11 @@ public void DevelopmentLevelCanBeInherited() {
 
 		county.SetDevelopmentLevel(4, date);
 		Assert.Equal(4, county.GetOwnOrInheritedDevelopmentLevel(date));
+		
+		// Development level set for de jure liege at a later date overrides the county's previously set level.
+		Date laterDate = date.ChangeByYears(1);
+		empire.SetDevelopmentLevel(12, laterDate);
+		Assert.Equal(12, county.GetOwnOrInheritedDevelopmentLevel(laterDate));
 	}
 
 	[Fact]

ImperatorToCK3.UnitTests/CommonUtils/IHistoryFieldTests.cs

@@ -0,0 +1,85 @@
+using commonItems;
+using commonItems.Collections;
+using ImperatorToCK3.CommonUtils;
+using System.Collections.Generic;
+using Xunit;
+
+namespace ImperatorToCK3.UnitTests.CommonUtils;
+
+public class IHistoryFieldTests {
+	[Fact]
+	public void ReturnsLatestEntryOnOrBeforeGivenDate() {
+		var field = CreateField();
+
+		field.AddEntryToHistory(null, "setter", "initValue");
+		field.AddEntryToHistory(new Date(100, 1, 1), "setter", "first");
+		field.AddEntryToHistory(new Date(200, 1, 1), "setter", "second");
+		field.AddEntryToHistory(new Date(200, 1, 1), "setter", "second-late");
+
+		var result = field.GetLastEntryWithDate(new Date(200, 1, 1));
+
+		Assert.Equal(new Date(200, 1, 1), result.Key);
+		Assert.Equal("second-late", result.Value);
+	}
+
+	[Fact]
+	public void ReturnsEntryFromMostRecentEarlierDateWhenExactDateMissing() {
+		var field = CreateField();
+
+		field.AddEntryToHistory(new Date(100, 1, 1), "setter", "first");
+		field.AddEntryToHistory(new Date(150, 1, 1), "setter", "second");
+
+		var result = field.GetLastEntryWithDate(new Date(140, 1, 1));
+
+		Assert.Equal(new Date(100, 1, 1), result.Key);
+		Assert.Equal("first", result.Value);
+	}
+
+	[Fact]
+	public void FallsBackToLastInitialEntryWhenNoEarlierDatedEntriesExist() {
+		var field = CreateField();
+
+		field.AddEntryToHistory(null, "setter", "initial1");
+		field.AddEntryToHistory(null, "setter", "initial2");
+		field.AddEntryToHistory(new Date(200, 1, 1), "setter", "future");
+
+		var result = field.GetLastEntryWithDate(new Date(150, 1, 1));
+
+		Assert.Null(result.Key);
+		var entry = Assert.IsType<KeyValuePair<string, object>>(result.Value);
+		Assert.Equal("setter", entry.Key);
+		Assert.Equal("initial2", entry.Value);
+	}
+
+	[Fact]
+	public void ReturnsNullPairWhenHistoryEmpty() {
+		var field = CreateField();
+
+		var result = field.GetLastEntryWithDate(new Date(50, 1, 1));
+
+		Assert.Null(result.Key);
+		Assert.Null(result.Value);
+	}
+
+	[Fact]
+	public void ReturnsLastInitialEntryWhenDateIsNull() {
+		var field = CreateField();
+
+		field.AddEntryToHistory(null, "setter", "initial1");
+		field.AddEntryToHistory(null, "setter", "initial2");
+		field.AddEntryToHistory(new Date(100, 1, 1), "setter", "dated");
+
+		var result = field.GetLastEntryWithDate(null);
+
+		Assert.Null(result.Key);
+		var entry = Assert.IsType<KeyValuePair<string, object>>(result.Value);
+		Assert.Equal("setter", entry.Key);
+		Assert.Equal("initial2", entry.Value);
+	}
+
+	private static IHistoryField CreateField() => new SimpleHistoryField(
+		fieldName: "field",
+		setterKeywords: new OrderedSet<string> { "setter" },
+		initialValue: null
+	);
+}

ImperatorToCK3.UnitTests/Imperator/Families/FamilyTests.cs

@@ -1,4 +1,5 @@
-using commonItems;
+using AwesomeAssertions;
+using commonItems;
 using ImperatorToCK3.Imperator.Families;
 using System;
 using System.Collections.Generic;
@@ -59,6 +60,8 @@ public void LinkingNullMemberIsLogged() {
 
 	[Fact]
 	public void IgnoredTokensAreSaved() {
+		Family.IgnoredTokens.Clear(); // Ensure no bleed-over from other tests.
+		
 		var reader1 = new BufferedReader("= { culture=paradoxian ignoredKeyword1=something ignoredKeyword2={} }");
 		var reader2 = new BufferedReader("= { ignoredKeyword1=stuff ignoredKeyword3=stuff }");
 		_ = Family.Parse(reader1, 1);
@@ -67,6 +70,6 @@ public void IgnoredTokensAreSaved() {
 		var expectedIgnoredTokens = new HashSet<string> {
 			"ignoredKeyword1", "ignoredKeyword2", "ignoredKeyword3"
 		};
-		Assert.True(Family.IgnoredTokens.SetEquals(expectedIgnoredTokens));
+		Family.IgnoredTokens.Should().BeEquivalentTo(expectedIgnoredTokens);
 	}
 }

ImperatorToCK3/CK3/Titles/LandedTitles.cs

@@ -1767,10 +1767,14 @@ public void LoadHistory(Configuration config, ModFilesystem ck3ModFS) {
 			Logger.Info($"Loaded {loadedHistoriesCount} title histories.");
 
 			// Add vanilla development to counties
-			// For counties that inherit development level from de jure lieges, assign it to them directly for better reliability.
-			foreach (var title in this.Where(t => t.Rank == TitleRank.county && t.GetDevelopmentLevel(ck3BookmarkDate) is null)) {
-				var inheritedDev = title.GetOwnOrInheritedDevelopmentLevel(ck3BookmarkDate);
-				title.SetDevelopmentLevel(inheritedDev ?? 0, ck3BookmarkDate);
+			// Assign development level directly to each county for better reliability, then remove it from duchies and above.
+			foreach (var county in Counties) {
+				var inheritedDev = county.GetOwnOrInheritedDevelopmentLevel(ck3BookmarkDate);
+				county.History.Fields.Remove("development_level");
+				county.SetDevelopmentLevel(inheritedDev ?? 0, ck3BookmarkDate);
+			}
+			foreach (var title in this.Where(t => t.Rank > TitleRank.county)) {
+				title.History.Fields.Remove("development_level");
 			}
 
 			// Remove history entries past the bookmark date.

ImperatorToCK3/CK3/Titles/Title.cs

@@ -1081,14 +1081,33 @@ public Dictionary<string, Title> GetDeFactoVassalsAndBelow(Date date, string ran
 	[SerializedName("cultural_names")] public Dictionary<string, string>? CulturalNames { get; private set; }
 
 	public int? GetOwnOrInheritedDevelopmentLevel(Date date) {
-		var ownDev = GetDevelopmentLevel(date);
-		if (ownDev is not null) { // if development level is already set, just return it
-			return ownDev;
-		}
-		if (deJureLiege is not null) { // if de jure liege exists, return their level
-			return deJureLiege.GetOwnOrInheritedDevelopmentLevel(date);
+		// Latest date (<= date) takes precedence.
+		// If multiple titles have the same date, lowest rank takes precedence.
+		// Consider all de jure lieges up to empire level.
+		var titlesToConsider = new List<Title> { this };
+		var currentLiege = DeJureLiege;
+		while (currentLiege is not null && currentLiege.Rank <= TitleRank.empire) {
+			titlesToConsider.Add(currentLiege);
+			currentLiege = currentLiege.DeJureLiege;
+		}
+		Date? bestDate = null;
+		Title? bestTitle = null;
+		foreach (var title in titlesToConsider) {
+			if (!title.History.Fields.TryGetValue("development_level", out var devField)) {
+				continue;
+			}
+			(Date? entryDate, object? value) = devField.GetLastEntryWithDate(date);
+			if (value is null) {
+				continue;
+			}
+
+			if (bestDate is null || (entryDate is not null && entryDate > bestDate) || (entryDate == bestDate && title.Rank < bestTitle?.Rank)) {
+				bestDate = entryDate;
+				bestTitle = title;
+			}
 		}
-		return null;
+
+		return bestTitle?.GetDevelopmentLevel(bestDate ?? date);
 	}
 
 	public ICollection<string> GetSuccessionLaws(Date date) {

ImperatorToCK3/CommonUtils/IHistoryField.cs

@@ -2,35 +2,52 @@
 using commonItems.Collections;
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using ZLinq;
 
 namespace ImperatorToCK3.CommonUtils;
 
 internal interface IHistoryField : IIdentifiable<string> {
-	public List<KeyValuePair<string, object>> InitialEntries { get; }
-	public SortedDictionary<Date, List<KeyValuePair<string, object>>> DateToEntriesDict { get; }
+	internal List<KeyValuePair<string, object>> InitialEntries { get; }
+	internal SortedDictionary<Date, List<KeyValuePair<string, object>>> DateToEntriesDict { get; }
 
-	public object? GetValue(Date? date);
+	internal object? GetValue(Date? date);
+	internal KeyValuePair<Date?, object?> GetLastEntryWithDate(Date? date) { // TODO: add tests for this
+		if (date is not null) {
+			var pairsWithEarlierOrSameDate = DateToEntriesDict.TakeWhile(d => d.Key <= date);
 
-	public void RemoveHistoryPastDate(Date date) {
+			foreach (var (d, entries) in pairsWithEarlierOrSameDate.Reverse()) {
+				foreach (var entry in Enumerable.Reverse(entries)) {
+					return new(d, entry.Value);
+				}
+			}
+		}
+
+		var lastInitialEntry = InitialEntries.LastOrNull();
+		return lastInitialEntry is not null
+			? new(key: null, lastInitialEntry.Value)
+			: new KeyValuePair<Date?, object?>(key: null, value: null);
+	}
+
+	internal void RemoveHistoryPastDate(Date date) {
 		foreach (var item in DateToEntriesDict.AsValueEnumerable().Where(kv => kv.Key > date).ToArray()) {
 			DateToEntriesDict.Remove(item.Key);
 		}
 	}
-	public void AddEntryToHistory(Date? date, string keyword, object value);
+	internal void AddEntryToHistory(Date? date, string keyword, object value);
 
 	/// <summary>
 	/// Removes all entries
 	/// </summary>
-	public void RemoveAllEntries() {
+	internal void RemoveAllEntries() {
 		RemoveAllEntries(_ => true);
 	}
 
 	/// <summary>
 	/// Removes all entries with values matching the predicate
 	/// </summary>
 	/// <param name="predicate"></param>
-	public int RemoveAllEntries(Func<object, bool> predicate) {
+	internal int RemoveAllEntries(Func<object, bool> predicate) {
 		int removed = 0;
 		removed += InitialEntries.RemoveAll(kvp => predicate(kvp.Value));
 		foreach (var datedEntriesBlock in DateToEntriesDict) {
@@ -40,9 +57,9 @@ public int RemoveAllEntries(Func<object, bool> predicate) {
 		return removed;
 	}
 
-	public void RegisterKeywords(Parser parser, Date date);
+	internal void RegisterKeywords(Parser parser, Date date);
 
-	public IEnumerable<KeyValuePair<string, object>> InitialEntriesForSerialization => InitialEntries;
+	internal IEnumerable<KeyValuePair<string, object>> InitialEntriesForSerialization => InitialEntries;
 
-	public IHistoryField Clone();
+	internal IHistoryField Clone();
 }

ImperatorToCK3/Outputter/TitlesOutputter.cs

@@ -26,9 +26,13 @@ private static async Task OutputTitlesHistory(string outputModPath, Title.Landed
 			alreadyOutputtedTitles.Add(title.Id);
 
 			// output the kingdom's de jure vassals' history
-			foreach (var (deJureVassalName, deJureVassal) in title.GetDeJureVassalsAndBelow()) {
+			// Order the outputted titles first by rank (duchies before counties), then by title ID.
+			var deJureVassalsAndBelow = title.GetDeJureVassalsAndBelow()
+				.OrderByDescending(t => t.Value.Rank)
+				.ThenBy(t => t.Key);
+			foreach (var (deJureVassalTitleId, deJureVassal) in deJureVassalsAndBelow) {
 				await deJureVassal.OutputHistory(historyOutput);
-				alreadyOutputtedTitles.Add(deJureVassalName);
+				alreadyOutputtedTitles.Add(deJureVassalTitleId);
 			}
 		}