Invention mappings for The Fallen Eagle mod and improved compatibilit…

…y for its 'After the Pharaohs' update (#2399)

closes #2093 

related to #2202

ImperatorToCK3/CK3/Characters/CharacterCollection.cs

@@ -824,4 +824,22 @@ internal void ConvertImperatorCharacterDNA(DNAFactory dnaFactory) {
 			}
 		}
 	}
+
+	public void RemoveUndefinedTraits(TraitMapper traitMapper) {
+		Logger.Info("Removing undefined traits from CK3 character history...");
+
+		var definedTraits = traitMapper.ValidCK3TraitIDs.ToHashSet();
+
+		foreach (var character in this) {
+			if (character.FromImperator) {
+				continue;
+			}
+
+			var traitsField = character.History.Fields["traits"];
+			int removedCount = traitsField.RemoveAllEntries(value => !definedTraits.Contains(value.ToString()?.RemQuotes() ?? string.Empty));
+			if (removedCount > 0) {
+				Logger.Debug($"Removed {removedCount} undefined traits from character {character.Id}.");
+			}
+		}
+	}
 }

ImperatorToCK3/CK3/Characters/CharactersLoader.cs

@@ -33,7 +33,7 @@ public void LoadCK3Characters(ModFilesystem ck3ModFS, Date bookmarkDate) {
 			"set_relation_ward", "set_relation_mentor",
 			"add_opinion", "make_concubine",
 		];
-		string[] fieldsToClear = ["friends", "best_friends", "lovers", "rivals", "nemesis", "primary_title"];
+		string[] fieldsToClear = ["friends", "best_friends", "lovers", "rivals", "nemesis", "primary_title", "dna"];
 
 		foreach (var character in loadedCharacters) {
 			// Remove post-bookmark history except for births and deaths.

ImperatorToCK3/CK3/Cultures/CultureCollection.cs

@@ -215,11 +215,11 @@ public void LoadInnovationIds(ModFilesystem ck3ModFS) {
 		return cultureMapper.Match(irCulture, ck3ProvinceId, irProvinceId, country.HistoricalTag);
 	}
 
-	public void ImportTechnology(CountryCollection countries, CultureMapper cultureMapper, ProvinceMapper provinceMapper, InventionsDB inventionsDB, LocDB irLocDB) { // TODO: add tests for this
+	public void ImportTechnology(CountryCollection countries, CultureMapper cultureMapper, ProvinceMapper provinceMapper, InventionsDB inventionsDB, LocDB irLocDB, OrderedDictionary<string, bool> ck3ModFlags) { // TODO: add tests for this
 		Logger.Info("Converting Imperator inventions to CK3 innovations...");
 
 		var innovationMapper = new InnovationMapper();
-		innovationMapper.LoadLinksAndBonuses("configurables/inventions_to_innovations_map.txt");
+		innovationMapper.LoadLinksAndBonuses("configurables/inventions_to_innovations_map.liquid", ck3ModFlags);
 		innovationMapper.LogUnmappedInventions(inventionsDB, irLocDB);
 		innovationMapper.RemoveMappingsWithInvalidInnovations(InnovationIds);
 

ImperatorToCK3/CK3/Provinces/ProvinceCollection.cs

@@ -1,6 +1,8 @@
 using commonItems;
 using commonItems.Collections;
 using commonItems.Mods;
+using ImperatorToCK3.CK3.Cultures;
+using ImperatorToCK3.CK3.Religions;
 using ImperatorToCK3.CK3.Titles;
 using ImperatorToCK3.CommonUtils.Map;
 using ImperatorToCK3.Exceptions;
@@ -9,8 +11,6 @@
 using ImperatorToCK3.Mappers.Religion;
 using Microsoft.VisualBasic.FileIO;
 using System.Collections.Generic;
-using System.Globalization;
-using System.IO;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -72,7 +72,7 @@ private void LoadProvincesHistory(ModFilesystem ck3ModFs) {
 		parser.ParseGameFolder("history/provinces", ck3ModFs, "txt", recursive: true);
 	}
 
-	public void ImportVanillaProvinces(ModFilesystem ck3ModFs) {
+	public void ImportVanillaProvinces(ModFilesystem ck3ModFs, ReligionCollection religions, CultureCollection cultures) {
 		var existingProvinceDefinitionsCount = Count;
 		Logger.Info("Importing vanilla provinces...");
 
@@ -83,6 +83,23 @@ public void ImportVanillaProvinces(ModFilesystem ck3ModFs) {
 		LoadProvincesHistory(ck3ModFs);
 		Logger.IncrementProgress();
 
+		// Cleanup: remove invalid faith and culture entries from province history
+		var validFaithIds = religions.Faiths.Select(f => f.Id).ToHashSet();
+		var validCultureIds = cultures.Select(c => c.Id).ToHashSet();
+		foreach (var province in this) {
+			var faithField = province.History.Fields["faith"];
+			int removedCount = faithField.RemoveAllEntries(value => !validFaithIds.Contains(value.ToString()?.RemQuotes() ?? string.Empty));
+			if (removedCount > 0) {
+				Logger.Debug($"Removed {removedCount} invalid faith entries from province {province.Id}.");
+			}
+
+			var cultureField = province.History.Fields["culture"];
+			removedCount = cultureField.RemoveAllEntries(value => !validCultureIds.Contains(value.ToString()?.RemQuotes() ?? string.Empty));
+			if (removedCount > 0) {
+				Logger.Debug($"Removed {removedCount} invalid culture entries from province {province.Id}.");
+			}
+		}
+
 		// Now load the provinces that don't have unique entries in history/provinces.
 		// They instead use history/province_mapping.
 		foreach (var (newProvinceId, baseProvinceId) in new ProvinceMappings(ck3ModFs)) {

ImperatorToCK3/CK3/Religions/Faith.cs

@@ -24,6 +24,13 @@ public Faith(string id, FaithData faithData, Religion religion) {
 		DoctrineIds = faithData.DoctrineIds.ToOrderedSet();
 		holySiteIds = faithData.HolySiteIds.ToOrderedSet();
 		attributes = [.. faithData.Attributes];
+
+		// Fixup for issue found in TFE: add reformed_icon if faith has unreformed_faith_doctrine.
+		if (DoctrineIds.Contains("unreformed_faith_doctrine") && !attributes.Any(pair => pair.Key == "reformed_icon")) {
+			// Use the icon attribute.
+			var icon = attributes.FirstOrDefault(pair => pair.Key == "icon");
+			attributes = [.. attributes, new KeyValuePair<string, StringOfItem>("reformed_icon", icon.Value)];
+		}
 	}
 
 	private readonly OrderedSet<string> holySiteIds;

ImperatorToCK3/CK3/Titles/LandedTitles.cs

@@ -338,6 +338,15 @@ public void CleanUpHistory(CharacterCollection characters, Date ck3BookmarkDate)
 					}
 				}
 			}
+
+			// Remove undated succession_laws entries; the game doesn't seem to like them.
+			foreach (var title in this) {
+				if (!title.History.Fields.TryGetValue("succession_laws", out var successionLawsField)) {
+					continue;
+				}
+
+				successionLawsField.InitialEntries.RemoveAll(entry => true);
+			}
 		}
 
 		internal void ImportImperatorCountries(

ImperatorToCK3/CK3/World.cs

@@ -134,15 +134,17 @@ public World(Imperator.World impWorld, Configuration config, Thread? irCoaExtrac
 			() => CK3CoaMapper = new(ModFS)
 		);
 
+		OrderedDictionary<string, bool> ck3ModFlags = config.GetCK3ModFlags();
+
 		Parallel.Invoke(
 			() => { // depends on ck3ColorFactory and CulturalPillars
 				// Load CK3 cultures from CK3 mod filesystem.
 				Logger.Info("Loading cultural pillars...");
-				CulturalPillars = new(ck3ColorFactory, config.GetCK3ModFlags());
+				CulturalPillars = new(ck3ColorFactory, ck3ModFlags);
 				CulturalPillars.LoadPillars(ModFS);
 				Logger.Info("Loading converter cultural pillars...");
 				CulturalPillars.LoadConverterPillars("configurables/cultural_pillars");
-				Cultures = new CultureCollection(ck3ColorFactory, CulturalPillars, config.GetCK3ModFlags());
+				Cultures = new CultureCollection(ck3ColorFactory, CulturalPillars, ck3ModFlags);
 				Cultures.LoadNameLists(ModFS);
 				Cultures.LoadInnovationIds(ModFS);
 				Cultures.LoadCultures(ModFS);
@@ -213,7 +215,7 @@ public World(Imperator.World impWorld, Configuration config, Thread? irCoaExtrac
 		var religionMapper = new ReligionMapper(Religions, imperatorRegionMapper, ck3RegionMapper);
 
 		Parallel.Invoke(
-			() => Cultures.ImportTechnology(impWorld.Countries, cultureMapper, provinceMapper, impWorld.InventionsDB, impWorld.LocDB),
+			() => Cultures.ImportTechnology(impWorld.Countries, cultureMapper, provinceMapper, impWorld.InventionsDB, impWorld.LocDB, ck3ModFlags),
 
 			() => { // depends on religionMapper
 				// Check if all I:R religions have a base mapping.
@@ -243,6 +245,9 @@ public World(Imperator.World impWorld, Configuration config, Thread? irCoaExtrac
                 		Logger.Warn($"No base mapping found for I:R culture {cultureStr}!");
                 	}
                 }
+			},
+			() => { // depends on TraitMapper and CK3 characters being loaded
+				Characters.RemoveUndefinedTraits(traitMapper);
 			}
 		);
 
@@ -308,7 +313,7 @@ public World(Imperator.World impWorld, Configuration config, Thread? irCoaExtrac
 
 		// Now we can deal with provinces since we know to whom to assign them. We first import vanilla province data.
 		// Some of it will be overwritten, but not all.
-		Provinces.ImportVanillaProvinces(ModFS);
+		Provinces.ImportVanillaProvinces(ModFS, Religions, Cultures);
 
 		// Next we import Imperator provinces and translate them ontop a significant part of all imported provinces.
 		Provinces.ImportImperatorProvinces(impWorld, MapData, LandedTitles, cultureMapper, religionMapper, provinceMapper, CorrectedDate, config);

ImperatorToCK3/CommonUtils/IHistoryField.cs

@@ -30,11 +30,14 @@ public void RemoveAllEntries() {
 	/// Removes all entries with values matching the predicate
 	/// </summary>
 	/// <param name="predicate"></param>
-	public void RemoveAllEntries(Func<object, bool> predicate) {
-		InitialEntries.RemoveAll(kvp => predicate(kvp.Value));
+	public int RemoveAllEntries(Func<object, bool> predicate) {
+		int removed = 0;
+		removed += InitialEntries.RemoveAll(kvp => predicate(kvp.Value));
 		foreach (var datedEntriesBlock in DateToEntriesDict) {
-			datedEntriesBlock.Value.RemoveAll(kvp => predicate(kvp.Value));
+			removed += datedEntriesBlock.Value.RemoveAll(kvp => predicate(kvp.Value));
 		}
+
+		return removed;
 	}
 
 	public void RegisterKeywords(Parser parser, Date date);

ImperatorToCK3/Data_Files/configurables/culture_map.txt

@@ -582,6 +582,9 @@ link = { ck3=butr ir=gyzantian ir=psyllic ir=nasamonian ir=perorsian ir=phazani
 link = { ck3=numidian ir=capariensian ir=lotophagoi ir=abannaean }
 
 ## Persian and Iranian
+# TFE
+link = { ck3 = median ir = median }
+# vanilla CK3
 link = { ck3 = persian ir = persian } # https://steamcommunity.com/sharedfiles/filedetails/?id=3098496649
 link = { ck3 = daylamite ir = daylamite } # https://steamcommunity.com/sharedfiles/filedetails/?id=3098496649
 link = { ck3 = baloch ir = baloch } # https://steamcommunity.com/sharedfiles/filedetails/?id=3098496649
@@ -591,7 +594,7 @@ link = { ck3 = daylamite ir = hyrcanian }
 link = { ck3 = parthian ir = parthian }
 link = {
 	ck3 = kurdish
-	ir = median
+	ir = median # TODO: add a separate Median culture to the converter instead of mapping to Kurdish
 	ir = cardukoi # added by Invictus https://en.wikipedia.org/wiki/Carduchii, TODO: create a separate CK3 culture for them
 }
 link = { ck3 = afghan ir = pactyan ir = sattagydian }