Invention mappings for The Fallen Eagle mod and improved compatibility for its 'After the Pharaohs' update

ImperatorToCK3/CK3/Characters/CharacterCollection.cs

@@ -661,156 +661,156 @@
 		Logger.IncrementProgress();
 	}
 
 	public void GenerateSuccessorsForOldCharacters(Title.LandedTitles titles, CultureCollection cultures, Date irSaveDate, Date ck3BookmarkDate, ulong randomSeed) {
 		Logger.Info("Generating successors for old characters...");
 
 		var oldCharacters = this
 			.Where(c => c.BirthDate < ck3BookmarkDate && c.DeathDate is null)
 			.Where(c => ck3BookmarkDate.DiffInYears(c.BirthDate) > 60)
 			.ToArray();
 
 		var titleHolderIds = titles.GetHolderIdsForAllTitlesExceptNobleFamilyTitles(ck3BookmarkDate);
 
 		var oldTitleHolders = oldCharacters
 			.Where(c => titleHolderIds.Contains(c.Id))
 			.ToArray();
 
 		// For characters that don't hold any titles, just set up a death date.
 		var randomForCharactersWithoutTitles = new Random((int)randomSeed);
 		foreach (var oldCharacter in oldCharacters.Except(oldTitleHolders)) {
 			// Roll a dice to determine how much longer the character will live.
 			var yearsToLive = randomForCharactersWithoutTitles.Next(0, 30);
 
 			// If the character is female and pregnant, make sure she doesn't die before the pregnancy ends.
 			if (oldCharacter is {Female: true, ImperatorCharacter: not null}) {
 				var lastPregnancy = oldCharacter.Pregnancies.OrderBy(p => p.BirthDate).LastOrDefault();
 				if (lastPregnancy is not null) {
 					oldCharacter.DeathDate = lastPregnancy.BirthDate.ChangeByYears(yearsToLive);
 					continue;
 				}
 			}
 
 			oldCharacter.DeathDate = irSaveDate.ChangeByYears(yearsToLive);
 		}
 
 		ConcurrentDictionary<string, Title[]> titlesByHolderId = new(titles
 			.Select(t => new {Title = t, HolderId = t.GetHolderId(ck3BookmarkDate)})
 			.Where(t => t.HolderId != "0")
 			.GroupBy(t => t.HolderId)
 			.ToDictionary(g => g.Key, g => g.Select(t => t.Title).ToArray()));
 
 		ConcurrentDictionary<string, string[]> cultureIdToMaleNames = new(cultures
 			.ToDictionary(c => c.Id, c => c.MaleNames.ToArray()));
 
 		// For title holders, generate successors and add them to title history.
 		Parallel.ForEach(oldTitleHolders, oldCharacter => {
 			// Get all titles held by the character.
 			var heldTitles = titlesByHolderId[oldCharacter.Id];
 			string? dynastyId = oldCharacter.GetDynastyId(ck3BookmarkDate);
 			string? dynastyHouseId = oldCharacter.GetDynastyHouseId(ck3BookmarkDate);
 			string? faithId = oldCharacter.GetFaithId(ck3BookmarkDate);
 			string? cultureId = oldCharacter.GetCultureId(ck3BookmarkDate);
 			string[] maleNames;
 			if (cultureId is not null) {
 				maleNames = cultureIdToMaleNames[cultureId];
 			} else {
 				Logger.Warn($"Failed to find male names for successors of {oldCharacter.Id}.");
 				maleNames = ["Alexander"];
 			}
 
 			var randomSeedForCharacter = randomSeed ^ (oldCharacter.ImperatorCharacter?.Id ?? 0);
 			var random = new Random((int)randomSeedForCharacter);
 
 			int successorCount = 0;
 			Character currentCharacter = oldCharacter;
 			Date currentCharacterBirthDate = currentCharacter.BirthDate;
 			while (ck3BookmarkDate.DiffInYears(currentCharacterBirthDate) >= 90) {
 				// If the character has living male children, the oldest one will be the successor.
 				var successorAndBirthDate = currentCharacter.Children
 					.Where(c => c is {Female: false, DeathDate: null})
 					.Select(c => new { Character = c, c.BirthDate })
 					.OrderBy(x => x.BirthDate)
 					.FirstOrDefault();
 
 				Character successor;
 				Date currentCharacterDeathDate;
 				Date successorBirthDate;
 				if (successorAndBirthDate is not null) {
 					successor = successorAndBirthDate.Character;
 					successorBirthDate = successorAndBirthDate.BirthDate;
 
 					// Roll a dice to determine how much longer the character will live.
 					// But make sure the successor is at least 16 years old when the old character dies.
 					var successorAgeAtBookmarkDate = ck3BookmarkDate.DiffInYears(successorBirthDate);
 					var yearsUntilSuccessorBecomesAnAdult = Math.Max(16 - successorAgeAtBookmarkDate, 0);
 
 					var yearsToLive = random.Next((int)Math.Ceiling(yearsUntilSuccessorBecomesAnAdult), 25);
 					int currentCharacterAge = random.Next(30 + yearsToLive, 80);
 					currentCharacterDeathDate = currentCharacterBirthDate.ChangeByYears(currentCharacterAge);
 					// Needs to be after the save date.
 					if (currentCharacterDeathDate <= irSaveDate) {
 						currentCharacterDeathDate = irSaveDate.ChangeByDays(1);
 					}
 				} else {
 					// We don't want all the generated successors on the map to have the same birth date.
 					var yearsUntilHeir = random.Next(1, 5);
 
 					// Make the old character live until the heir is at least 16 years old.
 					var successorAge = random.Next(yearsUntilHeir + 16, 30);
 					int currentCharacterAge = random.Next(30 + successorAge, 80);
 					currentCharacterDeathDate = currentCharacterBirthDate.ChangeByYears(currentCharacterAge);
 					if (currentCharacterDeathDate <= irSaveDate) {
 						currentCharacterDeathDate = irSaveDate.ChangeByDays(1);
 					}
 
 					// Generate a new successor.
 					string id = $"irtock3_{oldCharacter.Id}_successor_{successorCount}";
 					string firstName = maleNames[random.Next(0, maleNames.Length)];
 
 					successorBirthDate = currentCharacterDeathDate.ChangeByYears(-successorAge);
 					successor = new Character(id, firstName, successorBirthDate, this) {FromImperator = true};
 					Add(successor);
 					if (currentCharacter.Female) {
 						successor.Mother = currentCharacter;
 					} else {
 						successor.Father = currentCharacter;
 					}
 					if (cultureId is not null) {
 						successor.SetCultureId(cultureId, null);
 					}
 					if (faithId is not null) {
 						successor.SetFaithId(faithId, null);
 					}
 					if (dynastyId is not null) {
 						successor.SetDynastyId(dynastyId, null);
 					}
 					if (dynastyHouseId is not null) {
 						successor.SetDynastyHouseId(dynastyHouseId, null);
 					}
 				}
 
 				currentCharacter.DeathDate = currentCharacterDeathDate;
 				// On the old character death date, the successor should inherit all titles.
 				foreach (var heldTitle in heldTitles) {
 					heldTitle.SetHolder(successor, currentCharacterDeathDate);
 				}
 
 				// Move to the successor and repeat the process.
 				currentCharacter = successor;
 				currentCharacterBirthDate = successorBirthDate;
 				++successorCount;
 			}
 
 			// After the loop, currentCharacter should represent the successor at bookmark date.
 			// Set his DNA to avoid weird looking character on the bookmark screen in CK3.
 			currentCharacter.DNA = oldCharacter.DNA;
 
 			// Transfer gold to the living successor.
 			currentCharacter.Gold = oldCharacter.Gold;
 			oldCharacter.Gold = null;
 		});
 	}
 
 	internal void ConvertImperatorCharacterDNA(DNAFactory dnaFactory) {
 		Logger.Info("Converting Imperator character DNA to CK3...");
 		foreach (var character in this) {
@@ -824,4 +824,22 @@
 			}
 		}
 	}
+
+	public void RemoveUndefinedTraits(TraitMapper traitMapper) {
+		Logger.Info("Removing undefined traits from CK3 character history...");
+
+		var definedTraits = traitMapper.ValidCK3TraitIDs.ToHashSet();
+
+		foreach (var character in this) {
+			if (character.FromImperator) {
+				continue;
+			}
+
+			var traitsField = character.History.Fields["traits"];
+			int removedCount = traitsField.RemoveAllEntries(value => !definedTraits.Contains(value.ToString() ?? string.Empty));
+			if (removedCount > 0) {
+				Logger.Debug($"Removed {removedCount} undefined traits from character {character.Id}.");
+			}
+		}
+	}
 }

ImperatorToCK3/CK3/Characters/CharactersLoader.cs

@@ -33,7 +33,7 @@ public void LoadCK3Characters(ModFilesystem ck3ModFS, Date bookmarkDate) {
 			"set_relation_ward", "set_relation_mentor",
 			"add_opinion", "make_concubine",
 		];
-		string[] fieldsToClear = ["friends", "best_friends", "lovers", "rivals", "nemesis", "primary_title"];
+		string[] fieldsToClear = ["friends", "best_friends", "lovers", "rivals", "nemesis", "primary_title", "dna"];
 
 		foreach (var character in loadedCharacters) {
 			// Remove post-bookmark history except for births and deaths.

ImperatorToCK3/CK3/Cultures/CultureCollection.cs

@@ -215,11 +215,11 @@
 		return cultureMapper.Match(irCulture, ck3ProvinceId, irProvinceId, country.HistoricalTag);
 	}
 
-	public void ImportTechnology(CountryCollection countries, CultureMapper cultureMapper, ProvinceMapper provinceMapper, InventionsDB inventionsDB, LocDB irLocDB) { // TODO: add tests for this
+	public void ImportTechnology(CountryCollection countries, CultureMapper cultureMapper, ProvinceMapper provinceMapper, InventionsDB inventionsDB, LocDB irLocDB, OrderedDictionary<string, bool> ck3ModFlags) { // TODO: add tests for this
 		Logger.Info("Converting Imperator inventions to CK3 innovations...");
 
 		var innovationMapper = new InnovationMapper();
-		innovationMapper.LoadLinksAndBonuses("configurables/inventions_to_innovations_map.txt");
+		innovationMapper.LoadLinksAndBonuses("configurables/inventions_to_innovations_map.liquid", ck3ModFlags);
 		innovationMapper.LogUnmappedInventions(inventionsDB, irLocDB);
 		innovationMapper.RemoveMappingsWithInvalidInnovations(InnovationIds);
 

ImperatorToCK3/CK3/World.cs

@@ -134,15 +134,17 @@
 			() => CK3CoaMapper = new(ModFS)
 		);
 
+		OrderedDictionary<string, bool> ck3ModFlags = config.GetCK3ModFlags();
+
 		Parallel.Invoke(
 			() => { // depends on ck3ColorFactory and CulturalPillars
 				// Load CK3 cultures from CK3 mod filesystem.
 				Logger.Info("Loading cultural pillars...");
-				CulturalPillars = new(ck3ColorFactory, config.GetCK3ModFlags());
+				CulturalPillars = new(ck3ColorFactory, ck3ModFlags);
 				CulturalPillars.LoadPillars(ModFS);
 				Logger.Info("Loading converter cultural pillars...");
 				CulturalPillars.LoadConverterPillars("configurables/cultural_pillars");
-				Cultures = new CultureCollection(ck3ColorFactory, CulturalPillars, config.GetCK3ModFlags());
+				Cultures = new CultureCollection(ck3ColorFactory, CulturalPillars, ck3ModFlags);
 				Cultures.LoadNameLists(ModFS);
 				Cultures.LoadInnovationIds(ModFS);
 				Cultures.LoadCultures(ModFS);
@@ -213,7 +215,7 @@
 		var religionMapper = new ReligionMapper(Religions, imperatorRegionMapper, ck3RegionMapper);
 
 		Parallel.Invoke(
-			() => Cultures.ImportTechnology(impWorld.Countries, cultureMapper, provinceMapper, impWorld.InventionsDB, impWorld.LocDB),
+			() => Cultures.ImportTechnology(impWorld.Countries, cultureMapper, provinceMapper, impWorld.InventionsDB, impWorld.LocDB, ck3ModFlags),
 
 			() => { // depends on religionMapper
 				// Check if all I:R religions have a base mapping.
@@ -243,6 +245,9 @@
                 		Logger.Warn($"No base mapping found for I:R culture {cultureStr}!");
                 	}
                 }
+			},
+			() => { // depends on TraitMapper and CK3 characters being loaded
+				Characters.RemoveUndefinedTraits(traitMapper);
 			}
 		);
 
@@ -913,160 +918,160 @@
 		}
 	}
 
 	private void GenerateFillerHoldersForUnownedLands(CultureCollection cultures, Configuration config) {
 		Logger.Info("Generating filler holders for unowned lands...");
 		var date = config.CK3BookmarkDate;
 		List<Title> unheldCounties = [];
 		foreach (var county in LandedTitles.Counties) {
 			var holderId = county.GetHolderId(date);
 			if (holderId == "0") {
 				unheldCounties.Add(county);
 			} else if (Characters.TryGetValue(holderId, out var holder)) {
 				if (holder.DeathDate is not null && holder.DeathDate <= date) {
 					Logger.Debug($"Adding {county.Id} to unheld counties because holder {holderId} is dead.");
 					unheldCounties.Add(county);
 				}
 			}
 		}
 
 		var duchyIdToHolderDict = new Dictionary<string, Character>();
 
 		foreach (var county in unheldCounties) {
 			if (config.FillerDukes) {
 				var duchy = county.DeJureLiege;
 				if (duchy is not null && duchy.Rank == TitleRank.duchy) {
 					if (duchyIdToHolderDict.TryGetValue(duchy.Id, out var duchyHolder)) {
 						county.SetHolder(duchyHolder, date);
 						continue;
 					}
 				}
 			}
 
 			var candidateProvinces = new OrderedSet<Province>();
 			if (county.CapitalBaronyProvinceId is not null) {
 				// Give priority to capital province.
 				if (Provinces.TryGetValue(county.CapitalBaronyProvinceId.Value, out var capitalProvince)) {
 					candidateProvinces.Add(capitalProvince);
 				}
 			}
 
 			var allCountyProvinces = county.CountyProvinceIds
 				.Select(id => Provinces.TryGetValue(id, out var province) ? province : null)
 				.Where(p => p is not null)
 				.Select(p => p!);
 			candidateProvinces.UnionWith(allCountyProvinces);
 
 			int pseudoRandomSeed;
 			if (candidateProvinces.Count != 0) {
 				pseudoRandomSeed = (int)candidateProvinces.First().Id;
 			} else {
 				// Use county ID for seed if no province is available.
 				pseudoRandomSeed = county.Id.Aggregate(0, (current, c) => current + c);
 			}
 
 			// Determine culture of the holder.
 			var culture = candidateProvinces
 				.Select(p => p.GetCulture(date, cultures))
 				.FirstOrDefault(c => c is not null);
 			if (culture is null) {
 				Logger.Debug($"Trying to use de jure duchy for culture of holder for {county.Id}...");
 				var deJureDuchy = county.DeJureLiege;
 				if (deJureDuchy is not null) {
 					culture = Provinces
 						.Where(p => deJureDuchy.DuchyContainsProvince(p.Id))
 						.Select(p => p.GetCulture(date, cultures))
 						.FirstOrDefault(c => c is not null);
 				}
 				if (culture is null && deJureDuchy?.DeJureLiege is not null) {
 					Logger.Debug($"Trying to use de jure kingdom for culture of holder for {county.Id}...");
 					var deJureKingdom = deJureDuchy.DeJureLiege;
 					culture = Provinces
 						.Where(p => deJureKingdom.KingdomContainsProvince(p.Id))
 						.Select(p => p.GetCulture(date, cultures))
 						.FirstOrDefault(c => c is not null);
 				}
 				if (culture is null) {
 					Logger.Warn($"Found no fitting culture for generated holder of {county.Id}, " +
 					            $"using first culture from database!");
 					culture = cultures.First();
 				}
 			}
 
 			// Determine faith of the holder.
 			var faithId = candidateProvinces
 				.Select(p => p.GetFaithId(date))
 				.FirstOrDefault(f => f is not null);
 			if (faithId is null) {
 				Logger.Debug($"Trying to use de jure duchy for faith of holder for {county.Id}...");
 				var deJureDuchy = county.DeJureLiege;
 				if (deJureDuchy is not null) {
 					faithId = Provinces
 						.Where(p => deJureDuchy.DuchyContainsProvince(p.Id))
 						.Select(p => p.GetFaithId(date))
 						.FirstOrDefault(f => f is not null);
 				}
 				if (faithId is null && deJureDuchy?.DeJureLiege is not null) {
 					Logger.Debug($"Trying to use de jure kingdom for faith of holder for {county.Id}...");
 					var deJureKingdom = deJureDuchy.DeJureLiege;
 					faithId = Provinces
 						.Where(p => deJureKingdom.KingdomContainsProvince(p.Id))
 						.Select(p => p.GetFaithId(date))
 						.FirstOrDefault(f => f is not null);
 				}
 				if (faithId is null) {
 					Logger.Warn($"Found no fitting faith for generated holder of {county.Id}, " +
 					            $"using first faith from database!");
 					faithId = Religions.Faiths.First().Id;
 				}
 			}
 
 			bool female = false;
 			string name;
 			var maleNames = culture.MaleNames.ToImmutableList();
 			if (maleNames.Count > 0) {
 				name = maleNames.ElementAt(pseudoRandomSeed % maleNames.Count);
 			} else { // Generate a female if no male name is available.
 				female = true;
 				var femaleNames = culture.FemaleNames.ToImmutableList();
 				name = femaleNames.ElementAt(pseudoRandomSeed % femaleNames.Count);
 			}
 			int age = 18 + (pseudoRandomSeed % 60);
 			var holder = new Character($"IRToCK3_{county.Id}_holder", name, date, Characters) {
 				FromImperator = true,
 				Female = female,
 				BirthDate = date.ChangeByYears(-age)
 			};
 			holder.SetFaithId(faithId, null);
 			holder.SetCultureId(culture.Id, null);
 			Characters.AddOrReplace(holder);
 
 			var countyHoldingTypes = county.CountyProvinceIds
 				.Select(id => Provinces.TryGetValue(id, out var province) ? province : null)
 				.Where(p => p is not null)
 				.Select(p => p!.GetHoldingType(date))
 				.Where(t => t is not null)
 				.Select(t => t!)
 				.ToHashSet();
 			string government = countyHoldingTypes.Contains("castle_holding")
 				? "feudal_government"
 				: "tribal_government";
 
 			county.SetHolder(holder, date);
 			if (config.FillerDukes) {
 				var duchy = county.DeJureLiege;
 				if (duchy is null || duchy.Rank != TitleRank.duchy) {
 					continue;
 				}
 
 				duchy.SetHolder(holder, date);
 				duchy.SetGovernment(government, date);
 				duchyIdToHolderDict[duchy.Id] = holder;
 			} else {
 				county.SetGovernment(government, date);
 			}
 		}
 	}
 
 	private void DetermineCK3Dlcs(Configuration config) {
 		var dlcFolderPath = Path.Join(config.CK3Path, "game/dlc");
 		if (!Directory.Exists(dlcFolderPath)) {

ImperatorToCK3/CommonUtils/IHistoryField.cs

@@ -30,11 +30,14 @@ public void RemoveAllEntries() {
 	/// Removes all entries with values matching the predicate
 	/// </summary>
 	/// <param name="predicate"></param>
-	public void RemoveAllEntries(Func<object, bool> predicate) {
-		InitialEntries.RemoveAll(kvp => predicate(kvp.Value));
+	public int RemoveAllEntries(Func<object, bool> predicate) {
+		int removed = 0;
+		removed += InitialEntries.RemoveAll(kvp => predicate(kvp.Value));
 		foreach (var datedEntriesBlock in DateToEntriesDict) {
-			datedEntriesBlock.Value.RemoveAll(kvp => predicate(kvp.Value));
+			removed += datedEntriesBlock.Value.RemoveAll(kvp => predicate(kvp.Value));
 		}
+
+		return removed;
 	}
 
 	public void RegisterKeywords(Parser parser, Date date);