/* Yarn Spinner is licensed to you under the terms found in the file LICENSE.md. */ using System.Collections.Generic; using System.IO; using System.Linq; using UnityEditor; using UnityEditor.Callbacks; using UnityEngine; using Yarn.Unity; #if USE_ADDRESSABLES using UnityEditor.AddressableAssets; using UnityEditor.AddressableAssets.Settings; #endif #if !UNITY_6000_4_OR_NEWER // EntityId was introduced in Unity 6.4 to replace integer-based instance IDs. // We'll use a type alias so that versions earlier than that can keep working // with the older APIs. using EntityId = System.Int32; #endif #nullable enable namespace Yarn.Unity.Editor { /// /// Contains methods for performing high-level operations on Yarn projects, /// and their associated localization files. /// internal static class YarnProjectUtility { /// /// Creates a new .yarnproject asset in the same directory as the Yarn /// script represented by , and /// configures the script's importer to use the new Yarn Project. /// /// An importer for an existing Yarn /// script. /// The path to the created asset, relative to the Unity /// project root. internal static string CreateYarnProject(YarnImporter initialSourceAsset) { // Figure out where on disk this asset is var path = initialSourceAsset.assetPath; var directory = Path.GetDirectoryName(path); // Figure out a new, unique path for the localization we're creating var databaseFileName = $"Project.yarnproject"; var destinationPath = Path.Combine(directory, databaseFileName); destinationPath = AssetDatabase.GenerateUniqueAssetPath(destinationPath); // Create the program var newProject = YarnProjectUtility.CreateDefaultYarnProject(); newProject.SaveToFile(destinationPath); AssetDatabase.ImportAsset(destinationPath); AssetDatabase.SaveAssets(); return destinationPath; } /// /// Creates a Unity tweaked default Yarn Project. /// /// /// This is just a default Yarn Project with the exclusion file pattern /// set up to ignore ~ folders. /// /// A Unity default Yarn Project internal static Yarn.Compiler.Project CreateDefaultYarnProject() { // Create the program var newProject = new Yarn.Compiler.Project(); // Follow Unity's behaviour - exclude any content in a folder whose // name ends with a tilde // and also ignoring anything that is inside a sample folder newProject.ExcludeFilePatterns = new[] { "**/*~/*", "./Samples/Yarn Spinner*/*" }; return newProject; } /// /// Updates every localization .CSV file associated with this /// .yarnproject file. /// /// /// This method updates each localization file by performing the /// following operations: /// /// Inserts new entries if they're present in the base /// localization and not in the translated localization /// /// Removes entries if they're present in the translated /// localization and not in the base localization /// /// Detects if a line in the base localization has changed its /// Lock value from when the translated localization was created, and /// update its Comment /// /// An importer for an existing Yarn /// script. /// The path to the created asset, relative to the Unity /// project root. internal static void UpdateLocalizationCSVs(YarnProjectImporter yarnProjectImporter) { if (yarnProjectImporter.CanGenerateStringsTable == false) { Debug.LogError($"Can't update localization CSVs for Yarn Project \"{yarnProjectImporter.name}\" because not every line has a tag."); return; } var importData = yarnProjectImporter.ImportData; if (importData == null) { Debug.LogError($"Can't update localization CSVs for Yarn Project \"{yarnProjectImporter.name}\" because it failed to compile."); return; } var job = yarnProjectImporter.GetCompilationJob(); job.CompilationType = Compiler.CompilationJob.Type.StringsOnly; var result = Compiler.Compiler.Compile(job); var baseLocalizationStrings = yarnProjectImporter.GenerateStringsTable(result); if (baseLocalizationStrings == null) { Debug.LogError($"Can't update localization CSVs for Yarn Project \"{yarnProjectImporter.name}\" because it failed to compile."); return; } var localizations = importData.localizations; var modifiedFiles = new List(); try { AssetDatabase.StartAssetEditing(); foreach (var loc in localizations) { if (loc.languageID == importData.baseLanguageName) { // This is the base language - no strings file to // update. continue; } if (loc.stringsFile == null) { Debug.LogWarning($"Can't update localization for {loc.languageID} because it doesn't have a strings file.", yarnProjectImporter); continue; } var fileWasChanged = UpdateLocalizationFile(baseLocalizationStrings, loc.languageID, loc.stringsFile); if (fileWasChanged) { modifiedFiles.Add(loc.stringsFile); } } if (modifiedFiles.Count > 0) { Debug.Log($"Updated the following files: {string.Join(", ", modifiedFiles.Select(f => AssetDatabase.GetAssetPath(f)))}"); } else { Debug.Log($"No files needed updating."); } } finally { AssetDatabase.StopAssetEditing(); } } internal static void UpdateAssetAddresses(YarnProjectImporter importer) { #if USE_ADDRESSABLES var job = importer.GetCompilationJob(); job.CompilationType = Compiler.CompilationJob.Type.StringsOnly; var result = Compiler.Compiler.Compile(job); var lineIDs = importer.GenerateStringsTable(result).Select(s => s.ID); if (importer.ImportData == null) { throw new System.InvalidOperationException($"Can't update asset addresses: importer has no {nameof(importer.ImportData)}"); } // Get a map of language IDs to (lineID, asset path) pairs var languageToAssets = importer // Get the languages-to-source-assets map .ImportData.localizations // Get the asset folder for them .Select(l => new { l.languageID, l.assetsFolder }) // Only consider those that have an asset folder .Where(f => f.assetsFolder != null) // Get the path for the asset folder .Select(f => new { f.languageID, path = AssetDatabase.GetAssetPath(f.assetsFolder) }) // Use that to get the assets inside these folders .Select(f => new { f.languageID, assetPaths = FindAssetPathsForLineIDs(lineIDs, f.path, typeof(UnityEngine.Object)) }); var addressableAssetSettings = AddressableAssetSettingsDefaultObject.Settings; foreach (var languageToAsset in languageToAssets) { var assets = languageToAsset.assetPaths .Select(pair => new { LineID = pair.Key, GUID = AssetDatabase.AssetPathToGUID(pair.Value) }); foreach (var asset in assets) { // Find the existing entry for this asset, if it has one. AddressableAssetEntry entry = addressableAssetSettings.FindAssetEntry(asset.GUID); if (entry == null) { // This asset didn't have an entry. Create one in the // default group. entry = addressableAssetSettings.CreateOrMoveEntry(asset.GUID, addressableAssetSettings.DefaultGroup); } // Update the entry's address. entry.SetAddress(Localization.GetAddressForLine(asset.LineID, languageToAsset.languageID)); } } #else throw new System.NotSupportedException($"A method that requires the Addressable Assets package was called, but USE_ADDRESSABLES was not defined. Please either install Addressable Assets, or if you have already, add it to this project's compiler definitions."); #endif } internal static Dictionary FindAssetPathsForLineIDs(IEnumerable lineIDs, string assetsFolderPath, System.Type assetType) { // Find _all_ files in this director that are not .meta files and // whose main asset is equal to (or derived from) assetType var allFiles = Directory.EnumerateFiles(assetsFolderPath, "*", SearchOption.AllDirectories) .Where(path => path.EndsWith(".meta") == false) .Where(path => assetType.IsAssignableFrom(AssetDatabase.GetMainAssetTypeAtPath(path))); // Match files with those whose filenames contain a line ID // If a direct file match is found prefer that Dictionary assets = new(); foreach (var lineID in lineIDs) { var lineIDWithoutPrefix = lineID.Replace("line:", "").ToLowerInvariant(); var candidates = new List(); foreach (var asset in allFiles) { var file = Path.GetFileNameWithoutExtension(asset).ToLowerInvariant(); if (file == lineIDWithoutPrefix) { assets[lineID] = asset; break; } if (file.Contains(lineIDWithoutPrefix)) { candidates.Add(asset); } } var count = candidates.Count(); if (count > 0) { assets[lineID] = candidates.FirstOrDefault(); if (count > 1) { Debug.LogWarning($"Discovered {count} candidates for {lineID}. Selecting one.\nCandidates:\n" + string.Join("\n", candidates)); } } } return assets; } /// /// Verifies the TextAsset referred to by , and updates it if necessary. /// /// A collection of /// The language that provides strings /// for.false /// A TextAsset containing /// localized strings in CSV format. /// Whether was /// modified. private static bool UpdateLocalizationFile(IEnumerable baseLocalizationStrings, string language, TextAsset destinationLocalizationAsset) { var translatedStrings = StringTableEntry.ParseFromCSV(destinationLocalizationAsset.text); // Convert both enumerables to dictionaries, for easier lookup var baseDictionary = baseLocalizationStrings.ToDictionary(entry => entry.ID); var translatedDictionary = translatedStrings.ToDictionary(entry => entry.ID); // The list of line IDs present in each localisation var baseIDs = baseLocalizationStrings.Select(entry => entry.ID); var translatedIDs = translatedStrings.Select(entry => entry.ID); // The list of line IDs that are ONLY present in each localisation var onlyInBaseIDs = baseIDs.Except(translatedIDs); var onlyInTranslatedIDs = translatedIDs.Except(baseIDs); // Tracks if the translated localisation needed modifications // (either new lines added, old lines removed, or changed lines // flagged) var modificationsNeeded = false; // Remove every entry whose ID is only present in the translated // set. This entry has been removed from the base localization. foreach (var id in onlyInTranslatedIDs.ToList()) { translatedDictionary.Remove(id); modificationsNeeded = true; } // Conversely, for every entry that is only present in the base // localisation, we need to create a new entry for it. foreach (var id in onlyInBaseIDs) { StringTableEntry baseEntry = baseDictionary[id]; var newEntry = new StringTableEntry(baseEntry) { // Empty this text, so that it's apparent that a translated // version needs to be provided. Text = string.Empty, Language = language, }; translatedDictionary.Add(id, newEntry); modificationsNeeded = true; } // Finally, we need to check for any entries in the translated // localisation that: // 1. have the same line ID as one in the base, but // 2. have a different Lock (the hash of the text), which indicates // that the base text has changed. // First, get the list of IDs that are in both base and translated, // and then filter this list to any where the lock values differ var outOfDateLockIDs = baseDictionary.Keys .Intersect(translatedDictionary.Keys) .Where(id => baseDictionary[id].Lock != translatedDictionary[id].Lock); // Now loop over all of these, and update our translated dictionary // to include a note that it needs attention foreach (var id in outOfDateLockIDs) { // Get the translated entry as it currently exists var entry = translatedDictionary[id]; // Include a note that this entry is out of date entry.Text = $"(NEEDS UPDATE) {entry.Text}"; // Update the lock to match the new one entry.Lock = baseDictionary[id].Lock; // Put this modified entry back in the table translatedDictionary[id] = entry; modificationsNeeded = true; } // We're all done! if (modificationsNeeded == false) { // No changes needed to be done to the translated string table // entries. Stop here. return false; } // We need to produce a replacement CSV file for the translated // entries. var outputStringEntries = translatedDictionary.Values .OrderBy(entry => entry.File) .ThenBy(entry => int.Parse(entry.LineNumber)); var outputCSV = StringTableEntry.CreateCSV(outputStringEntries); // Write out the replacement text to this existing file, replacing // its existing contents var outputFile = AssetDatabase.GetAssetPath(destinationLocalizationAsset); File.WriteAllText(outputFile, outputCSV, System.Text.Encoding.UTF8); // Tell the asset database that the file needs to be reimported AssetDatabase.ImportAsset(outputFile); // Signal that the file was changed return true; } private static (List AllExistingTags, List ProjectImplicitTags) ExtantLineTags(YarnProjectImporter importer) { // First, gather all existing line tags across ALL yarn projects, so // that we don't accidentally overwrite an existing one. Do this by // finding all yarn projects, and get the string tags inside them. // By doing it in this way we get the same implicit tags from the // project as the importer would normally do, letting us then do a // direct comparision for them. var allYarnProjects = // get all yarn projects across the entire project AssetDatabase.FindAssets($"t:{nameof(YarnProject)}") // Get the path for each asset's GUID .Select(guid => AssetDatabase.GUIDToAssetPath(guid)) // Get the importer for each asset at this path .Select(path => AssetImporter.GetAtPath(path)) // Ensure it's a YarnProjectImporter .OfType() // Ensure that its import data is present .Where(i => i.ImportData != null) // get the project out, and also flag if it is the project for // THIS importer .Select(i => (Project: i.GetProject()!, IsThisImporter: i == importer)) // remove any nulls just in case any are found .Where(p => p.Project != null); #if YARNSPINNER_DEBUG var stopwatch = System.Diagnostics.Stopwatch.StartNew(); #endif var allExistingTags = new List(); var projectImplicitTags = new List(); // Compile all of these, and get whatever existing string tags they // had. Do each in isolation so that we can continue even if a // project contains a parse error. foreach (var (Project, IsThisImporter) in allYarnProjects) { var project = Project; var compilationJob = Yarn.Compiler.CompilationJob.CreateFromFiles(project.SourceFiles); compilationJob.CompilationType = Yarn.Compiler.CompilationJob.Type.StringsOnly; var result = Yarn.Compiler.Compiler.Compile(compilationJob); bool containsErrors = result.Diagnostics.Any(d => d.Severity == Compiler.Diagnostic.DiagnosticSeverity.Error); if (containsErrors) { Debug.LogWarning($"{project} has errors so cannot be scanned for tagging."); continue; } allExistingTags.AddRange(result.StringTable.Where(i => i.Value.isImplicitTag == false).Select(i => i.Key)); // we add the implicit lines IDs only for this project if (IsThisImporter) { projectImplicitTags.AddRange(result.StringTable.Where(i => i.Value.isImplicitTag == true).Select(i => i.Key)); } } #if YARNSPINNER_DEBUG stopwatch.Stop(); Debug.Log($"Checked {allYarnProjects.Count()} yarn files for line tags in {stopwatch.ElapsedMilliseconds}ms"); #endif return (allExistingTags, projectImplicitTags); } public static void AddLineTagsToFilesInYarnProject(YarnProjectImporter importer) { var (AllExistingTags, ProjectImplicitTags) = YarnProjectUtility.ExtantLineTags(importer); #if USE_UNITY_LOCALIZATION // if we are using Unity localisation we need to first remove the // implicit tags for this project from the strings table if (importer.UseUnityLocalisationSystem && importer.UnityLocalisationStringTableCollection != null) { foreach (var implicitTag in ProjectImplicitTags) { importer.UnityLocalisationStringTableCollection.RemoveEntry(implicitTag); } } #endif if (importer.ImportData == null) { Debug.LogError($"Can't add line tags to {importer.assetPath}, because it failed to compile."); return; } var modifiedFiles = new List(); try { AssetDatabase.StartAssetEditing(); foreach (var script in importer.ImportData.yarnFiles) { var assetPath = AssetDatabase.GetAssetPath(script); var contents = File.ReadAllText(assetPath); // Produce a version of this file that contains line tags // added where they're needed. var tagged = Yarn.Compiler.Utility.TagLines(contents, AllExistingTags ?? new List()); var taggedVersion = tagged.Item1; // if the file has an error it returns null we want to bail // out then otherwise we'd wipe the yarn file if (taggedVersion == null) { continue; } // If this produced a modified version of the file, write it // out and re-import it. if (contents != taggedVersion) { modifiedFiles.Add(Path.GetFileNameWithoutExtension(assetPath)); File.WriteAllText(assetPath, taggedVersion, System.Text.Encoding.UTF8); AssetDatabase.ImportAsset(assetPath, ImportAssetOptions.Default); AllExistingTags = tagged.Item2 as List; } } } catch (System.Exception e) { Debug.LogError($"Encountered an error when updating scripts: {e}"); return; } finally { AssetDatabase.StopAssetEditing(); } // Report on the work we did. if (modifiedFiles.Count > 0) { Debug.Log($"Updated the following files: {string.Join(", ", modifiedFiles)}"); } else { Debug.Log("No files needed updating."); } } /// /// Writes a .csv file to disk at the path indicated by , containing all of the lines found in the /// scripts referred to by . /// /// The YarnProjectImporter to extract /// strings from. /// The path to write the file to. /// if the file was written /// successfully, otherwise. /// Thrown when an error /// is encountered when generating the CSV data. /// Thrown when an error is encountered /// when writing the data to disk. internal static bool WriteStringsFile(string destination, YarnProjectImporter yarnProjectImporter) { // Perform a strings-only compilation to get a full strings table, // and generate the CSV. var job = yarnProjectImporter.GetCompilationJob(); job.CompilationType = Compiler.CompilationJob.Type.StringsOnly; var result = Compiler.Compiler.Compile(job); if (result.ContainsErrors) { // The project contains errors. Bail out. return false; } var stringTable = yarnProjectImporter.GenerateStringsTable(result); // If there was an error, bail out here if (stringTable == null) { return false; } // Convert the string tables to CSV... var outputCSV = StringTableEntry.CreateCSV(stringTable); // ...and write it to disk. File.WriteAllText(destination, outputCSV); return true; } /// /// Writes a .csv file to disk at the path indicated by , containing all of the lines found in the /// scripts referred to by that /// contain any metadata associated with them. /// /// The YarnProjectImporter to extract /// strings from. /// The path to write the file to. /// if the file was written /// successfully, otherwise. /// Thrown when an error /// is encountered when generating the CSV data. /// Thrown when an error is encountered /// when writing the data to disk. internal static bool WriteMetadataFile(string destination, YarnProjectImporter yarnProjectImporter) { var lineMetadataEntries = yarnProjectImporter.GenerateLineMetadataEntries(); // If there was an error, bail out here. if (lineMetadataEntries == null) { return false; } var outputCSV = LineMetadataTableEntry.CreateCSV(lineMetadataEntries); File.WriteAllText(destination, outputCSV); return true; } /// /// Upgrades an old-style Yarn Project to JSON. /// /// /// This method copies the text of the project to a new file adjacent to /// the project, and replaces the text of the project with a new empty /// JSON project. /// /// A YarnProjectImporter that represents the /// project that needs to be upgraded. internal static void UpgradeYarnProject(YarnProjectImporter importer) { // We need to copy out the variable declarations from the old Yarn // project before we replace it. // Get the current text of the old project var existingText = File.ReadAllText(importer.assetPath); // Does the existing text contain anything besides the default? var defaultProjectPattern = new System.Text.RegularExpressions.Regex(@"^title:.*?\n---[\n\s]*===[\n\s]*$", System.Text.RegularExpressions.RegexOptions.Multiline); if (defaultProjectPattern.IsMatch(existingText)) { // The project contains no content, so there's no need to copy // it out. } else { // Create a unique path to store our variables var newFilePath = Path.GetDirectoryName(importer.assetPath) + "/Variables.yarn"; newFilePath = AssetDatabase.GenerateUniqueAssetPath(newFilePath); // Write it out to the new file File.WriteAllText(newFilePath, existingText); } // Next, replace the existing project with a new one! var newProject = YarnProjectUtility.CreateDefaultYarnProject(); File.WriteAllText(importer.assetPath, newProject.GetJson()); // Finally, import the assets we've touched. AssetDatabase.Refresh(); } [OnOpenAsset(OnOpenAssetAttributeMode.Execute)] public static bool OnOpenAsset(EntityId instanceID) { // temporarily disabling the obsolete warning for the GetAssetPath call // but only for Unity <6.4, otherwise we want the warning // because this code needs to exist across multiple unity versions // and it's only an actual concern on 6.4+ we can disable it when earlier #if !UNITY_6000_4_OR_NEWER #pragma warning disable 0618 #endif var path = AssetDatabase.GetAssetPath(instanceID); #if !UNITY_6000_4_OR_NEWER #pragma warning restore 0618 #endif var project = AssetDatabase.LoadAssetAtPath(path); if (project == null) { return false; } var importer = AssetImporter.GetAtPath(path) as Yarn.Unity.Editor.YarnProjectImporter; if (importer == null) { return false; } var yp = Yarn.Compiler.Project.LoadFromFile(path); var files = yp.SourceFiles; if (files.Any()) { UnityEditorInternal.InternalEditorUtility.OpenFileAtLineExternal(System.IO.Path.GetDirectoryName(files.First()), 0); } return true; } [OnOpenAsset(OnOpenAssetAttributeMode.Validate)] public static bool OnValidateAsset(EntityId instanceID) { // temporarily disabling the obsolete warning for the GetAssetPath call // but only for Unity <6.4, otherwise we want the warning // because this code needs to exist across multiple unity versions // and it's only an actual concern on 6.4+ we can disable it when earlier #if !UNITY_6000_4_OR_NEWER #pragma warning disable 0618 #endif var path = AssetDatabase.GetAssetPath(instanceID); #if !UNITY_6000_4_OR_NEWER #pragma warning restore 0618 #endif var project = AssetDatabase.LoadAssetAtPath(path); if (project == null) { return false; } return true; } } }