/*
Yarn Spinner is licensed to you under the terms found in the file LICENSE.md.
*/
using System.Collections.Generic;
using System.IO;
using System.Linq;
using UnityEditor;
using UnityEditor.Callbacks;
using UnityEngine;
using Yarn.Unity;
#if USE_ADDRESSABLES
using UnityEditor.AddressableAssets;
using UnityEditor.AddressableAssets.Settings;
#endif
#if !UNITY_6000_4_OR_NEWER
// EntityId was introduced in Unity 6.4 to replace integer-based instance IDs.
// We'll use a type alias so that versions earlier than that can keep working
// with the older APIs.
using EntityId = System.Int32;
#endif
#nullable enable
namespace Yarn.Unity.Editor
{
///
/// Contains methods for performing high-level operations on Yarn projects,
/// and their associated localization files.
///
internal static class YarnProjectUtility
{
///
/// Creates a new .yarnproject asset in the same directory as the Yarn
/// script represented by , and
/// configures the script's importer to use the new Yarn Project.
///
/// An importer for an existing Yarn
/// script.
/// The path to the created asset, relative to the Unity
/// project root.
internal static string CreateYarnProject(YarnImporter initialSourceAsset)
{
// Figure out where on disk this asset is
var path = initialSourceAsset.assetPath;
var directory = Path.GetDirectoryName(path);
// Figure out a new, unique path for the localization we're creating
var databaseFileName = $"Project.yarnproject";
var destinationPath = Path.Combine(directory, databaseFileName);
destinationPath = AssetDatabase.GenerateUniqueAssetPath(destinationPath);
// Create the program
var newProject = YarnProjectUtility.CreateDefaultYarnProject();
newProject.SaveToFile(destinationPath);
AssetDatabase.ImportAsset(destinationPath);
AssetDatabase.SaveAssets();
return destinationPath;
}
///
/// Creates a Unity tweaked default Yarn Project.
///
///
/// This is just a default Yarn Project with the exclusion file pattern
/// set up to ignore ~ folders.
///
/// A Unity default Yarn Project
internal static Yarn.Compiler.Project CreateDefaultYarnProject()
{
// Create the program
var newProject = new Yarn.Compiler.Project();
// Follow Unity's behaviour - exclude any content in a folder whose
// name ends with a tilde
// and also ignoring anything that is inside a sample folder
newProject.ExcludeFilePatterns = new[] { "**/*~/*", "./Samples/Yarn Spinner*/*" };
return newProject;
}
///
/// Updates every localization .CSV file associated with this
/// .yarnproject file.
///
///
/// This method updates each localization file by performing the
/// following operations:
///
/// - Inserts new entries if they're present in the base
/// localization and not in the translated localization
///
/// - Removes entries if they're present in the translated
/// localization and not in the base localization
///
/// - Detects if a line in the base localization has changed its
/// Lock value from when the translated localization was created, and
/// update its Comment
///
/// An importer for an existing Yarn
/// script.
/// The path to the created asset, relative to the Unity
/// project root.
internal static void UpdateLocalizationCSVs(YarnProjectImporter yarnProjectImporter)
{
if (yarnProjectImporter.CanGenerateStringsTable == false)
{
Debug.LogError($"Can't update localization CSVs for Yarn Project \"{yarnProjectImporter.name}\" because not every line has a tag.");
return;
}
var importData = yarnProjectImporter.ImportData;
if (importData == null)
{
Debug.LogError($"Can't update localization CSVs for Yarn Project \"{yarnProjectImporter.name}\" because it failed to compile.");
return;
}
var job = yarnProjectImporter.GetCompilationJob();
job.CompilationType = Compiler.CompilationJob.Type.StringsOnly;
var result = Compiler.Compiler.Compile(job);
var baseLocalizationStrings = yarnProjectImporter.GenerateStringsTable(result);
if (baseLocalizationStrings == null)
{
Debug.LogError($"Can't update localization CSVs for Yarn Project \"{yarnProjectImporter.name}\" because it failed to compile.");
return;
}
var localizations = importData.localizations;
var modifiedFiles = new List();
try
{
AssetDatabase.StartAssetEditing();
foreach (var loc in localizations)
{
if (loc.languageID == importData.baseLanguageName)
{
// This is the base language - no strings file to
// update.
continue;
}
if (loc.stringsFile == null)
{
Debug.LogWarning($"Can't update localization for {loc.languageID} because it doesn't have a strings file.", yarnProjectImporter);
continue;
}
var fileWasChanged = UpdateLocalizationFile(baseLocalizationStrings, loc.languageID, loc.stringsFile);
if (fileWasChanged)
{
modifiedFiles.Add(loc.stringsFile);
}
}
if (modifiedFiles.Count > 0)
{
Debug.Log($"Updated the following files: {string.Join(", ", modifiedFiles.Select(f => AssetDatabase.GetAssetPath(f)))}");
}
else
{
Debug.Log($"No files needed updating.");
}
}
finally
{
AssetDatabase.StopAssetEditing();
}
}
internal static void UpdateAssetAddresses(YarnProjectImporter importer)
{
#if USE_ADDRESSABLES
var job = importer.GetCompilationJob();
job.CompilationType = Compiler.CompilationJob.Type.StringsOnly;
var result = Compiler.Compiler.Compile(job);
var lineIDs = importer.GenerateStringsTable(result).Select(s => s.ID);
if (importer.ImportData == null)
{
throw new System.InvalidOperationException($"Can't update asset addresses: importer has no {nameof(importer.ImportData)}");
}
// Get a map of language IDs to (lineID, asset path) pairs
var languageToAssets = importer
// Get the languages-to-source-assets map
.ImportData.localizations
// Get the asset folder for them
.Select(l => new { l.languageID, l.assetsFolder })
// Only consider those that have an asset folder
.Where(f => f.assetsFolder != null)
// Get the path for the asset folder
.Select(f => new { f.languageID, path = AssetDatabase.GetAssetPath(f.assetsFolder) })
// Use that to get the assets inside these folders
.Select(f => new { f.languageID, assetPaths = FindAssetPathsForLineIDs(lineIDs, f.path, typeof(UnityEngine.Object)) });
var addressableAssetSettings = AddressableAssetSettingsDefaultObject.Settings;
foreach (var languageToAsset in languageToAssets)
{
var assets = languageToAsset.assetPaths
.Select(pair => new { LineID = pair.Key, GUID = AssetDatabase.AssetPathToGUID(pair.Value) });
foreach (var asset in assets)
{
// Find the existing entry for this asset, if it has one.
AddressableAssetEntry entry = addressableAssetSettings.FindAssetEntry(asset.GUID);
if (entry == null)
{
// This asset didn't have an entry. Create one in the
// default group.
entry = addressableAssetSettings.CreateOrMoveEntry(asset.GUID, addressableAssetSettings.DefaultGroup);
}
// Update the entry's address.
entry.SetAddress(Localization.GetAddressForLine(asset.LineID, languageToAsset.languageID));
}
}
#else
throw new System.NotSupportedException($"A method that requires the Addressable Assets package was called, but USE_ADDRESSABLES was not defined. Please either install Addressable Assets, or if you have already, add it to this project's compiler definitions.");
#endif
}
internal static Dictionary FindAssetPathsForLineIDs(IEnumerable lineIDs, string assetsFolderPath, System.Type assetType)
{
// Find _all_ files in this director that are not .meta files and
// whose main asset is equal to (or derived from) assetType
var allFiles = Directory.EnumerateFiles(assetsFolderPath, "*", SearchOption.AllDirectories)
.Where(path => path.EndsWith(".meta") == false)
.Where(path => assetType.IsAssignableFrom(AssetDatabase.GetMainAssetTypeAtPath(path)));
// Match files with those whose filenames contain a line ID
// If a direct file match is found prefer that
Dictionary assets = new();
foreach (var lineID in lineIDs)
{
var lineIDWithoutPrefix = lineID.Replace("line:", "").ToLowerInvariant();
var candidates = new List();
foreach (var asset in allFiles)
{
var file = Path.GetFileNameWithoutExtension(asset).ToLowerInvariant();
if (file == lineIDWithoutPrefix)
{
assets[lineID] = asset;
break;
}
if (file.Contains(lineIDWithoutPrefix))
{
candidates.Add(asset);
}
}
var count = candidates.Count();
if (count > 0)
{
assets[lineID] = candidates.FirstOrDefault();
if (count > 1)
{
Debug.LogWarning($"Discovered {count} candidates for {lineID}. Selecting one.\nCandidates:\n" + string.Join("\n", candidates));
}
}
}
return assets;
}
///
/// Verifies the TextAsset referred to by , and updates it if necessary.
///
/// A collection of
/// The language that provides strings
/// for.false
/// A TextAsset containing
/// localized strings in CSV format.
/// Whether was
/// modified.
private static bool UpdateLocalizationFile(IEnumerable baseLocalizationStrings, string language, TextAsset destinationLocalizationAsset)
{
var translatedStrings = StringTableEntry.ParseFromCSV(destinationLocalizationAsset.text);
// Convert both enumerables to dictionaries, for easier lookup
var baseDictionary = baseLocalizationStrings.ToDictionary(entry => entry.ID);
var translatedDictionary = translatedStrings.ToDictionary(entry => entry.ID);
// The list of line IDs present in each localisation
var baseIDs = baseLocalizationStrings.Select(entry => entry.ID);
var translatedIDs = translatedStrings.Select(entry => entry.ID);
// The list of line IDs that are ONLY present in each localisation
var onlyInBaseIDs = baseIDs.Except(translatedIDs);
var onlyInTranslatedIDs = translatedIDs.Except(baseIDs);
// Tracks if the translated localisation needed modifications
// (either new lines added, old lines removed, or changed lines
// flagged)
var modificationsNeeded = false;
// Remove every entry whose ID is only present in the translated
// set. This entry has been removed from the base localization.
foreach (var id in onlyInTranslatedIDs.ToList())
{
translatedDictionary.Remove(id);
modificationsNeeded = true;
}
// Conversely, for every entry that is only present in the base
// localisation, we need to create a new entry for it.
foreach (var id in onlyInBaseIDs)
{
StringTableEntry baseEntry = baseDictionary[id];
var newEntry = new StringTableEntry(baseEntry)
{
// Empty this text, so that it's apparent that a translated
// version needs to be provided.
Text = string.Empty,
Language = language,
};
translatedDictionary.Add(id, newEntry);
modificationsNeeded = true;
}
// Finally, we need to check for any entries in the translated
// localisation that:
// 1. have the same line ID as one in the base, but
// 2. have a different Lock (the hash of the text), which indicates
// that the base text has changed.
// First, get the list of IDs that are in both base and translated,
// and then filter this list to any where the lock values differ
var outOfDateLockIDs = baseDictionary.Keys
.Intersect(translatedDictionary.Keys)
.Where(id => baseDictionary[id].Lock != translatedDictionary[id].Lock);
// Now loop over all of these, and update our translated dictionary
// to include a note that it needs attention
foreach (var id in outOfDateLockIDs)
{
// Get the translated entry as it currently exists
var entry = translatedDictionary[id];
// Include a note that this entry is out of date
entry.Text = $"(NEEDS UPDATE) {entry.Text}";
// Update the lock to match the new one
entry.Lock = baseDictionary[id].Lock;
// Put this modified entry back in the table
translatedDictionary[id] = entry;
modificationsNeeded = true;
}
// We're all done!
if (modificationsNeeded == false)
{
// No changes needed to be done to the translated string table
// entries. Stop here.
return false;
}
// We need to produce a replacement CSV file for the translated
// entries.
var outputStringEntries = translatedDictionary.Values
.OrderBy(entry => entry.File)
.ThenBy(entry => int.Parse(entry.LineNumber));
var outputCSV = StringTableEntry.CreateCSV(outputStringEntries);
// Write out the replacement text to this existing file, replacing
// its existing contents
var outputFile = AssetDatabase.GetAssetPath(destinationLocalizationAsset);
File.WriteAllText(outputFile, outputCSV, System.Text.Encoding.UTF8);
// Tell the asset database that the file needs to be reimported
AssetDatabase.ImportAsset(outputFile);
// Signal that the file was changed
return true;
}
private static (List AllExistingTags, List ProjectImplicitTags) ExtantLineTags(YarnProjectImporter importer)
{
// First, gather all existing line tags across ALL yarn projects, so
// that we don't accidentally overwrite an existing one. Do this by
// finding all yarn projects, and get the string tags inside them.
// By doing it in this way we get the same implicit tags from the
// project as the importer would normally do, letting us then do a
// direct comparision for them.
var allYarnProjects =
// get all yarn projects across the entire project
AssetDatabase.FindAssets($"t:{nameof(YarnProject)}")
// Get the path for each asset's GUID
.Select(guid => AssetDatabase.GUIDToAssetPath(guid))
// Get the importer for each asset at this path
.Select(path => AssetImporter.GetAtPath(path))
// Ensure it's a YarnProjectImporter
.OfType()
// Ensure that its import data is present
.Where(i => i.ImportData != null)
// get the project out, and also flag if it is the project for
// THIS importer
.Select(i => (Project: i.GetProject()!, IsThisImporter: i == importer))
// remove any nulls just in case any are found
.Where(p => p.Project != null);
#if YARNSPINNER_DEBUG
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
#endif
var allExistingTags = new List();
var projectImplicitTags = new List();
// Compile all of these, and get whatever existing string tags they
// had. Do each in isolation so that we can continue even if a
// project contains a parse error.
foreach (var (Project, IsThisImporter) in allYarnProjects)
{
var project = Project;
var compilationJob = Yarn.Compiler.CompilationJob.CreateFromFiles(project.SourceFiles);
compilationJob.CompilationType = Yarn.Compiler.CompilationJob.Type.StringsOnly;
var result = Yarn.Compiler.Compiler.Compile(compilationJob);
bool containsErrors = result.Diagnostics.Any(d => d.Severity == Compiler.Diagnostic.DiagnosticSeverity.Error);
if (containsErrors)
{
Debug.LogWarning($"{project} has errors so cannot be scanned for tagging.");
continue;
}
allExistingTags.AddRange(result.StringTable.Where(i => i.Value.isImplicitTag == false).Select(i => i.Key));
// we add the implicit lines IDs only for this project
if (IsThisImporter)
{
projectImplicitTags.AddRange(result.StringTable.Where(i => i.Value.isImplicitTag == true).Select(i => i.Key));
}
}
#if YARNSPINNER_DEBUG
stopwatch.Stop();
Debug.Log($"Checked {allYarnProjects.Count()} yarn files for line tags in {stopwatch.ElapsedMilliseconds}ms");
#endif
return (allExistingTags, projectImplicitTags);
}
public static void AddLineTagsToFilesInYarnProject(YarnProjectImporter importer)
{
var (AllExistingTags, ProjectImplicitTags) = YarnProjectUtility.ExtantLineTags(importer);
#if USE_UNITY_LOCALIZATION
// if we are using Unity localisation we need to first remove the
// implicit tags for this project from the strings table
if (importer.UseUnityLocalisationSystem && importer.UnityLocalisationStringTableCollection != null)
{
foreach (var implicitTag in ProjectImplicitTags)
{
importer.UnityLocalisationStringTableCollection.RemoveEntry(implicitTag);
}
}
#endif
if (importer.ImportData == null)
{
Debug.LogError($"Can't add line tags to {importer.assetPath}, because it failed to compile.");
return;
}
var modifiedFiles = new List();
try
{
AssetDatabase.StartAssetEditing();
foreach (var script in importer.ImportData.yarnFiles)
{
var assetPath = AssetDatabase.GetAssetPath(script);
var contents = File.ReadAllText(assetPath);
// Produce a version of this file that contains line tags
// added where they're needed.
var tagged = Yarn.Compiler.Utility.TagLines(contents, AllExistingTags ?? new List());
var taggedVersion = tagged.Item1;
// if the file has an error it returns null we want to bail
// out then otherwise we'd wipe the yarn file
if (taggedVersion == null)
{
continue;
}
// If this produced a modified version of the file, write it
// out and re-import it.
if (contents != taggedVersion)
{
modifiedFiles.Add(Path.GetFileNameWithoutExtension(assetPath));
File.WriteAllText(assetPath, taggedVersion, System.Text.Encoding.UTF8);
AssetDatabase.ImportAsset(assetPath, ImportAssetOptions.Default);
AllExistingTags = tagged.Item2 as List;
}
}
}
catch (System.Exception e)
{
Debug.LogError($"Encountered an error when updating scripts: {e}");
return;
}
finally
{
AssetDatabase.StopAssetEditing();
}
// Report on the work we did.
if (modifiedFiles.Count > 0)
{
Debug.Log($"Updated the following files: {string.Join(", ", modifiedFiles)}");
}
else
{
Debug.Log("No files needed updating.");
}
}
///
/// Writes a .csv file to disk at the path indicated by , containing all of the lines found in the
/// scripts referred to by .
///
/// The YarnProjectImporter to extract
/// strings from.
/// The path to write the file to.
/// if the file was written
/// successfully, otherwise.
/// Thrown when an error
/// is encountered when generating the CSV data.
/// Thrown when an error is encountered
/// when writing the data to disk.
internal static bool WriteStringsFile(string destination, YarnProjectImporter yarnProjectImporter)
{
// Perform a strings-only compilation to get a full strings table,
// and generate the CSV.
var job = yarnProjectImporter.GetCompilationJob();
job.CompilationType = Compiler.CompilationJob.Type.StringsOnly;
var result = Compiler.Compiler.Compile(job);
if (result.ContainsErrors)
{
// The project contains errors. Bail out.
return false;
}
var stringTable = yarnProjectImporter.GenerateStringsTable(result);
// If there was an error, bail out here
if (stringTable == null)
{
return false;
}
// Convert the string tables to CSV...
var outputCSV = StringTableEntry.CreateCSV(stringTable);
// ...and write it to disk.
File.WriteAllText(destination, outputCSV);
return true;
}
///
/// Writes a .csv file to disk at the path indicated by , containing all of the lines found in the
/// scripts referred to by that
/// contain any metadata associated with them.
///
/// The YarnProjectImporter to extract
/// strings from.
/// The path to write the file to.
/// if the file was written
/// successfully, otherwise.
/// Thrown when an error
/// is encountered when generating the CSV data.
/// Thrown when an error is encountered
/// when writing the data to disk.
internal static bool WriteMetadataFile(string destination, YarnProjectImporter yarnProjectImporter)
{
var lineMetadataEntries = yarnProjectImporter.GenerateLineMetadataEntries();
// If there was an error, bail out here.
if (lineMetadataEntries == null)
{
return false;
}
var outputCSV = LineMetadataTableEntry.CreateCSV(lineMetadataEntries);
File.WriteAllText(destination, outputCSV);
return true;
}
///
/// Upgrades an old-style Yarn Project to JSON.
///
///
/// This method copies the text of the project to a new file adjacent to
/// the project, and replaces the text of the project with a new empty
/// JSON project.
///
/// A YarnProjectImporter that represents the
/// project that needs to be upgraded.
internal static void UpgradeYarnProject(YarnProjectImporter importer)
{
// We need to copy out the variable declarations from the old Yarn
// project before we replace it.
// Get the current text of the old project
var existingText = File.ReadAllText(importer.assetPath);
// Does the existing text contain anything besides the default?
var defaultProjectPattern = new System.Text.RegularExpressions.Regex(@"^title:.*?\n---[\n\s]*===[\n\s]*$", System.Text.RegularExpressions.RegexOptions.Multiline);
if (defaultProjectPattern.IsMatch(existingText))
{
// The project contains no content, so there's no need to copy
// it out.
}
else
{
// Create a unique path to store our variables
var newFilePath = Path.GetDirectoryName(importer.assetPath) + "/Variables.yarn";
newFilePath = AssetDatabase.GenerateUniqueAssetPath(newFilePath);
// Write it out to the new file
File.WriteAllText(newFilePath, existingText);
}
// Next, replace the existing project with a new one!
var newProject = YarnProjectUtility.CreateDefaultYarnProject();
File.WriteAllText(importer.assetPath, newProject.GetJson());
// Finally, import the assets we've touched.
AssetDatabase.Refresh();
}
[OnOpenAsset(OnOpenAssetAttributeMode.Execute)]
public static bool OnOpenAsset(EntityId instanceID)
{
// temporarily disabling the obsolete warning for the GetAssetPath call
// but only for Unity <6.4, otherwise we want the warning
// because this code needs to exist across multiple unity versions
// and it's only an actual concern on 6.4+ we can disable it when earlier
#if !UNITY_6000_4_OR_NEWER
#pragma warning disable 0618
#endif
var path = AssetDatabase.GetAssetPath(instanceID);
#if !UNITY_6000_4_OR_NEWER
#pragma warning restore 0618
#endif
var project = AssetDatabase.LoadAssetAtPath(path);
if (project == null)
{
return false;
}
var importer = AssetImporter.GetAtPath(path) as Yarn.Unity.Editor.YarnProjectImporter;
if (importer == null)
{
return false;
}
var yp = Yarn.Compiler.Project.LoadFromFile(path);
var files = yp.SourceFiles;
if (files.Any())
{
UnityEditorInternal.InternalEditorUtility.OpenFileAtLineExternal(System.IO.Path.GetDirectoryName(files.First()), 0);
}
return true;
}
[OnOpenAsset(OnOpenAssetAttributeMode.Validate)]
public static bool OnValidateAsset(EntityId instanceID)
{
// temporarily disabling the obsolete warning for the GetAssetPath call
// but only for Unity <6.4, otherwise we want the warning
// because this code needs to exist across multiple unity versions
// and it's only an actual concern on 6.4+ we can disable it when earlier
#if !UNITY_6000_4_OR_NEWER
#pragma warning disable 0618
#endif
var path = AssetDatabase.GetAssetPath(instanceID);
#if !UNITY_6000_4_OR_NEWER
#pragma warning restore 0618
#endif
var project = AssetDatabase.LoadAssetAtPath(path);
if (project == null)
{
return false;
}
return true;
}
}
}