source/i18n/L10n.cpp

   1 /* Copyright (C) 2023 Wildfire Games.
   2  *
   3  * Permission is hereby granted, free of charge, to any person obtaining
   4  * a copy of this software and associated documentation files (the
   5  * "Software"), to deal in the Software without restriction, including
   6  * without limitation the rights to use, copy, modify, merge, publish,
   7  * distribute, sublicense, and/or sell copies of the Software, and to
   8  * permit persons to whom the Software is furnished to do so, subject to
   9  * the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included
  12  * in all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  15  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  16  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  17  * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
  18  * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  19  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  20  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 #include "precompiled.h"
  24
  25 #include "i18n/L10n.h"
  26
  27 #include "gui/GUIManager.h"
  28 #include "lib/external_libraries/tinygettext.h"
  29 #include "lib/file/file_system.h"
  30 #include "lib/utf8.h"
  31 #include "ps/CLogger.h"
  32 #include "ps/ConfigDB.h"
  33 #include "ps/Filesystem.h"
  34 #include "ps/GameSetup/GameSetup.h"
  35
  36 #include <boost/algorithm/string/predicate.hpp>
  37 #include <sstream>
  38 #include <string>
  39
  40 namespace
  41 {
  42 /**
  43  * Determines the list of locales that the game supports.
  44  *
  45  * LoadListOfAvailableLocales() checks the locale codes of the translation
  46  * files in the 'l10n' folder of the virtual filesystem. If it finds a
  47  * translation file prefixed with a locale code followed by a dot, it
  48  * determines that the game supports that locale.
  49  */
  50 std::vector<icu::Locale> LoadListOfAvailableLocales()
  51 {
  52         // US is always available.
  53         std::vector<icu::Locale> availableLocales{icu::Locale::getUS()};
  54
  55         VfsPaths filenames;
  56         if (vfs::GetPathnames(g_VFS, L"l10n/", L"*.po", filenames) < 0)
  57                 return availableLocales;
  58
  59         availableLocales.reserve(filenames.size());
  60
  61         for (const VfsPath& path : filenames)
  62         {
  63                 // Note: PO files follow this naming convention: "l10n/<locale code>.<mod name>.po". For example: "l10n/gl.public.po".
  64                 const std::string filename = utf8_from_wstring(path.string()).substr(strlen("l10n/"));
  65                 const size_t lengthToFirstDot = filename.find('.');
  66                 const std::string localeCode = filename.substr(0, lengthToFirstDot);
  67                 icu::Locale locale(icu::Locale::createCanonical(localeCode.c_str()));
  68                 const auto it = std::find(availableLocales.begin(), availableLocales.end(), locale);
  69                 if (it != availableLocales.end())
  70                         continue;
  71
  72                 availableLocales.push_back(std::move(locale));
  73         }
  74         availableLocales.shrink_to_fit();
  75
  76         return availableLocales;
  77 }
  78
  79 Status ReloadChangedFileCB(void* param, const VfsPath& path)
  80 {
  81         return static_cast<L10n*>(param)->ReloadChangedFile(path);
  82 }
  83
  84 /**
  85  * Loads the specified content of a PO file into the specified dictionary.
  86  *
  87  * Used by LoadDictionaryForCurrentLocale() to add entries to the game
  88  * translations @link dictionary.
  89  *
  90  * @param poContent Content of a PO file as a string.
  91  * @param dictionary Dictionary where the entries from the PO file should be
  92  *        stored.
  93  */
  94 void ReadPoIntoDictionary(const std::string& poContent, tinygettext::Dictionary* dictionary)
  95 {
  96         try
  97         {
  98                 std::istringstream inputStream(poContent);
  99                 tinygettext::POParser::parse("virtual PO file", inputStream, *dictionary);
 100         }
 101         catch (std::exception& e)
 102         {
 103                 LOGERROR("[Localization] Exception while reading virtual PO file: %s", e.what());
 104         }
 105 }
 106
 107 /**
 108  * Creates an ICU date formatted with the specified settings.
 109  *
 110  * @param type Whether formatted dates must show both the date and the time,
 111  *        only the date or only the time.
 112  * @param style ICU style to format dates by default.
 113  * @param locale Locale that the date formatter should use to parse strings.
 114  *        It has no relevance for date formatting, only matters for date
 115  *        parsing.
 116  * @return ICU date formatter.
 117  */
 118 std::unique_ptr<icu::DateFormat> CreateDateTimeInstance(const L10n::DateTimeType& type, const icu::DateFormat::EStyle& style, const icu::Locale& locale)
 119 {
 120         switch (type)
 121         {
 122         case L10n::Date:
 123                 return std::unique_ptr<icu::DateFormat>{
 124                         icu::SimpleDateFormat::createDateInstance(style, locale)};
 125
 126         case L10n::Time:
 127                 return std::unique_ptr<icu::DateFormat>{
 128                         icu::SimpleDateFormat::createTimeInstance(style, locale)};
 129
 130         case L10n::DateTime: FALLTHROUGH;
 131         default:
 132                 return std::unique_ptr<icu::DateFormat>{
 133                         icu::SimpleDateFormat::createDateTimeInstance(style, style, locale)};
 134         }
 135 }
 136
 137 } // anonymous namespace
 138
 139 L10n::L10n()
 140         : m_Dictionary(std::make_unique<tinygettext::Dictionary>())
 141 {
 142         // Determine whether or not to print tinygettext messages to the standard
 143         // error output, which it tinygettext's default behavior, but not ours.
 144         bool tinygettext_debug = false;
 145         CFG_GET_VAL("tinygettext.debug", tinygettext_debug);
 146         if (!tinygettext_debug)
 147         {
 148                 tinygettext::Log::log_info_callback = 0;
 149                 tinygettext::Log::log_warning_callback = 0;
 150                 tinygettext::Log::log_error_callback = 0;
 151         }
 152
 153         m_AvailableLocales = LoadListOfAvailableLocales();
 154
 155         ReevaluateCurrentLocaleAndReload();
 156
 157         // Handle hotloading
 158         RegisterFileReloadFunc(ReloadChangedFileCB, this);
 159 }
 160
 161 L10n::~L10n()
 162 {
 163         UnregisterFileReloadFunc(ReloadChangedFileCB, this);
 164 }
 165
 166 const icu::Locale& L10n::GetCurrentLocale() const
 167 {
 168         return m_CurrentLocale;
 169 }
 170
 171 bool L10n::SaveLocale(const std::string& localeCode) const
 172 {
 173         if (localeCode == "long" && InDevelopmentCopy())
 174         {
 175                 g_ConfigDB.SetValueString(CFG_USER, "locale", "long");
 176                 return true;
 177         }
 178         return SaveLocale(icu::Locale(icu::Locale::createCanonical(localeCode.c_str())));
 179 }
 180
 181 bool L10n::SaveLocale(const icu::Locale& locale) const
 182 {
 183         if (!ValidateLocale(locale))
 184                 return false;
 185
 186         g_ConfigDB.SetValueString(CFG_USER, "locale", locale.getName());
 187         return g_ConfigDB.WriteValueToFile(CFG_USER, "locale", locale.getName());
 188 }
 189
 190 bool L10n::ValidateLocale(const std::string& localeCode) const
 191 {
 192         return ValidateLocale(icu::Locale::createCanonical(localeCode.c_str()));
 193 }
 194
 195 // Returns true if both of these conditions are true:
 196 //  1. ICU has resources for that locale (which also ensures it's a valid locale string)
 197 //  2. Either a dictionary for language_country or for language is available.
 198 bool L10n::ValidateLocale(const icu::Locale& locale) const
 199 {
 200         if (locale.isBogus())
 201                 return false;
 202
 203         return !GetFallbackToAvailableDictLocale(locale).empty();
 204 }
 205
 206 std::vector<std::wstring> L10n::GetDictionariesForLocale(const std::string& locale) const
 207 {
 208         std::vector<std::wstring> ret;
 209         VfsPaths filenames;
 210
 211         std::wstring dictName = GetFallbackToAvailableDictLocale(icu::Locale::createCanonical(locale.c_str()));
 212         vfs::GetPathnames(g_VFS, L"l10n/", dictName.append(L".*.po").c_str(), filenames);
 213
 214         for (const VfsPath& path : filenames)
 215                 ret.push_back(path.Filename().string());
 216
 217         return ret;
 218 }
 219
 220 std::wstring L10n::GetFallbackToAvailableDictLocale(const std::string& locale) const
 221 {
 222         return GetFallbackToAvailableDictLocale(icu::Locale::createCanonical(locale.c_str()));
 223 }
 224
 225 std::wstring L10n::GetFallbackToAvailableDictLocale(const icu::Locale& locale) const
 226 {
 227         std::wstringstream stream;
 228
 229         const auto checkLangAndCountry = [&locale](const icu::Locale& l)
 230         {
 231                 return strcmp(locale.getLanguage(), l.getLanguage()) == 0
 232                         && strcmp(locale.getCountry(), l.getCountry()) == 0;
 233         };
 234
 235         if (strcmp(locale.getCountry(), "") != 0
 236                 && std::find_if(m_AvailableLocales.begin(), m_AvailableLocales.end(), checkLangAndCountry) !=
 237                 m_AvailableLocales.end())
 238         {
 239                 stream << locale.getLanguage() << L"_" << locale.getCountry();
 240                 return stream.str();
 241         }
 242
 243         const auto checkLang = [&locale](const icu::Locale& l)
 244         {
 245                 return strcmp(locale.getLanguage(), l.getLanguage()) == 0;
 246         };
 247
 248         if (std::find_if(m_AvailableLocales.begin(), m_AvailableLocales.end(), checkLang) !=
 249                 m_AvailableLocales.end())
 250         {
 251                 stream << locale.getLanguage();
 252                 return stream.str();
 253         }
 254
 255         return L"";
 256 }
 257
 258 std::string L10n::GetDictionaryLocale(const std::string& configLocaleString) const
 259 {
 260         icu::Locale out;
 261         GetDictionaryLocale(configLocaleString, out);
 262         return out.getName();
 263 }
 264
 265 // First, try to get a valid locale from the config, then check if the system locale can be used and otherwise fall back to en_US.
 266 void L10n::GetDictionaryLocale(const std::string& configLocaleString, icu::Locale& outLocale) const
 267 {
 268         if (!configLocaleString.empty())
 269         {
 270                 icu::Locale configLocale = icu::Locale::createCanonical(configLocaleString.c_str());
 271                 if (ValidateLocale(configLocale))
 272                 {
 273                         outLocale = configLocale;
 274                         return;
 275                 }
 276                 else
 277                         LOGWARNING("The configured locale is not valid or no translations are available. Falling back to another locale.");
 278         }
 279
 280         icu::Locale systemLocale = icu::Locale::getDefault();
 281         if (ValidateLocale(systemLocale))
 282                 outLocale = systemLocale;
 283         else
 284                 outLocale = icu::Locale::getUS();
 285 }
 286
 287 // Try to find the best dictionary locale based on user configuration and system locale, set the currentLocale and reload the dictionary.
 288 void L10n::ReevaluateCurrentLocaleAndReload()
 289 {
 290         std::string locale;
 291         CFG_GET_VAL("locale", locale);
 292
 293         if (locale == "long")
 294         {
 295                 // Set ICU to en_US to have a valid language for displaying dates
 296                 m_CurrentLocale = icu::Locale::getUS();
 297                 m_CurrentLocaleIsOriginalGameLocale = false;
 298                 m_UseLongStrings = true;
 299         }
 300         else
 301         {
 302                 GetDictionaryLocale(locale, m_CurrentLocale);
 303                 m_CurrentLocaleIsOriginalGameLocale = (m_CurrentLocale == icu::Locale::getUS()) == 1;
 304                 m_UseLongStrings = false;
 305         }
 306         LoadDictionaryForCurrentLocale();
 307 }
 308
 309 // Get all locales supported by ICU.
 310 std::vector<std::string> L10n::GetAllLocales() const
 311 {
 312         std::vector<std::string> ret;
 313         int32_t count;
 314         const icu::Locale* icuSupportedLocales = icu::Locale::getAvailableLocales(count);
 315         for (int i=0; i<count; ++i)
 316                 ret.push_back(icuSupportedLocales[i].getName());
 317         return ret;
 318 }
 319
 320
 321 bool L10n::UseLongStrings() const
 322 {
 323         return m_UseLongStrings;
 324 };
 325
 326 std::vector<std::string> L10n::GetSupportedLocaleBaseNames() const
 327 {
 328         std::vector<std::string> supportedLocaleCodes;
 329         for (const icu::Locale& locale : m_AvailableLocales)
 330         {
 331                 if (!InDevelopmentCopy() && strcmp(locale.getBaseName(), "long") == 0)
 332                         continue;
 333                 supportedLocaleCodes.push_back(locale.getBaseName());
 334         }
 335         return supportedLocaleCodes;
 336 }
 337
 338 std::vector<std::wstring> L10n::GetSupportedLocaleDisplayNames() const
 339 {
 340         std::vector<std::wstring> supportedLocaleDisplayNames;
 341         for (const icu::Locale& locale : m_AvailableLocales)
 342         {
 343                 if (strcmp(locale.getBaseName(), "long") == 0)
 344                 {
 345                         if (InDevelopmentCopy())
 346                                 supportedLocaleDisplayNames.push_back(wstring_from_utf8(Translate("Long strings")));
 347                         continue;
 348                 }
 349
 350                 icu::UnicodeString utf16LocaleDisplayName;
 351                 locale.getDisplayName(locale, utf16LocaleDisplayName);
 352                 char localeDisplayName[512];
 353                 icu::CheckedArrayByteSink sink(localeDisplayName, ARRAY_SIZE(localeDisplayName));
 354                 utf16LocaleDisplayName.toUTF8(sink);
 355                 ENSURE(!sink.Overflowed());
 356
 357                 supportedLocaleDisplayNames.push_back(wstring_from_utf8(std::string(localeDisplayName, sink.NumberOfBytesWritten())));
 358         }
 359         return supportedLocaleDisplayNames;
 360 }
 361
 362 std::string L10n::GetCurrentLocaleString() const
 363 {
 364         return m_CurrentLocale.getName();
 365 }
 366
 367 std::string L10n::GetLocaleLanguage(const std::string& locale) const
 368 {
 369         icu::Locale loc = icu::Locale::createCanonical(locale.c_str());
 370         return loc.getLanguage();
 371 }
 372
 373 std::string L10n::GetLocaleBaseName(const std::string& locale) const
 374 {
 375         icu::Locale loc = icu::Locale::createCanonical(locale.c_str());
 376         return loc.getBaseName();
 377 }
 378
 379 std::string L10n::GetLocaleCountry(const std::string& locale) const
 380 {
 381         icu::Locale loc = icu::Locale::createCanonical(locale.c_str());
 382         return loc.getCountry();
 383 }
 384
 385 std::string L10n::GetLocaleScript(const std::string& locale) const
 386 {
 387         icu::Locale loc = icu::Locale::createCanonical(locale.c_str());
 388         return loc.getScript();
 389 }
 390
 391 std::string L10n::Translate(const std::string& sourceString) const
 392 {
 393         if (!m_CurrentLocaleIsOriginalGameLocale)
 394                 return m_Dictionary->translate(sourceString);
 395
 396         return sourceString;
 397 }
 398
 399 std::string L10n::TranslateWithContext(const std::string& context, const std::string& sourceString) const
 400 {
 401         if (!m_CurrentLocaleIsOriginalGameLocale)
 402                 return m_Dictionary->translate_ctxt(context, sourceString);
 403
 404         return sourceString;
 405 }
 406
 407 std::string L10n::TranslatePlural(const std::string& singularSourceString, const std::string& pluralSourceString, int number) const
 408 {
 409         if (!m_CurrentLocaleIsOriginalGameLocale)
 410                 return m_Dictionary->translate_plural(singularSourceString, pluralSourceString, number);
 411
 412         if (number == 1)
 413                 return singularSourceString;
 414
 415         return pluralSourceString;
 416 }
 417
 418 std::string L10n::TranslatePluralWithContext(const std::string& context, const std::string& singularSourceString, const std::string& pluralSourceString, int number) const
 419 {
 420         if (!m_CurrentLocaleIsOriginalGameLocale)
 421                 return m_Dictionary->translate_ctxt_plural(context, singularSourceString, pluralSourceString, number);
 422
 423         if (number == 1)
 424                 return singularSourceString;
 425
 426         return pluralSourceString;
 427 }
 428
 429 std::string L10n::TranslateLines(const std::string& sourceString) const
 430 {
 431         std::string targetString;
 432         std::stringstream stringOfLines(sourceString);
 433         std::string line;
 434
 435         while (std::getline(stringOfLines, line))
 436         {
 437                 if (!line.empty())
 438                         targetString.append(Translate(line));
 439                 targetString.append("\n");
 440         }
 441
 442         return targetString;
 443 }
 444
 445 UDate L10n::ParseDateTime(const std::string& dateTimeString, const std::string& dateTimeFormat, const icu::Locale& locale) const
 446 {
 447         UErrorCode success = U_ZERO_ERROR;
 448         icu::UnicodeString utf16DateTimeString = icu::UnicodeString::fromUTF8(dateTimeString.c_str());
 449         icu::UnicodeString utf16DateTimeFormat = icu::UnicodeString::fromUTF8(dateTimeFormat.c_str());
 450
 451         const icu::SimpleDateFormat dateFormatter{utf16DateTimeFormat, locale, success};
 452         return dateFormatter.parse(utf16DateTimeString, success);
 453 }
 454
 455 std::string L10n::LocalizeDateTime(const UDate dateTime, const DateTimeType& type, const icu::DateFormat::EStyle& style) const
 456 {
 457         icu::UnicodeString utf16Date;
 458
 459         const std::unique_ptr<const icu::DateFormat> dateFormatter{
 460                 CreateDateTimeInstance(type, style, m_CurrentLocale)};
 461         dateFormatter->format(dateTime, utf16Date);
 462         char utf8Date[512];
 463         icu::CheckedArrayByteSink sink(utf8Date, ARRAY_SIZE(utf8Date));
 464         utf16Date.toUTF8(sink);
 465         ENSURE(!sink.Overflowed());
 466
 467         return std::string(utf8Date, sink.NumberOfBytesWritten());
 468 }
 469
 470 std::string L10n::FormatMillisecondsIntoDateString(const UDate milliseconds, const std::string& formatString, bool useLocalTimezone) const
 471 {
 472         UErrorCode status = U_ZERO_ERROR;
 473         icu::UnicodeString dateString;
 474         std::string resultString;
 475
 476         icu::UnicodeString unicodeFormat = icu::UnicodeString::fromUTF8(formatString.c_str());
 477         icu::SimpleDateFormat dateFormat{unicodeFormat, status};
 478         if (U_FAILURE(status))
 479                 LOGERROR("Error creating SimpleDateFormat: %s", u_errorName(status));
 480
 481         status = U_ZERO_ERROR;
 482         std::unique_ptr<icu::Calendar> calendar{
 483                 useLocalTimezone ?
 484                         icu::Calendar::createInstance(m_CurrentLocale, status) :
 485                         icu::Calendar::createInstance(*icu::TimeZone::getGMT(), m_CurrentLocale, status)};
 486
 487         if (U_FAILURE(status))
 488                 LOGERROR("Error creating calendar: %s", u_errorName(status));
 489
 490         dateFormat.adoptCalendar(calendar.release());
 491         dateFormat.format(milliseconds, dateString);
 492
 493         dateString.toUTF8String(resultString);
 494         return resultString;
 495 }
 496
 497 std::string L10n::FormatDecimalNumberIntoString(double number) const
 498 {
 499         UErrorCode success = U_ZERO_ERROR;
 500         icu::UnicodeString utf16Number;
 501         std::unique_ptr<icu::NumberFormat> numberFormatter{
 502                 icu::NumberFormat::createInstance(m_CurrentLocale, UNUM_DECIMAL, success)};
 503         numberFormatter->format(number, utf16Number);
 504         char utf8Number[512];
 505         icu::CheckedArrayByteSink sink(utf8Number, ARRAY_SIZE(utf8Number));
 506         utf16Number.toUTF8(sink);
 507         ENSURE(!sink.Overflowed());
 508
 509         return std::string(utf8Number, sink.NumberOfBytesWritten());
 510 }
 511
 512 VfsPath L10n::LocalizePath(const VfsPath& sourcePath) const
 513 {
 514         VfsPath localizedPath = sourcePath.Parent() / L"l10n" /
 515                 wstring_from_utf8(m_CurrentLocale.getLanguage()) / sourcePath.Filename();
 516         if (!VfsFileExists(localizedPath))
 517                 return sourcePath;
 518
 519         return localizedPath;
 520 }
 521
 522 Status L10n::ReloadChangedFile(const VfsPath& path)
 523 {
 524         if (!boost::algorithm::starts_with(path.string(), L"l10n/"))
 525                 return INFO::OK;
 526
 527         if (path.Extension() != L".po")
 528                 return INFO::OK;
 529
 530         // If the file was deleted, ignore it
 531         if (!VfsFileExists(path))
 532                 return INFO::OK;
 533
 534         std::wstring dictName = GetFallbackToAvailableDictLocale(m_CurrentLocale);
 535         if (m_UseLongStrings)
 536                 dictName = L"long";
 537         if (dictName.empty())
 538                 return INFO::OK;
 539
 540         // Only the currently used language is loaded, so ignore all others
 541         if (path.string().rfind(dictName) == std::string::npos)
 542                 return INFO::OK;
 543
 544         LOGMESSAGE("Hotloading translations from '%s'", path.string8());
 545
 546         CVFSFile file;
 547         if (file.Load(g_VFS, path) != PSRETURN_OK)
 548         {
 549                 LOGERROR("Failed to read translations from '%s'", path.string8());
 550                 return ERR::FAIL;
 551         }
 552
 553         std::string content = file.DecodeUTF8();
 554         ReadPoIntoDictionary(content, m_Dictionary.get());
 555
 556         if (g_GUI)
 557                 g_GUI->ReloadAllPages();
 558
 559         return INFO::OK;
 560 }
 561
 562 void L10n::LoadDictionaryForCurrentLocale()
 563 {
 564         m_Dictionary = std::make_unique<tinygettext::Dictionary>();
 565         VfsPaths filenames;
 566
 567         if (m_UseLongStrings)
 568         {
 569                 if (vfs::GetPathnames(g_VFS, L"l10n/", L"long.*.po", filenames) < 0)
 570                         return;
 571         }
 572         else
 573         {
 574                 std::wstring dictName = GetFallbackToAvailableDictLocale(m_CurrentLocale);
 575                 if (vfs::GetPathnames(g_VFS, L"l10n/", dictName.append(L".*.po").c_str(), filenames) < 0)
 576                 {
 577                         LOGERROR("No files for the dictionary found, but at this point the input should already be validated!");
 578                         return;
 579                 }
 580         }
 581
 582         for (const VfsPath& path : filenames)
 583         {
 584                 CVFSFile file;
 585                 file.Load(g_VFS, path);
 586                 std::string content = file.DecodeUTF8();
 587                 ReadPoIntoDictionary(content, m_Dictionary.get());
 588         }
 589 }