source/i18n/L10n.cpp

   1 /* Copyright (C) 2023 Wildfire Games.
   2  *
   3  * Permission is hereby granted, free of charge, to any person obtaining
   4  * a copy of this software and associated documentation files (the
   5  * "Software"), to deal in the Software without restriction, including
   6  * without limitation the rights to use, copy, modify, merge, publish,
   7  * distribute, sublicense, and/or sell copies of the Software, and to
   8  * permit persons to whom the Software is furnished to do so, subject to
   9  * the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included
  12  * in all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  15  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  16  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  17  * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
  18  * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  19  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  20  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 #include "precompiled.h"
  24
  25 #include "i18n/L10n.h"
  26
  27 #include "gui/GUIManager.h"
  28 #include "lib/external_libraries/tinygettext.h"
  29 #include "lib/file/file_system.h"
  30 #include "lib/utf8.h"
  31 #include "ps/CLogger.h"
  32 #include "ps/ConfigDB.h"
  33 #include "ps/Filesystem.h"
  34 #include "ps/GameSetup/GameSetup.h"
  35
  36 #include <boost/algorithm/string.hpp>
  37 #include <boost/concept_check.hpp>
  38 #include <sstream>
  39 #include <string>
  40
  41 namespace
  42 {
  43 /**
  44  * Determines the list of locales that the game supports.
  45  *
  46  * LoadListOfAvailableLocales() checks the locale codes of the translation
  47  * files in the 'l10n' folder of the virtual filesystem. If it finds a
  48  * translation file prefixed with a locale code followed by a dot, it
  49  * determines that the game supports that locale.
  50  */
  51 std::vector<icu::Locale> LoadListOfAvailableLocales()
  52 {
  53         // US is always available.
  54         std::vector<icu::Locale> availableLocales{icu::Locale::getUS()};
  55
  56         VfsPaths filenames;
  57         if (vfs::GetPathnames(g_VFS, L"l10n/", L"*.po", filenames) < 0)
  58                 return availableLocales;
  59
  60         availableLocales.reserve(filenames.size());
  61
  62         for (const VfsPath& path : filenames)
  63         {
  64                 // Note: PO files follow this naming convention: "l10n/<locale code>.<mod name>.po". For example: "l10n/gl.public.po".
  65                 const std::string filename = utf8_from_wstring(path.string()).substr(strlen("l10n/"));
  66                 const size_t lengthToFirstDot = filename.find('.');
  67                 const std::string localeCode = filename.substr(0, lengthToFirstDot);
  68                 icu::Locale locale(icu::Locale::createCanonical(localeCode.c_str()));
  69                 const auto it = std::find(availableLocales.begin(), availableLocales.end(), locale);
  70                 if (it != availableLocales.end())
  71                         continue;
  72
  73                 availableLocales.push_back(std::move(locale));
  74         }
  75         availableLocales.shrink_to_fit();
  76
  77         return availableLocales;
  78 }
  79
  80 Status ReloadChangedFileCB(void* param, const VfsPath& path)
  81 {
  82         return static_cast<L10n*>(param)->ReloadChangedFile(path);
  83 }
  84
  85 /**
  86  * Loads the specified content of a PO file into the specified dictionary.
  87  *
  88  * Used by LoadDictionaryForCurrentLocale() to add entries to the game
  89  * translations @link dictionary.
  90  *
  91  * @param poContent Content of a PO file as a string.
  92  * @param dictionary Dictionary where the entries from the PO file should be
  93  *        stored.
  94  */
  95 void ReadPoIntoDictionary(const std::string& poContent, tinygettext::Dictionary* dictionary)
  96 {
  97         try
  98         {
  99                 std::istringstream inputStream(poContent);
 100                 tinygettext::POParser::parse("virtual PO file", inputStream, *dictionary);
 101         }
 102         catch (std::exception& e)
 103         {
 104                 LOGERROR("[Localization] Exception while reading virtual PO file: %s", e.what());
 105         }
 106 }
 107
 108 /**
 109  * Creates an ICU date formatted with the specified settings.
 110  *
 111  * @param type Whether formatted dates must show both the date and the time,
 112  *        only the date or only the time.
 113  * @param style ICU style to format dates by default.
 114  * @param locale Locale that the date formatter should use to parse strings.
 115  *        It has no relevance for date formatting, only matters for date
 116  *        parsing.
 117  * @return ICU date formatter.
 118  */
 119 std::unique_ptr<icu::DateFormat> CreateDateTimeInstance(const L10n::DateTimeType& type, const icu::DateFormat::EStyle& style, const icu::Locale& locale)
 120 {
 121         switch (type)
 122         {
 123         case L10n::Date:
 124                 return std::unique_ptr<icu::DateFormat>{
 125                         icu::SimpleDateFormat::createDateInstance(style, locale)};
 126
 127         case L10n::Time:
 128                 return std::unique_ptr<icu::DateFormat>{
 129                         icu::SimpleDateFormat::createTimeInstance(style, locale)};
 130
 131         case L10n::DateTime: FALLTHROUGH;
 132         default:
 133                 return std::unique_ptr<icu::DateFormat>{
 134                         icu::SimpleDateFormat::createDateTimeInstance(style, style, locale)};
 135         }
 136 }
 137
 138 } // anonymous namespace
 139
 140 L10n::L10n()
 141         : m_Dictionary(std::make_unique<tinygettext::Dictionary>())
 142 {
 143         // Determine whether or not to print tinygettext messages to the standard
 144         // error output, which it tinygettext's default behavior, but not ours.
 145         bool tinygettext_debug = false;
 146         CFG_GET_VAL("tinygettext.debug", tinygettext_debug);
 147         if (!tinygettext_debug)
 148         {
 149                 tinygettext::Log::log_info_callback = 0;
 150                 tinygettext::Log::log_warning_callback = 0;
 151                 tinygettext::Log::log_error_callback = 0;
 152         }
 153
 154         m_AvailableLocales = LoadListOfAvailableLocales();
 155
 156         ReevaluateCurrentLocaleAndReload();
 157
 158         // Handle hotloading
 159         RegisterFileReloadFunc(ReloadChangedFileCB, this);
 160 }
 161
 162 L10n::~L10n()
 163 {
 164         UnregisterFileReloadFunc(ReloadChangedFileCB, this);
 165 }
 166
 167 const icu::Locale& L10n::GetCurrentLocale() const
 168 {
 169         return m_CurrentLocale;
 170 }
 171
 172 bool L10n::SaveLocale(const std::string& localeCode) const
 173 {
 174         if (localeCode == "long" && InDevelopmentCopy())
 175         {
 176                 g_ConfigDB.SetValueString(CFG_USER, "locale", "long");
 177                 return true;
 178         }
 179         return SaveLocale(icu::Locale(icu::Locale::createCanonical(localeCode.c_str())));
 180 }
 181
 182 bool L10n::SaveLocale(const icu::Locale& locale) const
 183 {
 184         if (!ValidateLocale(locale))
 185                 return false;
 186
 187         g_ConfigDB.SetValueString(CFG_USER, "locale", locale.getName());
 188         return g_ConfigDB.WriteValueToFile(CFG_USER, "locale", locale.getName());
 189 }
 190
 191 bool L10n::ValidateLocale(const std::string& localeCode) const
 192 {
 193         return ValidateLocale(icu::Locale::createCanonical(localeCode.c_str()));
 194 }
 195
 196 // Returns true if both of these conditions are true:
 197 //  1. ICU has resources for that locale (which also ensures it's a valid locale string)
 198 //  2. Either a dictionary for language_country or for language is available.
 199 bool L10n::ValidateLocale(const icu::Locale& locale) const
 200 {
 201         if (locale.isBogus())
 202                 return false;
 203
 204         return !GetFallbackToAvailableDictLocale(locale).empty();
 205 }
 206
 207 std::vector<std::wstring> L10n::GetDictionariesForLocale(const std::string& locale) const
 208 {
 209         std::vector<std::wstring> ret;
 210         VfsPaths filenames;
 211
 212         std::wstring dictName = GetFallbackToAvailableDictLocale(icu::Locale::createCanonical(locale.c_str()));
 213         vfs::GetPathnames(g_VFS, L"l10n/", dictName.append(L".*.po").c_str(), filenames);
 214
 215         for (const VfsPath& path : filenames)
 216                 ret.push_back(path.Filename().string());
 217
 218         return ret;
 219 }
 220
 221 std::wstring L10n::GetFallbackToAvailableDictLocale(const std::string& locale) const
 222 {
 223         return GetFallbackToAvailableDictLocale(icu::Locale::createCanonical(locale.c_str()));
 224 }
 225
 226 std::wstring L10n::GetFallbackToAvailableDictLocale(const icu::Locale& locale) const
 227 {
 228         std::wstringstream stream;
 229
 230         const auto checkLangAndCountry = [&locale](const icu::Locale& l)
 231         {
 232                 return strcmp(locale.getLanguage(), l.getLanguage()) == 0
 233                         && strcmp(locale.getCountry(), l.getCountry()) == 0;
 234         };
 235
 236         if (strcmp(locale.getCountry(), "") != 0
 237                 && std::find_if(m_AvailableLocales.begin(), m_AvailableLocales.end(), checkLangAndCountry) !=
 238                 m_AvailableLocales.end())
 239         {
 240                 stream << locale.getLanguage() << L"_" << locale.getCountry();
 241                 return stream.str();
 242         }
 243
 244         const auto checkLang = [&locale](const icu::Locale& l)
 245         {
 246                 return strcmp(locale.getLanguage(), l.getLanguage()) == 0;
 247         };
 248
 249         if (std::find_if(m_AvailableLocales.begin(), m_AvailableLocales.end(), checkLang) !=
 250                 m_AvailableLocales.end())
 251         {
 252                 stream << locale.getLanguage();
 253                 return stream.str();
 254         }
 255
 256         return L"";
 257 }
 258
 259 std::string L10n::GetDictionaryLocale(const std::string& configLocaleString) const
 260 {
 261         icu::Locale out;
 262         GetDictionaryLocale(configLocaleString, out);
 263         return out.getName();
 264 }
 265
 266 // First, try to get a valid locale from the config, then check if the system locale can be used and otherwise fall back to en_US.
 267 void L10n::GetDictionaryLocale(const std::string& configLocaleString, icu::Locale& outLocale) const
 268 {
 269         if (!configLocaleString.empty())
 270         {
 271                 icu::Locale configLocale = icu::Locale::createCanonical(configLocaleString.c_str());
 272                 if (ValidateLocale(configLocale))
 273                 {
 274                         outLocale = configLocale;
 275                         return;
 276                 }
 277                 else
 278                         LOGWARNING("The configured locale is not valid or no translations are available. Falling back to another locale.");
 279         }
 280
 281         icu::Locale systemLocale = icu::Locale::getDefault();
 282         if (ValidateLocale(systemLocale))
 283                 outLocale = systemLocale;
 284         else
 285                 outLocale = icu::Locale::getUS();
 286 }
 287
 288 // Try to find the best dictionary locale based on user configuration and system locale, set the currentLocale and reload the dictionary.
 289 void L10n::ReevaluateCurrentLocaleAndReload()
 290 {
 291         std::string locale;
 292         CFG_GET_VAL("locale", locale);
 293
 294         if (locale == "long")
 295         {
 296                 // Set ICU to en_US to have a valid language for displaying dates
 297                 m_CurrentLocale = icu::Locale::getUS();
 298                 m_CurrentLocaleIsOriginalGameLocale = false;
 299                 m_UseLongStrings = true;
 300         }
 301         else
 302         {
 303                 GetDictionaryLocale(locale, m_CurrentLocale);
 304                 m_CurrentLocaleIsOriginalGameLocale = (m_CurrentLocale == icu::Locale::getUS()) == 1;
 305                 m_UseLongStrings = false;
 306         }
 307         LoadDictionaryForCurrentLocale();
 308 }
 309
 310 // Get all locales supported by ICU.
 311 std::vector<std::string> L10n::GetAllLocales() const
 312 {
 313         std::vector<std::string> ret;
 314         int32_t count;
 315         const icu::Locale* icuSupportedLocales = icu::Locale::getAvailableLocales(count);
 316         for (int i=0; i<count; ++i)
 317                 ret.push_back(icuSupportedLocales[i].getName());
 318         return ret;
 319 }
 320
 321
 322 bool L10n::UseLongStrings() const
 323 {
 324         return m_UseLongStrings;
 325 };
 326
 327 std::vector<std::string> L10n::GetSupportedLocaleBaseNames() const
 328 {
 329         std::vector<std::string> supportedLocaleCodes;
 330         for (const icu::Locale& locale : m_AvailableLocales)
 331         {
 332                 if (!InDevelopmentCopy() && strcmp(locale.getBaseName(), "long") == 0)
 333                         continue;
 334                 supportedLocaleCodes.push_back(locale.getBaseName());
 335         }
 336         return supportedLocaleCodes;
 337 }
 338
 339 std::vector<std::wstring> L10n::GetSupportedLocaleDisplayNames() const
 340 {
 341         std::vector<std::wstring> supportedLocaleDisplayNames;
 342         for (const icu::Locale& locale : m_AvailableLocales)
 343         {
 344                 if (strcmp(locale.getBaseName(), "long") == 0)
 345                 {
 346                         if (InDevelopmentCopy())
 347                                 supportedLocaleDisplayNames.push_back(wstring_from_utf8(Translate("Long strings")));
 348                         continue;
 349                 }
 350
 351                 icu::UnicodeString utf16LocaleDisplayName;
 352                 locale.getDisplayName(locale, utf16LocaleDisplayName);
 353                 char localeDisplayName[512];
 354                 icu::CheckedArrayByteSink sink(localeDisplayName, ARRAY_SIZE(localeDisplayName));
 355                 utf16LocaleDisplayName.toUTF8(sink);
 356                 ENSURE(!sink.Overflowed());
 357
 358                 supportedLocaleDisplayNames.push_back(wstring_from_utf8(std::string(localeDisplayName, sink.NumberOfBytesWritten())));
 359         }
 360         return supportedLocaleDisplayNames;
 361 }
 362
 363 std::string L10n::GetCurrentLocaleString() const
 364 {
 365         return m_CurrentLocale.getName();
 366 }
 367
 368 std::string L10n::GetLocaleLanguage(const std::string& locale) const
 369 {
 370         icu::Locale loc = icu::Locale::createCanonical(locale.c_str());
 371         return loc.getLanguage();
 372 }
 373
 374 std::string L10n::GetLocaleBaseName(const std::string& locale) const
 375 {
 376         icu::Locale loc = icu::Locale::createCanonical(locale.c_str());
 377         return loc.getBaseName();
 378 }
 379
 380 std::string L10n::GetLocaleCountry(const std::string& locale) const
 381 {
 382         icu::Locale loc = icu::Locale::createCanonical(locale.c_str());
 383         return loc.getCountry();
 384 }
 385
 386 std::string L10n::GetLocaleScript(const std::string& locale) const
 387 {
 388         icu::Locale loc = icu::Locale::createCanonical(locale.c_str());
 389         return loc.getScript();
 390 }
 391
 392 std::string L10n::Translate(const std::string& sourceString) const
 393 {
 394         if (!m_CurrentLocaleIsOriginalGameLocale)
 395                 return m_Dictionary->translate(sourceString);
 396
 397         return sourceString;
 398 }
 399
 400 std::string L10n::TranslateWithContext(const std::string& context, const std::string& sourceString) const
 401 {
 402         if (!m_CurrentLocaleIsOriginalGameLocale)
 403                 return m_Dictionary->translate_ctxt(context, sourceString);
 404
 405         return sourceString;
 406 }
 407
 408 std::string L10n::TranslatePlural(const std::string& singularSourceString, const std::string& pluralSourceString, int number) const
 409 {
 410         if (!m_CurrentLocaleIsOriginalGameLocale)
 411                 return m_Dictionary->translate_plural(singularSourceString, pluralSourceString, number);
 412
 413         if (number == 1)
 414                 return singularSourceString;
 415
 416         return pluralSourceString;
 417 }
 418
 419 std::string L10n::TranslatePluralWithContext(const std::string& context, const std::string& singularSourceString, const std::string& pluralSourceString, int number) const
 420 {
 421         if (!m_CurrentLocaleIsOriginalGameLocale)
 422                 return m_Dictionary->translate_ctxt_plural(context, singularSourceString, pluralSourceString, number);
 423
 424         if (number == 1)
 425                 return singularSourceString;
 426
 427         return pluralSourceString;
 428 }
 429
 430 std::string L10n::TranslateLines(const std::string& sourceString) const
 431 {
 432         std::string targetString;
 433         std::stringstream stringOfLines(sourceString);
 434         std::string line;
 435
 436         while (std::getline(stringOfLines, line))
 437         {
 438                 if (!line.empty())
 439                         targetString.append(Translate(line));
 440                 targetString.append("\n");
 441         }
 442
 443         return targetString;
 444 }
 445
 446 UDate L10n::ParseDateTime(const std::string& dateTimeString, const std::string& dateTimeFormat, const icu::Locale& locale) const
 447 {
 448         UErrorCode success = U_ZERO_ERROR;
 449         icu::UnicodeString utf16DateTimeString = icu::UnicodeString::fromUTF8(dateTimeString.c_str());
 450         icu::UnicodeString utf16DateTimeFormat = icu::UnicodeString::fromUTF8(dateTimeFormat.c_str());
 451
 452         const icu::SimpleDateFormat dateFormatter{utf16DateTimeFormat, locale, success};
 453         return dateFormatter.parse(utf16DateTimeString, success);
 454 }
 455
 456 std::string L10n::LocalizeDateTime(const UDate dateTime, const DateTimeType& type, const icu::DateFormat::EStyle& style) const
 457 {
 458         icu::UnicodeString utf16Date;
 459
 460         const std::unique_ptr<const icu::DateFormat> dateFormatter{
 461                 CreateDateTimeInstance(type, style, m_CurrentLocale)};
 462         dateFormatter->format(dateTime, utf16Date);
 463         char utf8Date[512];
 464         icu::CheckedArrayByteSink sink(utf8Date, ARRAY_SIZE(utf8Date));
 465         utf16Date.toUTF8(sink);
 466         ENSURE(!sink.Overflowed());
 467
 468         return std::string(utf8Date, sink.NumberOfBytesWritten());
 469 }
 470
 471 std::string L10n::FormatMillisecondsIntoDateString(const UDate milliseconds, const std::string& formatString, bool useLocalTimezone) const
 472 {
 473         UErrorCode status = U_ZERO_ERROR;
 474         icu::UnicodeString dateString;
 475         std::string resultString;
 476
 477         icu::UnicodeString unicodeFormat = icu::UnicodeString::fromUTF8(formatString.c_str());
 478         icu::SimpleDateFormat dateFormat{unicodeFormat, status};
 479         if (U_FAILURE(status))
 480                 LOGERROR("Error creating SimpleDateFormat: %s", u_errorName(status));
 481
 482         status = U_ZERO_ERROR;
 483         std::unique_ptr<icu::Calendar> calendar{
 484                 useLocalTimezone ?
 485                         icu::Calendar::createInstance(m_CurrentLocale, status) :
 486                         icu::Calendar::createInstance(*icu::TimeZone::getGMT(), m_CurrentLocale, status)};
 487
 488         if (U_FAILURE(status))
 489                 LOGERROR("Error creating calendar: %s", u_errorName(status));
 490
 491         dateFormat.adoptCalendar(calendar.release());
 492         dateFormat.format(milliseconds, dateString);
 493
 494         dateString.toUTF8String(resultString);
 495         return resultString;
 496 }
 497
 498 std::string L10n::FormatDecimalNumberIntoString(double number) const
 499 {
 500         UErrorCode success = U_ZERO_ERROR;
 501         icu::UnicodeString utf16Number;
 502         std::unique_ptr<icu::NumberFormat> numberFormatter{
 503                 icu::NumberFormat::createInstance(m_CurrentLocale, UNUM_DECIMAL, success)};
 504         numberFormatter->format(number, utf16Number);
 505         char utf8Number[512];
 506         icu::CheckedArrayByteSink sink(utf8Number, ARRAY_SIZE(utf8Number));
 507         utf16Number.toUTF8(sink);
 508         ENSURE(!sink.Overflowed());
 509
 510         return std::string(utf8Number, sink.NumberOfBytesWritten());
 511 }
 512
 513 VfsPath L10n::LocalizePath(const VfsPath& sourcePath) const
 514 {
 515         VfsPath localizedPath = sourcePath.Parent() / L"l10n" /
 516                 wstring_from_utf8(m_CurrentLocale.getLanguage()) / sourcePath.Filename();
 517         if (!VfsFileExists(localizedPath))
 518                 return sourcePath;
 519
 520         return localizedPath;
 521 }
 522
 523 Status L10n::ReloadChangedFile(const VfsPath& path)
 524 {
 525         if (!boost::algorithm::starts_with(path.string(), L"l10n/"))
 526                 return INFO::OK;
 527
 528         if (path.Extension() != L".po")
 529                 return INFO::OK;
 530
 531         // If the file was deleted, ignore it
 532         if (!VfsFileExists(path))
 533                 return INFO::OK;
 534
 535         std::wstring dictName = GetFallbackToAvailableDictLocale(m_CurrentLocale);
 536         if (m_UseLongStrings)
 537                 dictName = L"long";
 538         if (dictName.empty())
 539                 return INFO::OK;
 540
 541         // Only the currently used language is loaded, so ignore all others
 542         if (path.string().rfind(dictName) == std::string::npos)
 543                 return INFO::OK;
 544
 545         LOGMESSAGE("Hotloading translations from '%s'", path.string8());
 546
 547         CVFSFile file;
 548         if (file.Load(g_VFS, path) != PSRETURN_OK)
 549         {
 550                 LOGERROR("Failed to read translations from '%s'", path.string8());
 551                 return ERR::FAIL;
 552         }
 553
 554         std::string content = file.DecodeUTF8();
 555         ReadPoIntoDictionary(content, m_Dictionary.get());
 556
 557         if (g_GUI)
 558                 g_GUI->ReloadAllPages();
 559
 560         return INFO::OK;
 561 }
 562
 563 void L10n::LoadDictionaryForCurrentLocale()
 564 {
 565         m_Dictionary = std::make_unique<tinygettext::Dictionary>();
 566         VfsPaths filenames;
 567
 568         if (m_UseLongStrings)
 569         {
 570                 if (vfs::GetPathnames(g_VFS, L"l10n/", L"long.*.po", filenames) < 0)
 571                         return;
 572         }
 573         else
 574         {
 575                 std::wstring dictName = GetFallbackToAvailableDictLocale(m_CurrentLocale);
 576                 if (vfs::GetPathnames(g_VFS, L"l10n/", dictName.append(L".*.po").c_str(), filenames) < 0)
 577                 {
 578                         LOGERROR("No files for the dictionary found, but at this point the input should already be validated!");
 579                         return;
 580                 }
 581         }
 582
 583         for (const VfsPath& path : filenames)
 584         {
 585                 CVFSFile file;
 586                 file.Load(g_VFS, path);
 587                 std::string content = file.DecodeUTF8();
 588                 ReadPoIntoDictionary(content, m_Dictionary.get());
 589         }
 590 }