source/i18n/L10n.cpp

   1 /* Copyright (C) 2017 Wildfire Games.
   2  *
   3  * Permission is hereby granted, free of charge, to any person obtaining
   4  * a copy of this software and associated documentation files (the
   5  * "Software"), to deal in the Software without restriction, including
   6  * without limitation the rights to use, copy, modify, merge, publish,
   7  * distribute, sublicense, and/or sell copies of the Software, and to
   8  * permit persons to whom the Software is furnished to do so, subject to
   9  * the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included
  12  * in all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  15  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  16  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  17  * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
  18  * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  19  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  20  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 #include "precompiled.h"
  24
  25 #include "i18n/L10n.h"
  26
  27 #include <boost/algorithm/string.hpp>
  28 #include <boost/concept_check.hpp>
  29 #include <iostream>
  30 #include <string>
  31
  32 #include "gui/GUIManager.h"
  33 #include "lib/file/file_system.h"
  34 #include "lib/utf8.h"
  35 #include "ps/CLogger.h"
  36 #include "ps/ConfigDB.h"
  37 #include "ps/Filesystem.h"
  38 #include "ps/GameSetup/GameSetup.h"
  39
  40 static Status ReloadChangedFileCB(void* param, const VfsPath& path)
  41 {
  42         return static_cast<L10n*>(param)->ReloadChangedFile(path);
  43 }
  44
  45 L10n::L10n()
  46         : dictionary(new tinygettext::Dictionary()), currentLocaleIsOriginalGameLocale(false), useLongStrings(false)
  47 {
  48         // Determine whether or not to print tinygettext messages to the standard
  49         // error output, which it tinygettext’s default behavior, but not ours.
  50         bool tinygettext_debug = false;
  51         CFG_GET_VAL("tinygettext.debug", tinygettext_debug);
  52         if (!tinygettext_debug)
  53         {
  54                 tinygettext::Log::log_info_callback = 0;
  55                 tinygettext::Log::log_warning_callback = 0;
  56                 tinygettext::Log::log_error_callback = 0;
  57         }
  58
  59         LoadListOfAvailableLocales();
  60         ReevaluateCurrentLocaleAndReload();
  61
  62         // Handle hotloading
  63         RegisterFileReloadFunc(ReloadChangedFileCB, this);
  64 }
  65
  66 L10n::~L10n()
  67 {
  68         UnregisterFileReloadFunc(ReloadChangedFileCB, this);
  69
  70         for (Locale* const& locale : availableLocales)
  71                 delete locale;
  72         delete dictionary;
  73 }
  74
  75 Locale L10n::GetCurrentLocale() const
  76 {
  77         return currentLocale;
  78 }
  79
  80 bool L10n::SaveLocale(const std::string& localeCode) const
  81 {
  82         if (localeCode == "long" && InDevelopmentCopy())
  83         {
  84                 g_ConfigDB.SetValueString(CFG_USER, "locale", "long");
  85                 return true;
  86         }
  87         return SaveLocale(Locale(Locale::createCanonical(localeCode.c_str())));
  88 }
  89
  90 bool L10n::SaveLocale(const Locale& locale) const
  91 {
  92         if (!ValidateLocale(locale))
  93                 return false;
  94
  95         g_ConfigDB.SetValueString(CFG_USER, "locale", locale.getName());
  96         return g_ConfigDB.WriteValueToFile(CFG_USER, "locale", locale.getName());
  97 }
  98
  99 bool L10n::ValidateLocale(const std::string& localeCode) const
 100 {
 101         return ValidateLocale(Locale::createCanonical(localeCode.c_str()));
 102 }
 103
 104 // Returns true if both of these conditions are true:
 105 //  1. ICU has resources for that locale (which also ensures it's a valid locale string)
 106 //  2. Either a dictionary for language_country or for language is available.
 107 bool L10n::ValidateLocale(const Locale& locale) const
 108 {
 109         if (locale.isBogus())
 110                 return false;
 111
 112         return !GetFallbackToAvailableDictLocale(locale).empty();
 113 }
 114
 115 std::vector<std::wstring> L10n::GetDictionariesForLocale(const std::string& locale) const
 116 {
 117         std::vector<std::wstring> ret;
 118         VfsPaths filenames;
 119
 120         std::wstring dictName = GetFallbackToAvailableDictLocale(Locale::createCanonical(locale.c_str()));
 121         vfs::GetPathnames(g_VFS, L"l10n/", dictName.append(L".*.po").c_str(), filenames);
 122
 123         for (const VfsPath& path : filenames)
 124                 ret.push_back(path.Filename().string());
 125
 126         return ret;
 127 }
 128
 129 std::wstring L10n::GetFallbackToAvailableDictLocale(const std::string& locale) const
 130 {
 131         return GetFallbackToAvailableDictLocale(Locale::createCanonical(locale.c_str()));
 132 }
 133
 134 std::wstring L10n::GetFallbackToAvailableDictLocale(const Locale& locale) const
 135 {
 136         std::wstringstream stream;
 137
 138         std::function<bool(const Locale* const&)> checkLangAndCountry = [&locale](const Locale* const& l) {
 139                 return strcmp(locale.getLanguage(), l->getLanguage()) == 0
 140                        && strcmp(locale.getCountry(), l->getCountry()) == 0;
 141         };
 142
 143         if (strcmp(locale.getCountry(), "") != 0
 144             && std::find_if(availableLocales.begin(), availableLocales.end(), checkLangAndCountry) != availableLocales.end())
 145         {
 146                 stream << locale.getLanguage() << L"_" << locale.getCountry();
 147                 return stream.str();
 148         }
 149
 150         std::function<bool(const Locale* const&)> checkLang = [&locale](const Locale* const& l) {
 151                 return strcmp(locale.getLanguage(), l->getLanguage()) == 0;
 152         };
 153
 154         if (std::find_if(availableLocales.begin(), availableLocales.end(), checkLang) != availableLocales.end())
 155         {
 156                 stream << locale.getLanguage();
 157                 return stream.str();
 158         }
 159
 160         return L"";
 161 }
 162
 163 std::string L10n::GetDictionaryLocale(const std::string& configLocaleString) const
 164 {
 165         Locale out;
 166         GetDictionaryLocale(configLocaleString, out);
 167         return out.getName();
 168 }
 169
 170 // First, try to get a valid locale from the config, then check if the system locale can be used and otherwise fall back to en_US.
 171 void L10n::GetDictionaryLocale(const std::string& configLocaleString, Locale& outLocale) const
 172 {
 173         if (!configLocaleString.empty())
 174         {
 175                 Locale configLocale = Locale::createCanonical(configLocaleString.c_str());
 176                 if (ValidateLocale(configLocale))
 177                 {
 178                         outLocale = configLocale;
 179                         return;
 180                 }
 181                 else
 182                         LOGWARNING("The configured locale is not valid or no translations are available. Falling back to another locale.");
 183         }
 184
 185         Locale systemLocale = Locale::getDefault();
 186         if (ValidateLocale(systemLocale))
 187                 outLocale = systemLocale;
 188         else
 189                 outLocale = Locale::getUS();
 190 }
 191
 192 // Try to find the best dictionary locale based on user configuration and system locale, set the currentLocale and reload the dictionary.
 193 void L10n::ReevaluateCurrentLocaleAndReload()
 194 {
 195         std::string locale;
 196         CFG_GET_VAL("locale", locale);
 197
 198         if (locale == "long")
 199         {
 200                 // Set ICU to en_US to have a valid language for displaying dates
 201                 currentLocale = Locale::getUS();
 202                 currentLocaleIsOriginalGameLocale = false;
 203                 useLongStrings = true;
 204         }
 205         else
 206         {
 207                 GetDictionaryLocale(locale, currentLocale);
 208                 currentLocaleIsOriginalGameLocale = (currentLocale == Locale::getUS()) == TRUE;
 209                 useLongStrings = false;
 210         }
 211         LoadDictionaryForCurrentLocale();
 212 }
 213
 214 // Get all locales supported by ICU.
 215 std::vector<std::string> L10n::GetAllLocales() const
 216 {
 217         std::vector<std::string> ret;
 218         int32_t count;
 219         const Locale* icuSupportedLocales = Locale::getAvailableLocales(count);
 220         for (int i=0; i<count; ++i)
 221                 ret.push_back(icuSupportedLocales[i].getName());
 222         return ret;
 223 }
 224
 225
 226 bool L10n::UseLongStrings() const
 227 {
 228         return useLongStrings;
 229 };
 230
 231 std::vector<std::string> L10n::GetSupportedLocaleBaseNames() const
 232 {
 233         std::vector<std::string> supportedLocaleCodes;
 234         for (Locale* const& locale : availableLocales)
 235         {
 236                 if (!InDevelopmentCopy() && strcmp(locale->getBaseName(), "long") == 0)
 237                         continue;
 238                 supportedLocaleCodes.push_back(locale->getBaseName());
 239         }
 240         return supportedLocaleCodes;
 241 }
 242
 243 std::vector<std::wstring> L10n::GetSupportedLocaleDisplayNames() const
 244 {
 245         std::vector<std::wstring> supportedLocaleDisplayNames;
 246         for (Locale* const& locale : availableLocales)
 247         {
 248                 if (strcmp(locale->getBaseName(), "long") == 0)
 249                 {
 250                         if (InDevelopmentCopy())
 251                                 supportedLocaleDisplayNames.push_back(wstring_from_utf8(Translate("Long strings")));
 252                         continue;
 253                 }
 254
 255                 UnicodeString utf16LocaleDisplayName;
 256                 locale->getDisplayName(*locale, utf16LocaleDisplayName);
 257                 char localeDisplayName[512];
 258                 CheckedArrayByteSink sink(localeDisplayName, ARRAY_SIZE(localeDisplayName));
 259                 utf16LocaleDisplayName.toUTF8(sink);
 260                 ENSURE(!sink.Overflowed());
 261
 262                 supportedLocaleDisplayNames.push_back(wstring_from_utf8(std::string(localeDisplayName, sink.NumberOfBytesWritten())));
 263         }
 264         return supportedLocaleDisplayNames;
 265 }
 266
 267 std::string L10n::GetCurrentLocaleString() const
 268 {
 269         return currentLocale.getName();
 270 }
 271
 272 std::string L10n::GetLocaleLanguage(const std::string& locale) const
 273 {
 274         Locale loc = Locale::createCanonical(locale.c_str());
 275         return loc.getLanguage();
 276 }
 277
 278 std::string L10n::GetLocaleBaseName(const std::string& locale) const
 279 {
 280         Locale loc = Locale::createCanonical(locale.c_str());
 281         return loc.getBaseName();
 282 }
 283
 284 std::string L10n::GetLocaleCountry(const std::string& locale) const
 285 {
 286         Locale loc = Locale::createCanonical(locale.c_str());
 287         return loc.getCountry();
 288 }
 289
 290 std::string L10n::GetLocaleScript(const std::string& locale) const
 291 {
 292         Locale loc = Locale::createCanonical(locale.c_str());
 293         return loc.getScript();
 294 }
 295
 296 std::string L10n::Translate(const std::string& sourceString) const
 297 {
 298         if (!currentLocaleIsOriginalGameLocale)
 299                 return dictionary->translate(sourceString);
 300
 301         return sourceString;
 302 }
 303
 304 std::string L10n::TranslateWithContext(const std::string& context, const std::string& sourceString) const
 305 {
 306         if (!currentLocaleIsOriginalGameLocale)
 307                 return dictionary->translate_ctxt(context, sourceString);
 308
 309         return sourceString;
 310 }
 311
 312 std::string L10n::TranslatePlural(const std::string& singularSourceString, const std::string& pluralSourceString, int number) const
 313 {
 314         if (!currentLocaleIsOriginalGameLocale)
 315                 return dictionary->translate_plural(singularSourceString, pluralSourceString, number);
 316
 317         if (number == 1)
 318                 return singularSourceString;
 319
 320         return pluralSourceString;
 321 }
 322
 323 std::string L10n::TranslatePluralWithContext(const std::string& context, const std::string& singularSourceString, const std::string& pluralSourceString, int number) const
 324 {
 325         if (!currentLocaleIsOriginalGameLocale)
 326                 return dictionary->translate_ctxt_plural(context, singularSourceString, pluralSourceString, number);
 327
 328         if (number == 1)
 329                 return singularSourceString;
 330
 331         return pluralSourceString;
 332 }
 333
 334 std::string L10n::TranslateLines(const std::string& sourceString) const
 335 {
 336         std::string targetString;
 337         std::stringstream stringOfLines(sourceString);
 338         std::string line;
 339
 340         while (std::getline(stringOfLines, line))
 341         {
 342                 if (!line.empty())
 343                         targetString.append(Translate(line));
 344                 targetString.append("\n");
 345         }
 346
 347         return targetString;
 348 }
 349
 350 UDate L10n::ParseDateTime(const std::string& dateTimeString, const std::string& dateTimeFormat, const Locale& locale) const
 351 {
 352         UErrorCode success = U_ZERO_ERROR;
 353         UnicodeString utf16DateTimeString = UnicodeString::fromUTF8(dateTimeString.c_str());
 354         UnicodeString utf16DateTimeFormat = UnicodeString::fromUTF8(dateTimeFormat.c_str());
 355
 356         DateFormat* dateFormatter = new SimpleDateFormat(utf16DateTimeFormat, locale, success);
 357         UDate date = dateFormatter->parse(utf16DateTimeString, success);
 358         delete dateFormatter;
 359
 360         return date;
 361 }
 362
 363 std::string L10n::LocalizeDateTime(const UDate dateTime, const DateTimeType& type, const DateFormat::EStyle& style) const
 364 {
 365         UnicodeString utf16Date;
 366
 367         DateFormat* dateFormatter = CreateDateTimeInstance(type, style, currentLocale);
 368         dateFormatter->format(dateTime, utf16Date);
 369         char utf8Date[512];
 370         CheckedArrayByteSink sink(utf8Date, ARRAY_SIZE(utf8Date));
 371         utf16Date.toUTF8(sink);
 372         ENSURE(!sink.Overflowed());
 373         delete dateFormatter;
 374
 375         return std::string(utf8Date, sink.NumberOfBytesWritten());
 376 }
 377
 378 std::string L10n::FormatMillisecondsIntoDateString(const UDate milliseconds, const std::string& formatString, bool useLocalTimezone) const
 379 {
 380         UErrorCode status = U_ZERO_ERROR;
 381         UnicodeString dateString;
 382         std::string resultString;
 383
 384         UnicodeString unicodeFormat = UnicodeString::fromUTF8(formatString.c_str());
 385         SimpleDateFormat* dateFormat = new SimpleDateFormat(unicodeFormat, status);
 386         if (U_FAILURE(status))
 387                 LOGERROR("Error creating SimpleDateFormat: %s", u_errorName(status));
 388
 389         const TimeZone* timeZone = useLocalTimezone ? TimeZone::createDefault() : TimeZone::getGMT() ;
 390
 391         status = U_ZERO_ERROR;
 392         Calendar* calendar = Calendar::createInstance(*timeZone, currentLocale, status);
 393         if (U_FAILURE(status))
 394                 LOGERROR("Error creating calendar: %s", u_errorName(status));
 395
 396         dateFormat->adoptCalendar(calendar);
 397         dateFormat->format(milliseconds, dateString);
 398         delete dateFormat;
 399
 400         dateString.toUTF8String(resultString);
 401         return resultString;
 402 }
 403
 404 std::string L10n::FormatDecimalNumberIntoString(double number) const
 405 {
 406         UErrorCode success = U_ZERO_ERROR;
 407         UnicodeString utf16Number;
 408         NumberFormat* numberFormatter = NumberFormat::createInstance(currentLocale, UNUM_DECIMAL, success);
 409         numberFormatter->format(number, utf16Number);
 410         char utf8Number[512];
 411         CheckedArrayByteSink sink(utf8Number, ARRAY_SIZE(utf8Number));
 412         utf16Number.toUTF8(sink);
 413         ENSURE(!sink.Overflowed());
 414
 415         return std::string(utf8Number, sink.NumberOfBytesWritten());
 416 }
 417
 418 VfsPath L10n::LocalizePath(const VfsPath& sourcePath) const
 419 {
 420         VfsPath localizedPath = sourcePath.Parent() / L"l10n" / wstring_from_utf8(currentLocale.getLanguage()) / sourcePath.Filename();
 421         if (!VfsFileExists(localizedPath))
 422                 return sourcePath;
 423
 424         return localizedPath;
 425 }
 426
 427 Status L10n::ReloadChangedFile(const VfsPath& path)
 428 {
 429         if (!boost::algorithm::starts_with(path.string(), L"l10n/"))
 430                 return INFO::OK;
 431
 432         if (path.Extension() != L".po")
 433                 return INFO::OK;
 434
 435         // If the file was deleted, ignore it
 436         if (!VfsFileExists(path))
 437                 return INFO::OK;
 438
 439         std::wstring dictName = GetFallbackToAvailableDictLocale(currentLocale);
 440         if (useLongStrings)
 441                 dictName = L"long";
 442         if (dictName.empty())
 443                 return INFO::OK;
 444
 445         // Only the currently used language is loaded, so ignore all others
 446         if (path.string().rfind(dictName) == std::string::npos)
 447                 return INFO::OK;
 448
 449         LOGMESSAGE("Hotloading translations from '%s'", path.string8());
 450
 451         CVFSFile file;
 452         if (file.Load(g_VFS, path) != PSRETURN_OK)
 453         {
 454                 LOGERROR("Failed to read translations from '%s'", path.string8());
 455                 return ERR::FAIL;
 456         }
 457
 458         std::string content = file.DecodeUTF8();
 459         ReadPoIntoDictionary(content, dictionary);
 460
 461         if (g_GUI)
 462                 g_GUI->ReloadAllPages();
 463
 464         return INFO::OK;
 465 }
 466
 467 void L10n::LoadDictionaryForCurrentLocale()
 468 {
 469         delete dictionary;
 470         dictionary = new tinygettext::Dictionary();
 471
 472         VfsPaths filenames;
 473
 474         if (useLongStrings)
 475         {
 476                 if (vfs::GetPathnames(g_VFS, L"l10n/", L"long.*.po", filenames) < 0)
 477                         return;
 478         }
 479         else
 480         {
 481                 std::wstring dictName = GetFallbackToAvailableDictLocale(currentLocale);
 482                 if (vfs::GetPathnames(g_VFS, L"l10n/", dictName.append(L".*.po").c_str(), filenames) < 0)
 483                 {
 484                         LOGERROR("No files for the dictionary found, but at this point the input should already be validated!");
 485                         return;
 486                 }
 487         }
 488
 489         for (const VfsPath& path : filenames)
 490         {
 491                 CVFSFile file;
 492                 file.Load(g_VFS, path);
 493                 std::string content = file.DecodeUTF8();
 494                 ReadPoIntoDictionary(content, dictionary);
 495         }
 496 }
 497
 498 void L10n::LoadListOfAvailableLocales()
 499 {
 500         for (Locale* const& locale : availableLocales)
 501                 delete locale;
 502         availableLocales.clear();
 503
 504         Locale* defaultLocale = new Locale(Locale::getUS());
 505         availableLocales.push_back(defaultLocale); // Always available.
 506
 507         VfsPaths filenames;
 508         if (vfs::GetPathnames(g_VFS, L"l10n/", L"*.po", filenames) < 0)
 509                 return;
 510
 511         for (const VfsPath& path : filenames)
 512         {
 513                 // Note: PO files follow this naming convention: “l10n/<locale code>.<mod name>.po”. For example: “l10n/gl.public.po”.
 514                 std::string filename = utf8_from_wstring(path.string()).substr(strlen("l10n/"));
 515                 size_t lengthToFirstDot = filename.find('.');
 516                 std::string localeCode = filename.substr(0, lengthToFirstDot);
 517                 Locale* locale = new Locale(Locale::createCanonical(localeCode.c_str()));
 518
 519                 auto it = std::find_if(availableLocales.begin(), availableLocales.end(), [&locale](Locale* const& l) {
 520                         return *locale == *l;
 521                 });
 522                 if (it != availableLocales.end())
 523                 {
 524                         delete locale;
 525                         continue;
 526                 }
 527
 528                 availableLocales.push_back(locale);
 529         }
 530 }
 531
 532 void L10n::ReadPoIntoDictionary(const std::string& poContent, tinygettext::Dictionary* dictionary) const
 533 {
 534         try
 535         {
 536                 std::istringstream inputStream(poContent);
 537                 tinygettext::POParser::parse("virtual PO file", inputStream, *dictionary);
 538         }
 539         catch(std::exception& e)
 540         {
 541                 LOGERROR("[Localization] Exception while reading virtual PO file: %s", e.what());
 542         }
 543 }
 544
 545 DateFormat* L10n::CreateDateTimeInstance(const L10n::DateTimeType& type, const DateFormat::EStyle& style, const Locale& locale) const
 546 {
 547         switch(type)
 548         {
 549         case Date:
 550                 return SimpleDateFormat::createDateInstance(style, locale);
 551
 552         case Time:
 553                 return SimpleDateFormat::createTimeInstance(style, locale);
 554
 555         case DateTime:
 556         default:
 557                 return SimpleDateFormat::createDateTimeInstance(style, style, locale);
 558         }
 559 }