1 /* Copyright (C) 2023 Wildfire Games.
3 * Permission is hereby granted, free of charge, to any person obtaining
4 * a copy of this software and associated documentation files (the
5 * "Software"), to deal in the Software without restriction, including
6 * without limitation the rights to use, copy, modify, merge, publish,
7 * distribute, sublicense, and/or sell copies of the Software, and to
8 * permit persons to whom the Software is furnished to do so, subject to
9 * the following conditions:
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
17 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
18 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
19 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
20 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
23 #include "precompiled.h"
25 #include "i18n/L10n.h"
27 #include "gui/GUIManager.h"
28 #include "lib/external_libraries/tinygettext.h"
29 #include "lib/file/file_system.h"
31 #include "ps/CLogger.h"
32 #include "ps/ConfigDB.h"
33 #include "ps/Filesystem.h"
34 #include "ps/GameSetup/GameSetup.h"
36 #include <boost/algorithm/string/predicate.hpp>
43 * Determines the list of locales that the game supports.
45 * LoadListOfAvailableLocales() checks the locale codes of the translation
46 * files in the 'l10n' folder of the virtual filesystem. If it finds a
47 * translation file prefixed with a locale code followed by a dot, it
48 * determines that the game supports that locale.
50 std::vector
<icu::Locale
> LoadListOfAvailableLocales()
52 // US is always available.
53 std::vector
<icu::Locale
> availableLocales
{icu::Locale::getUS()};
56 if (vfs::GetPathnames(g_VFS
, L
"l10n/", L
"*.po", filenames
) < 0)
57 return availableLocales
;
59 availableLocales
.reserve(filenames
.size());
61 for (const VfsPath
& path
: filenames
)
63 // Note: PO files follow this naming convention: "l10n/<locale code>.<mod name>.po". For example: "l10n/gl.public.po".
64 const std::string filename
= utf8_from_wstring(path
.string()).substr(strlen("l10n/"));
65 const size_t lengthToFirstDot
= filename
.find('.');
66 const std::string localeCode
= filename
.substr(0, lengthToFirstDot
);
67 icu::Locale
locale(icu::Locale::createCanonical(localeCode
.c_str()));
68 const auto it
= std::find(availableLocales
.begin(), availableLocales
.end(), locale
);
69 if (it
!= availableLocales
.end())
72 availableLocales
.push_back(std::move(locale
));
74 availableLocales
.shrink_to_fit();
76 return availableLocales
;
79 Status
ReloadChangedFileCB(void* param
, const VfsPath
& path
)
81 return static_cast<L10n
*>(param
)->ReloadChangedFile(path
);
85 * Loads the specified content of a PO file into the specified dictionary.
87 * Used by LoadDictionaryForCurrentLocale() to add entries to the game
88 * translations @link dictionary.
90 * @param poContent Content of a PO file as a string.
91 * @param dictionary Dictionary where the entries from the PO file should be
94 void ReadPoIntoDictionary(const std::string
& poContent
, tinygettext::Dictionary
* dictionary
)
98 std::istringstream
inputStream(poContent
);
99 tinygettext::POParser::parse("virtual PO file", inputStream
, *dictionary
);
101 catch (std::exception
& e
)
103 LOGERROR("[Localization] Exception while reading virtual PO file: %s", e
.what());
108 * Creates an ICU date formatted with the specified settings.
110 * @param type Whether formatted dates must show both the date and the time,
111 * only the date or only the time.
112 * @param style ICU style to format dates by default.
113 * @param locale Locale that the date formatter should use to parse strings.
114 * It has no relevance for date formatting, only matters for date
116 * @return ICU date formatter.
118 std::unique_ptr
<icu::DateFormat
> CreateDateTimeInstance(const L10n::DateTimeType
& type
, const icu::DateFormat::EStyle
& style
, const icu::Locale
& locale
)
123 return std::unique_ptr
<icu::DateFormat
>{
124 icu::SimpleDateFormat::createDateInstance(style
, locale
)};
127 return std::unique_ptr
<icu::DateFormat
>{
128 icu::SimpleDateFormat::createTimeInstance(style
, locale
)};
130 case L10n::DateTime
: FALLTHROUGH
;
132 return std::unique_ptr
<icu::DateFormat
>{
133 icu::SimpleDateFormat::createDateTimeInstance(style
, style
, locale
)};
137 } // anonymous namespace
140 : m_Dictionary(std::make_unique
<tinygettext::Dictionary
>())
142 // Determine whether or not to print tinygettext messages to the standard
143 // error output, which it tinygettext's default behavior, but not ours.
144 bool tinygettext_debug
= false;
145 CFG_GET_VAL("tinygettext.debug", tinygettext_debug
);
146 if (!tinygettext_debug
)
148 tinygettext::Log::log_info_callback
= 0;
149 tinygettext::Log::log_warning_callback
= 0;
150 tinygettext::Log::log_error_callback
= 0;
153 m_AvailableLocales
= LoadListOfAvailableLocales();
155 ReevaluateCurrentLocaleAndReload();
158 RegisterFileReloadFunc(ReloadChangedFileCB
, this);
163 UnregisterFileReloadFunc(ReloadChangedFileCB
, this);
166 const icu::Locale
& L10n::GetCurrentLocale() const
168 return m_CurrentLocale
;
171 bool L10n::SaveLocale(const std::string
& localeCode
) const
173 if (localeCode
== "long" && InDevelopmentCopy())
175 g_ConfigDB
.SetValueString(CFG_USER
, "locale", "long");
178 return SaveLocale(icu::Locale(icu::Locale::createCanonical(localeCode
.c_str())));
181 bool L10n::SaveLocale(const icu::Locale
& locale
) const
183 if (!ValidateLocale(locale
))
186 g_ConfigDB
.SetValueString(CFG_USER
, "locale", locale
.getName());
187 return g_ConfigDB
.WriteValueToFile(CFG_USER
, "locale", locale
.getName());
190 bool L10n::ValidateLocale(const std::string
& localeCode
) const
192 return ValidateLocale(icu::Locale::createCanonical(localeCode
.c_str()));
195 // Returns true if both of these conditions are true:
196 // 1. ICU has resources for that locale (which also ensures it's a valid locale string)
197 // 2. Either a dictionary for language_country or for language is available.
198 bool L10n::ValidateLocale(const icu::Locale
& locale
) const
200 if (locale
.isBogus())
203 return !GetFallbackToAvailableDictLocale(locale
).empty();
206 std::vector
<std::wstring
> L10n::GetDictionariesForLocale(const std::string
& locale
) const
208 std::vector
<std::wstring
> ret
;
211 std::wstring dictName
= GetFallbackToAvailableDictLocale(icu::Locale::createCanonical(locale
.c_str()));
212 vfs::GetPathnames(g_VFS
, L
"l10n/", dictName
.append(L
".*.po").c_str(), filenames
);
214 for (const VfsPath
& path
: filenames
)
215 ret
.push_back(path
.Filename().string());
220 std::wstring
L10n::GetFallbackToAvailableDictLocale(const std::string
& locale
) const
222 return GetFallbackToAvailableDictLocale(icu::Locale::createCanonical(locale
.c_str()));
225 std::wstring
L10n::GetFallbackToAvailableDictLocale(const icu::Locale
& locale
) const
227 std::wstringstream stream
;
229 const auto checkLangAndCountry
= [&locale
](const icu::Locale
& l
)
231 return strcmp(locale
.getLanguage(), l
.getLanguage()) == 0
232 && strcmp(locale
.getCountry(), l
.getCountry()) == 0;
235 if (strcmp(locale
.getCountry(), "") != 0
236 && std::find_if(m_AvailableLocales
.begin(), m_AvailableLocales
.end(), checkLangAndCountry
) !=
237 m_AvailableLocales
.end())
239 stream
<< locale
.getLanguage() << L
"_" << locale
.getCountry();
243 const auto checkLang
= [&locale
](const icu::Locale
& l
)
245 return strcmp(locale
.getLanguage(), l
.getLanguage()) == 0;
248 if (std::find_if(m_AvailableLocales
.begin(), m_AvailableLocales
.end(), checkLang
) !=
249 m_AvailableLocales
.end())
251 stream
<< locale
.getLanguage();
258 std::string
L10n::GetDictionaryLocale(const std::string
& configLocaleString
) const
261 GetDictionaryLocale(configLocaleString
, out
);
262 return out
.getName();
265 // First, try to get a valid locale from the config, then check if the system locale can be used and otherwise fall back to en_US.
266 void L10n::GetDictionaryLocale(const std::string
& configLocaleString
, icu::Locale
& outLocale
) const
268 if (!configLocaleString
.empty())
270 icu::Locale configLocale
= icu::Locale::createCanonical(configLocaleString
.c_str());
271 if (ValidateLocale(configLocale
))
273 outLocale
= configLocale
;
277 LOGWARNING("The configured locale is not valid or no translations are available. Falling back to another locale.");
280 icu::Locale systemLocale
= icu::Locale::getDefault();
281 if (ValidateLocale(systemLocale
))
282 outLocale
= systemLocale
;
284 outLocale
= icu::Locale::getUS();
287 // Try to find the best dictionary locale based on user configuration and system locale, set the currentLocale and reload the dictionary.
288 void L10n::ReevaluateCurrentLocaleAndReload()
291 CFG_GET_VAL("locale", locale
);
293 if (locale
== "long")
295 // Set ICU to en_US to have a valid language for displaying dates
296 m_CurrentLocale
= icu::Locale::getUS();
297 m_CurrentLocaleIsOriginalGameLocale
= false;
298 m_UseLongStrings
= true;
302 GetDictionaryLocale(locale
, m_CurrentLocale
);
303 m_CurrentLocaleIsOriginalGameLocale
= (m_CurrentLocale
== icu::Locale::getUS()) == 1;
304 m_UseLongStrings
= false;
306 LoadDictionaryForCurrentLocale();
309 // Get all locales supported by ICU.
310 std::vector
<std::string
> L10n::GetAllLocales() const
312 std::vector
<std::string
> ret
;
314 const icu::Locale
* icuSupportedLocales
= icu::Locale::getAvailableLocales(count
);
315 for (int i
=0; i
<count
; ++i
)
316 ret
.push_back(icuSupportedLocales
[i
].getName());
321 bool L10n::UseLongStrings() const
323 return m_UseLongStrings
;
326 std::vector
<std::string
> L10n::GetSupportedLocaleBaseNames() const
328 std::vector
<std::string
> supportedLocaleCodes
;
329 for (const icu::Locale
& locale
: m_AvailableLocales
)
331 if (!InDevelopmentCopy() && strcmp(locale
.getBaseName(), "long") == 0)
333 supportedLocaleCodes
.push_back(locale
.getBaseName());
335 return supportedLocaleCodes
;
338 std::vector
<std::wstring
> L10n::GetSupportedLocaleDisplayNames() const
340 std::vector
<std::wstring
> supportedLocaleDisplayNames
;
341 for (const icu::Locale
& locale
: m_AvailableLocales
)
343 if (strcmp(locale
.getBaseName(), "long") == 0)
345 if (InDevelopmentCopy())
346 supportedLocaleDisplayNames
.push_back(wstring_from_utf8(Translate("Long strings")));
350 icu::UnicodeString utf16LocaleDisplayName
;
351 locale
.getDisplayName(locale
, utf16LocaleDisplayName
);
352 char localeDisplayName
[512];
353 icu::CheckedArrayByteSink
sink(localeDisplayName
, ARRAY_SIZE(localeDisplayName
));
354 utf16LocaleDisplayName
.toUTF8(sink
);
355 ENSURE(!sink
.Overflowed());
357 supportedLocaleDisplayNames
.push_back(wstring_from_utf8(std::string(localeDisplayName
, sink
.NumberOfBytesWritten())));
359 return supportedLocaleDisplayNames
;
362 std::string
L10n::GetCurrentLocaleString() const
364 return m_CurrentLocale
.getName();
367 std::string
L10n::GetLocaleLanguage(const std::string
& locale
) const
369 icu::Locale loc
= icu::Locale::createCanonical(locale
.c_str());
370 return loc
.getLanguage();
373 std::string
L10n::GetLocaleBaseName(const std::string
& locale
) const
375 icu::Locale loc
= icu::Locale::createCanonical(locale
.c_str());
376 return loc
.getBaseName();
379 std::string
L10n::GetLocaleCountry(const std::string
& locale
) const
381 icu::Locale loc
= icu::Locale::createCanonical(locale
.c_str());
382 return loc
.getCountry();
385 std::string
L10n::GetLocaleScript(const std::string
& locale
) const
387 icu::Locale loc
= icu::Locale::createCanonical(locale
.c_str());
388 return loc
.getScript();
391 std::string
L10n::Translate(const std::string
& sourceString
) const
393 if (!m_CurrentLocaleIsOriginalGameLocale
)
394 return m_Dictionary
->translate(sourceString
);
399 std::string
L10n::TranslateWithContext(const std::string
& context
, const std::string
& sourceString
) const
401 if (!m_CurrentLocaleIsOriginalGameLocale
)
402 return m_Dictionary
->translate_ctxt(context
, sourceString
);
407 std::string
L10n::TranslatePlural(const std::string
& singularSourceString
, const std::string
& pluralSourceString
, int number
) const
409 if (!m_CurrentLocaleIsOriginalGameLocale
)
410 return m_Dictionary
->translate_plural(singularSourceString
, pluralSourceString
, number
);
413 return singularSourceString
;
415 return pluralSourceString
;
418 std::string
L10n::TranslatePluralWithContext(const std::string
& context
, const std::string
& singularSourceString
, const std::string
& pluralSourceString
, int number
) const
420 if (!m_CurrentLocaleIsOriginalGameLocale
)
421 return m_Dictionary
->translate_ctxt_plural(context
, singularSourceString
, pluralSourceString
, number
);
424 return singularSourceString
;
426 return pluralSourceString
;
429 std::string
L10n::TranslateLines(const std::string
& sourceString
) const
431 std::string targetString
;
432 std::stringstream
stringOfLines(sourceString
);
435 while (std::getline(stringOfLines
, line
))
438 targetString
.append(Translate(line
));
439 targetString
.append("\n");
445 UDate
L10n::ParseDateTime(const std::string
& dateTimeString
, const std::string
& dateTimeFormat
, const icu::Locale
& locale
) const
447 UErrorCode success
= U_ZERO_ERROR
;
448 icu::UnicodeString utf16DateTimeString
= icu::UnicodeString::fromUTF8(dateTimeString
.c_str());
449 icu::UnicodeString utf16DateTimeFormat
= icu::UnicodeString::fromUTF8(dateTimeFormat
.c_str());
451 const icu::SimpleDateFormat dateFormatter
{utf16DateTimeFormat
, locale
, success
};
452 return dateFormatter
.parse(utf16DateTimeString
, success
);
455 std::string
L10n::LocalizeDateTime(const UDate dateTime
, const DateTimeType
& type
, const icu::DateFormat::EStyle
& style
) const
457 icu::UnicodeString utf16Date
;
459 const std::unique_ptr
<const icu::DateFormat
> dateFormatter
{
460 CreateDateTimeInstance(type
, style
, m_CurrentLocale
)};
461 dateFormatter
->format(dateTime
, utf16Date
);
463 icu::CheckedArrayByteSink
sink(utf8Date
, ARRAY_SIZE(utf8Date
));
464 utf16Date
.toUTF8(sink
);
465 ENSURE(!sink
.Overflowed());
467 return std::string(utf8Date
, sink
.NumberOfBytesWritten());
470 std::string
L10n::FormatMillisecondsIntoDateString(const UDate milliseconds
, const std::string
& formatString
, bool useLocalTimezone
) const
472 UErrorCode status
= U_ZERO_ERROR
;
473 icu::UnicodeString dateString
;
474 std::string resultString
;
476 icu::UnicodeString unicodeFormat
= icu::UnicodeString::fromUTF8(formatString
.c_str());
477 icu::SimpleDateFormat dateFormat
{unicodeFormat
, status
};
478 if (U_FAILURE(status
))
479 LOGERROR("Error creating SimpleDateFormat: %s", u_errorName(status
));
481 status
= U_ZERO_ERROR
;
482 std::unique_ptr
<icu::Calendar
> calendar
{
484 icu::Calendar::createInstance(m_CurrentLocale
, status
) :
485 icu::Calendar::createInstance(*icu::TimeZone::getGMT(), m_CurrentLocale
, status
)};
487 if (U_FAILURE(status
))
488 LOGERROR("Error creating calendar: %s", u_errorName(status
));
490 dateFormat
.adoptCalendar(calendar
.release());
491 dateFormat
.format(milliseconds
, dateString
);
493 dateString
.toUTF8String(resultString
);
497 std::string
L10n::FormatDecimalNumberIntoString(double number
) const
499 UErrorCode success
= U_ZERO_ERROR
;
500 icu::UnicodeString utf16Number
;
501 std::unique_ptr
<icu::NumberFormat
> numberFormatter
{
502 icu::NumberFormat::createInstance(m_CurrentLocale
, UNUM_DECIMAL
, success
)};
503 numberFormatter
->format(number
, utf16Number
);
504 char utf8Number
[512];
505 icu::CheckedArrayByteSink
sink(utf8Number
, ARRAY_SIZE(utf8Number
));
506 utf16Number
.toUTF8(sink
);
507 ENSURE(!sink
.Overflowed());
509 return std::string(utf8Number
, sink
.NumberOfBytesWritten());
512 VfsPath
L10n::LocalizePath(const VfsPath
& sourcePath
) const
514 VfsPath localizedPath
= sourcePath
.Parent() / L
"l10n" /
515 wstring_from_utf8(m_CurrentLocale
.getLanguage()) / sourcePath
.Filename();
516 if (!VfsFileExists(localizedPath
))
519 return localizedPath
;
522 Status
L10n::ReloadChangedFile(const VfsPath
& path
)
524 if (!boost::algorithm::starts_with(path
.string(), L
"l10n/"))
527 if (path
.Extension() != L
".po")
530 // If the file was deleted, ignore it
531 if (!VfsFileExists(path
))
534 std::wstring dictName
= GetFallbackToAvailableDictLocale(m_CurrentLocale
);
535 if (m_UseLongStrings
)
537 if (dictName
.empty())
540 // Only the currently used language is loaded, so ignore all others
541 if (path
.string().rfind(dictName
) == std::string::npos
)
544 LOGMESSAGE("Hotloading translations from '%s'", path
.string8());
547 if (file
.Load(g_VFS
, path
) != PSRETURN_OK
)
549 LOGERROR("Failed to read translations from '%s'", path
.string8());
553 std::string content
= file
.DecodeUTF8();
554 ReadPoIntoDictionary(content
, m_Dictionary
.get());
557 g_GUI
->ReloadAllPages();
562 void L10n::LoadDictionaryForCurrentLocale()
564 m_Dictionary
= std::make_unique
<tinygettext::Dictionary
>();
567 if (m_UseLongStrings
)
569 if (vfs::GetPathnames(g_VFS
, L
"l10n/", L
"long.*.po", filenames
) < 0)
574 std::wstring dictName
= GetFallbackToAvailableDictLocale(m_CurrentLocale
);
575 if (vfs::GetPathnames(g_VFS
, L
"l10n/", dictName
.append(L
".*.po").c_str(), filenames
) < 0)
577 LOGERROR("No files for the dictionary found, but at this point the input should already be validated!");
582 for (const VfsPath
& path
: filenames
)
585 file
.Load(g_VFS
, path
);
586 std::string content
= file
.DecodeUTF8();
587 ReadPoIntoDictionary(content
, m_Dictionary
.get());