Lib/gettext.py

   1 """Internationalization and localization support.
   2
   3 This module provides internationalization (I18N) and localization (L10N)
   4 support for your Python programs by providing an interface to the GNU gettext
   5 message catalog library.
   6
   7 I18N refers to the operation by which a program is made aware of multiple
   8 languages.  L10N refers to the adaptation of your program, once
   9 internationalized, to the local language and cultural habits.
  10
  11 """
  12
  13 # This module represents the integration of work, contributions, feedback, and
  14 # suggestions from the following people:
  15 #
  16 # Martin von Loewis, who wrote the initial implementation of the underlying
  17 # C-based libintlmodule (later renamed _gettext), along with a skeletal
  18 # gettext.py implementation.
  19 #
  20 # Peter Funk, who wrote fintl.py, a fairly complete wrapper around intlmodule,
  21 # which also included a pure-Python implementation to read .mo files if
  22 # intlmodule wasn't available.
  23 #
  24 # James Henstridge, who also wrote a gettext.py module, which has some
  25 # interesting, but currently unsupported experimental features: the notion of
  26 # a Catalog class and instances, and the ability to add to a catalog file via
  27 # a Python API.
  28 #
  29 # Barry Warsaw integrated these modules, wrote the .install() API and code,
  30 # and conformed all C and Python code to Python's coding standards.
  31 #
  32 # Francois Pinard and Marc-Andre Lemburg also contributed valuably to this
  33 # module.
  34 #
  35 # J. David Ibanez implemented plural forms. Bruno Haible fixed some bugs.
  36 #
  37 # TODO:
  38 # - Lazy loading of .mo files.  Currently the entire catalog is loaded into
  39 #   memory, but that's probably bad for large translated programs.  Instead,
  40 #   the lexical sort of original strings in GNU .mo files should be exploited
  41 #   to do binary searches and lazy initializations.  Or you might want to use
  42 #   the undocumented double-hash algorithm for .mo files with hash tables, but
  43 #   you'll need to study the GNU gettext code to do this.
  44 #
  45 # - Support Solaris .mo file formats.  Unfortunately, we've been unable to
  46 #   find this format documented anywhere.
  47
  48
  49 import locale, copy, os, re, struct, sys
  50 from errno import ENOENT
  51
  52
  53 __all__ = ['NullTranslations', 'GNUTranslations', 'Catalog',
  54            'find', 'translation', 'install', 'textdomain', 'bindtextdomain',
  55            'dgettext', 'dngettext', 'gettext', 'ngettext',
  56            ]
  57
  58 _default_localedir = os.path.join(sys.prefix, 'share', 'locale')
  59
  60
  61 def test(condition, true, false):
  62     """
  63     Implements the C expression:
  64
  65       condition ? true : false
  66
  67     Required to correctly interpret plural forms.
  68     """
  69     if condition:
  70         return true
  71     else:
  72         return false
  73
  74
  75 def c2py(plural):
  76     """Gets a C expression as used in PO files for plural forms and returns a
  77     Python lambda function that implements an equivalent expression.
  78     """
  79     # Security check, allow only the "n" identifier
  80     from io import StringIO
  81     import token, tokenize
  82     tokens = tokenize.generate_tokens(StringIO(plural).readline)
  83     try:
  84         danger = [x for x in tokens if x[0] == token.NAME and x[1] != 'n']
  85     except tokenize.TokenError:
  86         raise ValueError('plural forms expression error, maybe unbalanced parenthesis')
  87     else:
  88         if danger:
  89             raise ValueError('plural forms expression could be dangerous')
  90
  91     # Replace some C operators by their Python equivalents
  92     plural = plural.replace('&&', ' and ')
  93     plural = plural.replace('||', ' or ')
  94
  95     expr = re.compile(r'\!([^=])')
  96     plural = expr.sub(' not \\1', plural)
  97
  98     # Regular expression and replacement function used to transform
  99     # "a?b:c" to "test(a,b,c)".
 100     expr = re.compile(r'(.*?)\?(.*?):(.*)')
 101     def repl(x):
 102         return "test(%s, %s, %s)" % (x.group(1), x.group(2),
 103                                      expr.sub(repl, x.group(3)))
 104
 105     # Code to transform the plural expression, taking care of parentheses
 106     stack = ['']
 107     for c in plural:
 108         if c == '(':
 109             stack.append('')
 110         elif c == ')':
 111             if len(stack) == 1:
 112                 # Actually, we never reach this code, because unbalanced
 113                 # parentheses get caught in the security check at the
 114                 # beginning.
 115                 raise ValueError('unbalanced parenthesis in plural form')
 116             s = expr.sub(repl, stack.pop())
 117             stack[-1] += '(%s)' % s
 118         else:
 119             stack[-1] += c
 120     plural = expr.sub(repl, stack.pop())
 121
 122     return eval('lambda n: int(%s)' % plural)
 123
 124
 125
 126 def _expand_lang(locale):
 127     from locale import normalize
 128     locale = normalize(locale)
 129     COMPONENT_CODESET   = 1 << 0
 130     COMPONENT_TERRITORY = 1 << 1
 131     COMPONENT_MODIFIER  = 1 << 2
 132     # split up the locale into its base components
 133     mask = 0
 134     pos = locale.find('@')
 135     if pos >= 0:
 136         modifier = locale[pos:]
 137         locale = locale[:pos]
 138         mask |= COMPONENT_MODIFIER
 139     else:
 140         modifier = ''
 141     pos = locale.find('.')
 142     if pos >= 0:
 143         codeset = locale[pos:]
 144         locale = locale[:pos]
 145         mask |= COMPONENT_CODESET
 146     else:
 147         codeset = ''
 148     pos = locale.find('_')
 149     if pos >= 0:
 150         territory = locale[pos:]
 151         locale = locale[:pos]
 152         mask |= COMPONENT_TERRITORY
 153     else:
 154         territory = ''
 155     language = locale
 156     ret = []
 157     for i in range(mask+1):
 158         if not (i & ~mask):  # if all components for this combo exist ...
 159             val = language
 160             if i & COMPONENT_TERRITORY: val += territory
 161             if i & COMPONENT_CODESET:   val += codeset
 162             if i & COMPONENT_MODIFIER:  val += modifier
 163             ret.append(val)
 164     ret.reverse()
 165     return ret
 166
 167
 168
 169 class NullTranslations:
 170     def __init__(self, fp=None):
 171         self._info = {}
 172         self._charset = None
 173         self._output_charset = None
 174         self._fallback = None
 175         if fp is not None:
 176             self._parse(fp)
 177
 178     def _parse(self, fp):
 179         pass
 180
 181     def add_fallback(self, fallback):
 182         if self._fallback:
 183             self._fallback.add_fallback(fallback)
 184         else:
 185             self._fallback = fallback
 186
 187     def gettext(self, message):
 188         if self._fallback:
 189             return self._fallback.gettext(message)
 190         return message
 191
 192     def lgettext(self, message):
 193         if self._fallback:
 194             return self._fallback.lgettext(message)
 195         return message
 196
 197     def ngettext(self, msgid1, msgid2, n):
 198         if self._fallback:
 199             return self._fallback.ngettext(msgid1, msgid2, n)
 200         if n == 1:
 201             return msgid1
 202         else:
 203             return msgid2
 204
 205     def lngettext(self, msgid1, msgid2, n):
 206         if self._fallback:
 207             return self._fallback.lngettext(msgid1, msgid2, n)
 208         if n == 1:
 209             return msgid1
 210         else:
 211             return msgid2
 212
 213     def info(self):
 214         return self._info
 215
 216     def charset(self):
 217         return self._charset
 218
 219     def output_charset(self):
 220         return self._output_charset
 221
 222     def set_output_charset(self, charset):
 223         self._output_charset = charset
 224
 225     def install(self, names=None):
 226         import builtins
 227         builtins.__dict__['_'] = self.gettext
 228         if hasattr(names, "__contains__"):
 229             if "gettext" in names:
 230                 builtins.__dict__['gettext'] = builtins.__dict__['_']
 231             if "ngettext" in names:
 232                 builtins.__dict__['ngettext'] = self.ngettext
 233             if "lgettext" in names:
 234                 builtins.__dict__['lgettext'] = self.lgettext
 235             if "lngettext" in names:
 236                 builtins.__dict__['lngettext'] = self.lngettext
 237
 238
 239 class GNUTranslations(NullTranslations):
 240     # Magic number of .mo files
 241     LE_MAGIC = 0x950412de
 242     BE_MAGIC = 0xde120495
 243
 244     def _parse(self, fp):
 245         """Override this method to support alternative .mo formats."""
 246         unpack = struct.unpack
 247         filename = getattr(fp, 'name', '')
 248         # Parse the .mo file header, which consists of 5 little endian 32
 249         # bit words.
 250         self._catalog = catalog = {}
 251         self.plural = lambda n: int(n != 1) # germanic plural by default
 252         buf = fp.read()
 253         buflen = len(buf)
 254         # Are we big endian or little endian?
 255         magic = unpack('<I', buf[:4])[0]
 256         if magic == self.LE_MAGIC:
 257             version, msgcount, masteridx, transidx = unpack('<4I', buf[4:20])
 258             ii = '<II'
 259         elif magic == self.BE_MAGIC:
 260             version, msgcount, masteridx, transidx = unpack('>4I', buf[4:20])
 261             ii = '>II'
 262         else:
 263             raise IOError(0, 'Bad magic number', filename)
 264         # Now put all messages from the .mo file buffer into the catalog
 265         # dictionary.
 266         for i in range(0, msgcount):
 267             mlen, moff = unpack(ii, buf[masteridx:masteridx+8])
 268             mend = moff + mlen
 269             tlen, toff = unpack(ii, buf[transidx:transidx+8])
 270             tend = toff + tlen
 271             if mend < buflen and tend < buflen:
 272                 msg = buf[moff:mend]
 273                 tmsg = buf[toff:tend]
 274             else:
 275                 raise IOError(0, 'File is corrupt', filename)
 276             # See if we're looking at GNU .mo conventions for metadata
 277             if mlen == 0:
 278                 # Catalog description
 279                 lastk = k = None
 280                 for b_item in tmsg.split('\n'.encode("ascii")):
 281                     item = b_item.decode().strip()
 282                     if not item:
 283                         continue
 284                     if ':' in item:
 285                         k, v = item.split(':', 1)
 286                         k = k.strip().lower()
 287                         v = v.strip()
 288                         self._info[k] = v
 289                         lastk = k
 290                     elif lastk:
 291                         self._info[lastk] += '\n' + item
 292                     if k == 'content-type':
 293                         self._charset = v.split('charset=')[1]
 294                     elif k == 'plural-forms':
 295                         v = v.split(';')
 296                         plural = v[1].split('plural=')[1]
 297                         self.plural = c2py(plural)
 298             # Note: we unconditionally convert both msgids and msgstrs to
 299             # Unicode using the character encoding specified in the charset
 300             # parameter of the Content-Type header.  The gettext documentation
 301             # strongly encourages msgids to be us-ascii, but some appliations
 302             # require alternative encodings (e.g. Zope's ZCML and ZPT).  For
 303             # traditional gettext applications, the msgid conversion will
 304             # cause no problems since us-ascii should always be a subset of
 305             # the charset encoding.  We may want to fall back to 8-bit msgids
 306             # if the Unicode conversion fails.
 307             charset = self._charset or 'ascii'
 308             if b'\x00' in msg:
 309                 # Plural forms
 310                 msgid1, msgid2 = msg.split(b'\x00')
 311                 tmsg = tmsg.split(b'\x00')
 312                 msgid1 = str(msgid1, charset)
 313                 for i, x in enumerate(tmsg):
 314                     catalog[(msgid1, i)] = str(x, charset)
 315             else:
 316                 catalog[str(msg, charset)] = str(tmsg, charset)
 317             # advance to next entry in the seek tables
 318             masteridx += 8
 319             transidx += 8
 320
 321     def lgettext(self, message):
 322         missing = object()
 323         tmsg = self._catalog.get(message, missing)
 324         if tmsg is missing:
 325             if self._fallback:
 326                 return self._fallback.lgettext(message)
 327             return message
 328         if self._output_charset:
 329             return tmsg.encode(self._output_charset)
 330         return tmsg.encode(locale.getpreferredencoding())
 331
 332     def lngettext(self, msgid1, msgid2, n):
 333         try:
 334             tmsg = self._catalog[(msgid1, self.plural(n))]
 335             if self._output_charset:
 336                 return tmsg.encode(self._output_charset)
 337             return tmsg.encode(locale.getpreferredencoding())
 338         except KeyError:
 339             if self._fallback:
 340                 return self._fallback.lngettext(msgid1, msgid2, n)
 341             if n == 1:
 342                 return msgid1
 343             else:
 344                 return msgid2
 345
 346     def gettext(self, message):
 347         missing = object()
 348         tmsg = self._catalog.get(message, missing)
 349         if tmsg is missing:
 350             if self._fallback:
 351                 return self._fallback.gettext(message)
 352             return message
 353         return tmsg
 354
 355     def ngettext(self, msgid1, msgid2, n):
 356         try:
 357             tmsg = self._catalog[(msgid1, self.plural(n))]
 358         except KeyError:
 359             if self._fallback:
 360                 return self._fallback.ngettext(msgid1, msgid2, n)
 361             if n == 1:
 362                 tmsg = msgid1
 363             else:
 364                 tmsg = msgid2
 365         return tmsg
 366
 367
 368 # Locate a .mo file using the gettext strategy
 369 def find(domain, localedir=None, languages=None, all=False):
 370     # Get some reasonable defaults for arguments that were not supplied
 371     if localedir is None:
 372         localedir = _default_localedir
 373     if languages is None:
 374         languages = []
 375         for envar in ('LANGUAGE', 'LC_ALL', 'LC_MESSAGES', 'LANG'):
 376             val = os.environ.get(envar)
 377             if val:
 378                 languages = val.split(':')
 379                 break
 380         if 'C' not in languages:
 381             languages.append('C')
 382     # now normalize and expand the languages
 383     nelangs = []
 384     for lang in languages:
 385         for nelang in _expand_lang(lang):
 386             if nelang not in nelangs:
 387                 nelangs.append(nelang)
 388     # select a language
 389     if all:
 390         result = []
 391     else:
 392         result = None
 393     for lang in nelangs:
 394         if lang == 'C':
 395             break
 396         mofile = os.path.join(localedir, lang, 'LC_MESSAGES', '%s.mo' % domain)
 397         if os.path.exists(mofile):
 398             if all:
 399                 result.append(mofile)
 400             else:
 401                 return mofile
 402     return result
 403
 404
 405
 406 # a mapping between absolute .mo file path and Translation object
 407 _translations = {}
 408
 409 def translation(domain, localedir=None, languages=None,
 410                 class_=None, fallback=False, codeset=None):
 411     if class_ is None:
 412         class_ = GNUTranslations
 413     mofiles = find(domain, localedir, languages, all=True)
 414     if not mofiles:
 415         if fallback:
 416             return NullTranslations()
 417         raise IOError(ENOENT, 'No translation file found for domain', domain)
 418     # Avoid opening, reading, and parsing the .mo file after it's been done
 419     # once.
 420     result = None
 421     for mofile in mofiles:
 422         key = os.path.abspath(mofile)
 423         t = _translations.get(key)
 424         if t is None:
 425             with open(mofile, 'rb') as fp:
 426                 t = _translations.setdefault(key, class_(fp))
 427         # Copy the translation object to allow setting fallbacks and
 428         # output charset. All other instance data is shared with the
 429         # cached object.
 430         t = copy.copy(t)
 431         if codeset:
 432             t.set_output_charset(codeset)
 433         if result is None:
 434             result = t
 435         else:
 436             result.add_fallback(t)
 437     return result
 438
 439
 440 def install(domain, localedir=None, codeset=None, names=None):
 441     t = translation(domain, localedir, fallback=True, codeset=codeset)
 442     t.install(names)
 443
 444
 445
 446 # a mapping b/w domains and locale directories
 447 _localedirs = {}
 448 # a mapping b/w domains and codesets
 449 _localecodesets = {}
 450 # current global domain, `messages' used for compatibility w/ GNU gettext
 451 _current_domain = 'messages'
 452
 453
 454 def textdomain(domain=None):
 455     global _current_domain
 456     if domain is not None:
 457         _current_domain = domain
 458     return _current_domain
 459
 460
 461 def bindtextdomain(domain, localedir=None):
 462     global _localedirs
 463     if localedir is not None:
 464         _localedirs[domain] = localedir
 465     return _localedirs.get(domain, _default_localedir)
 466
 467
 468 def bind_textdomain_codeset(domain, codeset=None):
 469     global _localecodesets
 470     if codeset is not None:
 471         _localecodesets[domain] = codeset
 472     return _localecodesets.get(domain)
 473
 474
 475 def dgettext(domain, message):
 476     try:
 477         t = translation(domain, _localedirs.get(domain, None),
 478                         codeset=_localecodesets.get(domain))
 479     except IOError:
 480         return message
 481     return t.gettext(message)
 482
 483 def ldgettext(domain, message):
 484     try:
 485         t = translation(domain, _localedirs.get(domain, None),
 486                         codeset=_localecodesets.get(domain))
 487     except IOError:
 488         return message
 489     return t.lgettext(message)
 490
 491 def dngettext(domain, msgid1, msgid2, n):
 492     try:
 493         t = translation(domain, _localedirs.get(domain, None),
 494                         codeset=_localecodesets.get(domain))
 495     except IOError:
 496         if n == 1:
 497             return msgid1
 498         else:
 499             return msgid2
 500     return t.ngettext(msgid1, msgid2, n)
 501
 502 def ldngettext(domain, msgid1, msgid2, n):
 503     try:
 504         t = translation(domain, _localedirs.get(domain, None),
 505                         codeset=_localecodesets.get(domain))
 506     except IOError:
 507         if n == 1:
 508             return msgid1
 509         else:
 510             return msgid2
 511     return t.lngettext(msgid1, msgid2, n)
 512
 513 def gettext(message):
 514     return dgettext(_current_domain, message)
 515
 516 def lgettext(message):
 517     return ldgettext(_current_domain, message)
 518
 519 def ngettext(msgid1, msgid2, n):
 520     return dngettext(_current_domain, msgid1, msgid2, n)
 521
 522 def lngettext(msgid1, msgid2, n):
 523     return ldngettext(_current_domain, msgid1, msgid2, n)
 524
 525 # dcgettext() has been deemed unnecessary and is not implemented.
 526
 527 # James Henstridge's Catalog constructor from GNOME gettext.  Documented usage
 528 # was:
 529 #
 530 #    import gettext
 531 #    cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR)
 532 #    _ = cat.gettext
 533 #    print _('Hello World')
 534
 535 # The resulting catalog object currently don't support access through a
 536 # dictionary API, which was supported (but apparently unused) in GNOME
 537 # gettext.
 538
 539 Catalog = translation