1 """Internationalization and localization support.
3 This module provides internationalization (I18N) and localization (L10N)
4 support for your Python programs by providing an interface to the GNU gettext
5 message catalog library.
7 I18N refers to the operation by which a program is made aware of multiple
8 languages. L10N refers to the adaptation of your program, once
9 internationalized, to the local language and cultural habits.
13 # This module represents the integration of work, contributions, feedback, and
14 # suggestions from the following people:
16 # Martin von Loewis, who wrote the initial implementation of the underlying
17 # C-based libintlmodule (later renamed _gettext), along with a skeletal
18 # gettext.py implementation.
20 # Peter Funk, who wrote fintl.py, a fairly complete wrapper around intlmodule,
21 # which also included a pure-Python implementation to read .mo files if
22 # intlmodule wasn't available.
24 # James Henstridge, who also wrote a gettext.py module, which has some
25 # interesting, but currently unsupported experimental features: the notion of
26 # a Catalog class and instances, and the ability to add to a catalog file via
29 # Barry Warsaw integrated these modules, wrote the .install() API and code,
30 # and conformed all C and Python code to Python's coding standards.
32 # Francois Pinard and Marc-Andre Lemburg also contributed valuably to this
35 # J. David Ibanez implemented plural forms. Bruno Haible fixed some bugs.
38 # - Lazy loading of .mo files. Currently the entire catalog is loaded into
39 # memory, but that's probably bad for large translated programs. Instead,
40 # the lexical sort of original strings in GNU .mo files should be exploited
41 # to do binary searches and lazy initializations. Or you might want to use
42 # the undocumented double-hash algorithm for .mo files with hash tables, but
43 # you'll need to study the GNU gettext code to do this.
45 # - Support Solaris .mo file formats. Unfortunately, we've been unable to
46 # find this format documented anywhere.
49 import locale
, copy
, os
, re
, struct
, sys
50 from errno
import ENOENT
53 __all__
= ['NullTranslations', 'GNUTranslations', 'Catalog',
54 'find', 'translation', 'install', 'textdomain', 'bindtextdomain',
55 'dgettext', 'dngettext', 'gettext', 'ngettext',
58 _default_localedir
= os
.path
.join(sys
.prefix
, 'share', 'locale')
61 def test(condition
, true
, false
):
63 Implements the C expression:
65 condition ? true : false
67 Required to correctly interpret plural forms.
76 """Gets a C expression as used in PO files for plural forms and returns a
77 Python lambda function that implements an equivalent expression.
79 # Security check, allow only the "n" identifier
80 from io
import StringIO
81 import token
, tokenize
82 tokens
= tokenize
.generate_tokens(StringIO(plural
).readline
)
84 danger
= [x
for x
in tokens
if x
[0] == token
.NAME
and x
[1] != 'n']
85 except tokenize
.TokenError
:
86 raise ValueError('plural forms expression error, maybe unbalanced parenthesis')
89 raise ValueError('plural forms expression could be dangerous')
91 # Replace some C operators by their Python equivalents
92 plural
= plural
.replace('&&', ' and ')
93 plural
= plural
.replace('||', ' or ')
95 expr
= re
.compile(r
'\!([^=])')
96 plural
= expr
.sub(' not \\1', plural
)
98 # Regular expression and replacement function used to transform
99 # "a?b:c" to "test(a,b,c)".
100 expr
= re
.compile(r
'(.*?)\?(.*?):(.*)')
102 return "test(%s, %s, %s)" % (x
.group(1), x
.group(2),
103 expr
.sub(repl
, x
.group(3)))
105 # Code to transform the plural expression, taking care of parentheses
112 # Actually, we never reach this code, because unbalanced
113 # parentheses get caught in the security check at the
115 raise ValueError('unbalanced parenthesis in plural form')
116 s
= expr
.sub(repl
, stack
.pop())
117 stack
[-1] += '(%s)' % s
120 plural
= expr
.sub(repl
, stack
.pop())
122 return eval('lambda n: int(%s)' % plural
)
126 def _expand_lang(locale
):
127 from locale
import normalize
128 locale
= normalize(locale
)
129 COMPONENT_CODESET
= 1 << 0
130 COMPONENT_TERRITORY
= 1 << 1
131 COMPONENT_MODIFIER
= 1 << 2
132 # split up the locale into its base components
134 pos
= locale
.find('@')
136 modifier
= locale
[pos
:]
137 locale
= locale
[:pos
]
138 mask |
= COMPONENT_MODIFIER
141 pos
= locale
.find('.')
143 codeset
= locale
[pos
:]
144 locale
= locale
[:pos
]
145 mask |
= COMPONENT_CODESET
148 pos
= locale
.find('_')
150 territory
= locale
[pos
:]
151 locale
= locale
[:pos
]
152 mask |
= COMPONENT_TERRITORY
157 for i
in range(mask
+1):
158 if not (i
& ~mask
): # if all components for this combo exist ...
160 if i
& COMPONENT_TERRITORY
: val
+= territory
161 if i
& COMPONENT_CODESET
: val
+= codeset
162 if i
& COMPONENT_MODIFIER
: val
+= modifier
169 class NullTranslations
:
170 def __init__(self
, fp
=None):
173 self
._output
_charset
= None
174 self
._fallback
= None
178 def _parse(self
, fp
):
181 def add_fallback(self
, fallback
):
183 self
._fallback
.add_fallback(fallback
)
185 self
._fallback
= fallback
187 def gettext(self
, message
):
189 return self
._fallback
.gettext(message
)
192 def lgettext(self
, message
):
194 return self
._fallback
.lgettext(message
)
197 def ngettext(self
, msgid1
, msgid2
, n
):
199 return self
._fallback
.ngettext(msgid1
, msgid2
, n
)
205 def lngettext(self
, msgid1
, msgid2
, n
):
207 return self
._fallback
.lngettext(msgid1
, msgid2
, n
)
219 def output_charset(self
):
220 return self
._output
_charset
222 def set_output_charset(self
, charset
):
223 self
._output
_charset
= charset
225 def install(self
, names
=None):
227 builtins
.__dict
__['_'] = self
.gettext
228 if hasattr(names
, "__contains__"):
229 if "gettext" in names
:
230 builtins
.__dict
__['gettext'] = builtins
.__dict
__['_']
231 if "ngettext" in names
:
232 builtins
.__dict
__['ngettext'] = self
.ngettext
233 if "lgettext" in names
:
234 builtins
.__dict
__['lgettext'] = self
.lgettext
235 if "lngettext" in names
:
236 builtins
.__dict
__['lngettext'] = self
.lngettext
239 class GNUTranslations(NullTranslations
):
240 # Magic number of .mo files
241 LE_MAGIC
= 0x950412de
242 BE_MAGIC
= 0xde120495
244 def _parse(self
, fp
):
245 """Override this method to support alternative .mo formats."""
246 unpack
= struct
.unpack
247 filename
= getattr(fp
, 'name', '')
248 # Parse the .mo file header, which consists of 5 little endian 32
250 self
._catalog
= catalog
= {}
251 self
.plural
= lambda n
: int(n
!= 1) # germanic plural by default
254 # Are we big endian or little endian?
255 magic
= unpack('<I', buf
[:4])[0]
256 if magic
== self
.LE_MAGIC
:
257 version
, msgcount
, masteridx
, transidx
= unpack('<4I', buf
[4:20])
259 elif magic
== self
.BE_MAGIC
:
260 version
, msgcount
, masteridx
, transidx
= unpack('>4I', buf
[4:20])
263 raise IOError(0, 'Bad magic number', filename
)
264 # Now put all messages from the .mo file buffer into the catalog
266 for i
in range(0, msgcount
):
267 mlen
, moff
= unpack(ii
, buf
[masteridx
:masteridx
+8])
269 tlen
, toff
= unpack(ii
, buf
[transidx
:transidx
+8])
271 if mend
< buflen
and tend
< buflen
:
273 tmsg
= buf
[toff
:tend
]
275 raise IOError(0, 'File is corrupt', filename
)
276 # See if we're looking at GNU .mo conventions for metadata
278 # Catalog description
280 for b_item
in tmsg
.split('\n'.encode("ascii")):
281 item
= b_item
.decode().strip()
285 k
, v
= item
.split(':', 1)
286 k
= k
.strip().lower()
291 self
._info
[lastk
] += '\n' + item
292 if k
== 'content-type':
293 self
._charset
= v
.split('charset=')[1]
294 elif k
== 'plural-forms':
296 plural
= v
[1].split('plural=')[1]
297 self
.plural
= c2py(plural
)
298 # Note: we unconditionally convert both msgids and msgstrs to
299 # Unicode using the character encoding specified in the charset
300 # parameter of the Content-Type header. The gettext documentation
301 # strongly encourages msgids to be us-ascii, but some appliations
302 # require alternative encodings (e.g. Zope's ZCML and ZPT). For
303 # traditional gettext applications, the msgid conversion will
304 # cause no problems since us-ascii should always be a subset of
305 # the charset encoding. We may want to fall back to 8-bit msgids
306 # if the Unicode conversion fails.
307 charset
= self
._charset
or 'ascii'
310 msgid1
, msgid2
= msg
.split(b
'\x00')
311 tmsg
= tmsg
.split(b
'\x00')
312 msgid1
= str(msgid1
, charset
)
313 for i
, x
in enumerate(tmsg
):
314 catalog
[(msgid1
, i
)] = str(x
, charset
)
316 catalog
[str(msg
, charset
)] = str(tmsg
, charset
)
317 # advance to next entry in the seek tables
321 def lgettext(self
, message
):
323 tmsg
= self
._catalog
.get(message
, missing
)
326 return self
._fallback
.lgettext(message
)
328 if self
._output
_charset
:
329 return tmsg
.encode(self
._output
_charset
)
330 return tmsg
.encode(locale
.getpreferredencoding())
332 def lngettext(self
, msgid1
, msgid2
, n
):
334 tmsg
= self
._catalog
[(msgid1
, self
.plural(n
))]
335 if self
._output
_charset
:
336 return tmsg
.encode(self
._output
_charset
)
337 return tmsg
.encode(locale
.getpreferredencoding())
340 return self
._fallback
.lngettext(msgid1
, msgid2
, n
)
346 def gettext(self
, message
):
348 tmsg
= self
._catalog
.get(message
, missing
)
351 return self
._fallback
.gettext(message
)
355 def ngettext(self
, msgid1
, msgid2
, n
):
357 tmsg
= self
._catalog
[(msgid1
, self
.plural(n
))]
360 return self
._fallback
.ngettext(msgid1
, msgid2
, n
)
368 # Locate a .mo file using the gettext strategy
369 def find(domain
, localedir
=None, languages
=None, all
=False):
370 # Get some reasonable defaults for arguments that were not supplied
371 if localedir
is None:
372 localedir
= _default_localedir
373 if languages
is None:
375 for envar
in ('LANGUAGE', 'LC_ALL', 'LC_MESSAGES', 'LANG'):
376 val
= os
.environ
.get(envar
)
378 languages
= val
.split(':')
380 if 'C' not in languages
:
381 languages
.append('C')
382 # now normalize and expand the languages
384 for lang
in languages
:
385 for nelang
in _expand_lang(lang
):
386 if nelang
not in nelangs
:
387 nelangs
.append(nelang
)
396 mofile
= os
.path
.join(localedir
, lang
, 'LC_MESSAGES', '%s.mo' % domain
)
397 if os
.path
.exists(mofile
):
399 result
.append(mofile
)
406 # a mapping between absolute .mo file path and Translation object
409 def translation(domain
, localedir
=None, languages
=None,
410 class_
=None, fallback
=False, codeset
=None):
412 class_
= GNUTranslations
413 mofiles
= find(domain
, localedir
, languages
, all
=True)
416 return NullTranslations()
417 raise IOError(ENOENT
, 'No translation file found for domain', domain
)
418 # Avoid opening, reading, and parsing the .mo file after it's been done
421 for mofile
in mofiles
:
422 key
= os
.path
.abspath(mofile
)
423 t
= _translations
.get(key
)
425 with
open(mofile
, 'rb') as fp
:
426 t
= _translations
.setdefault(key
, class_(fp
))
427 # Copy the translation object to allow setting fallbacks and
428 # output charset. All other instance data is shared with the
432 t
.set_output_charset(codeset
)
436 result
.add_fallback(t
)
440 def install(domain
, localedir
=None, codeset
=None, names
=None):
441 t
= translation(domain
, localedir
, fallback
=True, codeset
=codeset
)
446 # a mapping b/w domains and locale directories
448 # a mapping b/w domains and codesets
450 # current global domain, `messages' used for compatibility w/ GNU gettext
451 _current_domain
= 'messages'
454 def textdomain(domain
=None):
455 global _current_domain
456 if domain
is not None:
457 _current_domain
= domain
458 return _current_domain
461 def bindtextdomain(domain
, localedir
=None):
463 if localedir
is not None:
464 _localedirs
[domain
] = localedir
465 return _localedirs
.get(domain
, _default_localedir
)
468 def bind_textdomain_codeset(domain
, codeset
=None):
469 global _localecodesets
470 if codeset
is not None:
471 _localecodesets
[domain
] = codeset
472 return _localecodesets
.get(domain
)
475 def dgettext(domain
, message
):
477 t
= translation(domain
, _localedirs
.get(domain
, None),
478 codeset
=_localecodesets
.get(domain
))
481 return t
.gettext(message
)
483 def ldgettext(domain
, message
):
485 t
= translation(domain
, _localedirs
.get(domain
, None),
486 codeset
=_localecodesets
.get(domain
))
489 return t
.lgettext(message
)
491 def dngettext(domain
, msgid1
, msgid2
, n
):
493 t
= translation(domain
, _localedirs
.get(domain
, None),
494 codeset
=_localecodesets
.get(domain
))
500 return t
.ngettext(msgid1
, msgid2
, n
)
502 def ldngettext(domain
, msgid1
, msgid2
, n
):
504 t
= translation(domain
, _localedirs
.get(domain
, None),
505 codeset
=_localecodesets
.get(domain
))
511 return t
.lngettext(msgid1
, msgid2
, n
)
513 def gettext(message
):
514 return dgettext(_current_domain
, message
)
516 def lgettext(message
):
517 return ldgettext(_current_domain
, message
)
519 def ngettext(msgid1
, msgid2
, n
):
520 return dngettext(_current_domain
, msgid1
, msgid2
, n
)
522 def lngettext(msgid1
, msgid2
, n
):
523 return ldngettext(_current_domain
, msgid1
, msgid2
, n
)
525 # dcgettext() has been deemed unnecessary and is not implemented.
527 # James Henstridge's Catalog constructor from GNOME gettext. Documented usage
531 # cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR)
533 # print _('Hello World')
535 # The resulting catalog object currently don't support access through a
536 # dictionary API, which was supported (but apparently unused) in GNOME
539 Catalog
= translation