1 """Internationalization and localization support.
3 This module provides internationalization (I18N) and localization (L10N)
4 support for your Python programs by providing an interface to the GNU gettext
5 message catalog library.
7 I18N refers to the operation by which a program is made aware of multiple
8 languages. L10N refers to the adaptation of your program, once
9 internationalized, to the local language and cultural habits.
13 # This module represents the integration of work, contributions, feedback, and
14 # suggestions from the following people:
16 # Martin von Loewis, who wrote the initial implementation of the underlying
17 # C-based libintlmodule (later renamed _gettext), along with a skeletal
18 # gettext.py implementation.
20 # Peter Funk, who wrote fintl.py, a fairly complete wrapper around intlmodule,
21 # which also included a pure-Python implementation to read .mo files if
22 # intlmodule wasn't available.
24 # James Henstridge, who also wrote a gettext.py module, which has some
25 # interesting, but currently unsupported experimental features: the notion of
26 # a Catalog class and instances, and the ability to add to a catalog file via
29 # Barry Warsaw integrated these modules, wrote the .install() API and code,
30 # and conformed all C and Python code to Python's coding standards.
32 # Francois Pinard and Marc-Andre Lemburg also contributed valuably to this
35 # J. David Ibanez implemented plural forms. Bruno Haible fixed some bugs.
38 # - Lazy loading of .mo files. Currently the entire catalog is loaded into
39 # memory, but that's probably bad for large translated programs. Instead,
40 # the lexical sort of original strings in GNU .mo files should be exploited
41 # to do binary searches and lazy initializations. Or you might want to use
42 # the undocumented double-hash algorithm for .mo files with hash tables, but
43 # you'll need to study the GNU gettext code to do this.
45 # - Support Solaris .mo file formats. Unfortunately, we've been unable to
46 # find this format documented anywhere.
49 import locale
, copy
, os
, re
, struct
, sys
50 from errno
import ENOENT
53 __all__
= ['NullTranslations', 'GNUTranslations', 'Catalog',
54 'find', 'translation', 'install', 'textdomain', 'bindtextdomain',
55 'dgettext', 'dngettext', 'gettext', 'ngettext',
58 _default_localedir
= os
.path
.join(sys
.prefix
, 'share', 'locale')
61 def test(condition
, true
, false
):
63 Implements the C expression:
65 condition ? true : false
67 Required to correctly interpret plural forms.
76 """Gets a C expression as used in PO files for plural forms and returns a
77 Python lambda function that implements an equivalent expression.
79 # Security check, allow only the "n" identifier
81 from cStringIO
import StringIO
83 from StringIO
import StringIO
84 import token
, tokenize
85 tokens
= tokenize
.generate_tokens(StringIO(plural
).readline
)
87 danger
= [x
for x
in tokens
if x
[0] == token
.NAME
and x
[1] != 'n']
88 except tokenize
.TokenError
:
90 'plural forms expression error, maybe unbalanced parenthesis'
93 raise ValueError, 'plural forms expression could be dangerous'
95 # Replace some C operators by their Python equivalents
96 plural
= plural
.replace('&&', ' and ')
97 plural
= plural
.replace('||', ' or ')
99 expr
= re
.compile(r
'\!([^=])')
100 plural
= expr
.sub(' not \\1', plural
)
102 # Regular expression and replacement function used to transform
103 # "a?b:c" to "test(a,b,c)".
104 expr
= re
.compile(r
'(.*?)\?(.*?):(.*)')
106 return "test(%s, %s, %s)" % (x
.group(1), x
.group(2),
107 expr
.sub(repl
, x
.group(3)))
109 # Code to transform the plural expression, taking care of parentheses
116 # Actually, we never reach this code, because unbalanced
117 # parentheses get caught in the security check at the
119 raise ValueError, 'unbalanced parenthesis in plural form'
120 s
= expr
.sub(repl
, stack
.pop())
121 stack
[-1] += '(%s)' % s
124 plural
= expr
.sub(repl
, stack
.pop())
126 return eval('lambda n: int(%s)' % plural
)
130 def _expand_lang(locale
):
131 from locale
import normalize
132 locale
= normalize(locale
)
133 COMPONENT_CODESET
= 1 << 0
134 COMPONENT_TERRITORY
= 1 << 1
135 COMPONENT_MODIFIER
= 1 << 2
136 # split up the locale into its base components
138 pos
= locale
.find('@')
140 modifier
= locale
[pos
:]
141 locale
= locale
[:pos
]
142 mask |
= COMPONENT_MODIFIER
145 pos
= locale
.find('.')
147 codeset
= locale
[pos
:]
148 locale
= locale
[:pos
]
149 mask |
= COMPONENT_CODESET
152 pos
= locale
.find('_')
154 territory
= locale
[pos
:]
155 locale
= locale
[:pos
]
156 mask |
= COMPONENT_TERRITORY
161 for i
in range(mask
+1):
162 if not (i
& ~mask
): # if all components for this combo exist ...
164 if i
& COMPONENT_TERRITORY
: val
+= territory
165 if i
& COMPONENT_CODESET
: val
+= codeset
166 if i
& COMPONENT_MODIFIER
: val
+= modifier
173 class NullTranslations
:
174 def __init__(self
, fp
=None):
177 self
._output
_charset
= None
178 self
._fallback
= None
182 def _parse(self
, fp
):
185 def add_fallback(self
, fallback
):
187 self
._fallback
.add_fallback(fallback
)
189 self
._fallback
= fallback
191 def gettext(self
, message
):
193 return self
._fallback
.gettext(message
)
196 def lgettext(self
, message
):
198 return self
._fallback
.lgettext(message
)
201 def ngettext(self
, msgid1
, msgid2
, n
):
203 return self
._fallback
.ngettext(msgid1
, msgid2
, n
)
209 def lngettext(self
, msgid1
, msgid2
, n
):
211 return self
._fallback
.lngettext(msgid1
, msgid2
, n
)
217 def ugettext(self
, message
):
219 return self
._fallback
.ugettext(message
)
220 return unicode(message
)
222 def ungettext(self
, msgid1
, msgid2
, n
):
224 return self
._fallback
.ungettext(msgid1
, msgid2
, n
)
226 return unicode(msgid1
)
228 return unicode(msgid2
)
236 def output_charset(self
):
237 return self
._output
_charset
239 def set_output_charset(self
, charset
):
240 self
._output
_charset
= charset
242 def install(self
, unicode=False):
244 __builtin__
.__dict
__['_'] = unicode and self
.ugettext
or self
.gettext
247 class GNUTranslations(NullTranslations
):
248 # Magic number of .mo files
249 LE_MAGIC
= 0x950412deL
250 BE_MAGIC
= 0xde120495L
252 def _parse(self
, fp
):
253 """Override this method to support alternative .mo formats."""
254 unpack
= struct
.unpack
255 filename
= getattr(fp
, 'name', '')
256 # Parse the .mo file header, which consists of 5 little endian 32
258 self
._catalog
= catalog
= {}
259 self
.plural
= lambda n
: int(n
!= 1) # germanic plural by default
262 # Are we big endian or little endian?
263 magic
= unpack('<I', buf
[:4])[0]
264 if magic
== self
.LE_MAGIC
:
265 version
, msgcount
, masteridx
, transidx
= unpack('<4I', buf
[4:20])
267 elif magic
== self
.BE_MAGIC
:
268 version
, msgcount
, masteridx
, transidx
= unpack('>4I', buf
[4:20])
271 raise IOError(0, 'Bad magic number', filename
)
272 # Now put all messages from the .mo file buffer into the catalog
274 for i
in xrange(0, msgcount
):
275 mlen
, moff
= unpack(ii
, buf
[masteridx
:masteridx
+8])
277 tlen
, toff
= unpack(ii
, buf
[transidx
:transidx
+8])
279 if mend
< buflen
and tend
< buflen
:
281 tmsg
= buf
[toff
:tend
]
283 raise IOError(0, 'File is corrupt', filename
)
284 # See if we're looking at GNU .mo conventions for metadata
286 # Catalog description
288 for item
in tmsg
.splitlines():
293 k
, v
= item
.split(':', 1)
294 k
= k
.strip().lower()
299 self
._info
[lastk
] += '\n' + item
300 if k
== 'content-type':
301 self
._charset
= v
.split('charset=')[1]
302 elif k
== 'plural-forms':
304 plural
= v
[1].split('plural=')[1]
305 self
.plural
= c2py(plural
)
306 # Note: we unconditionally convert both msgids and msgstrs to
307 # Unicode using the character encoding specified in the charset
308 # parameter of the Content-Type header. The gettext documentation
309 # strongly encourages msgids to be us-ascii, but some appliations
310 # require alternative encodings (e.g. Zope's ZCML and ZPT). For
311 # traditional gettext applications, the msgid conversion will
312 # cause no problems since us-ascii should always be a subset of
313 # the charset encoding. We may want to fall back to 8-bit msgids
314 # if the Unicode conversion fails.
317 msgid1
, msgid2
= msg
.split('\x00')
318 tmsg
= tmsg
.split('\x00')
320 msgid1
= unicode(msgid1
, self
._charset
)
321 tmsg
= [unicode(x
, self
._charset
) for x
in tmsg
]
322 for i
in range(len(tmsg
)):
323 catalog
[(msgid1
, i
)] = tmsg
[i
]
326 msg
= unicode(msg
, self
._charset
)
327 tmsg
= unicode(tmsg
, self
._charset
)
329 # advance to next entry in the seek tables
333 def gettext(self
, message
):
335 tmsg
= self
._catalog
.get(message
, missing
)
338 return self
._fallback
.gettext(message
)
340 # Encode the Unicode tmsg back to an 8-bit string, if possible
341 if self
._output
_charset
:
342 return tmsg
.encode(self
._output
_charset
)
344 return tmsg
.encode(self
._charset
)
347 def lgettext(self
, message
):
349 tmsg
= self
._catalog
.get(message
, missing
)
352 return self
._fallback
.lgettext(message
)
354 if self
._output
_charset
:
355 return tmsg
.encode(self
._output
_charset
)
356 return tmsg
.encode(locale
.getpreferredencoding())
358 def ngettext(self
, msgid1
, msgid2
, n
):
360 tmsg
= self
._catalog
[(msgid1
, self
.plural(n
))]
361 if self
._output
_charset
:
362 return tmsg
.encode(self
._output
_charset
)
364 return tmsg
.encode(self
._charset
)
368 return self
._fallback
.ngettext(msgid1
, msgid2
, n
)
374 def lngettext(self
, msgid1
, msgid2
, n
):
376 tmsg
= self
._catalog
[(msgid1
, self
.plural(n
))]
377 if self
._output
_charset
:
378 return tmsg
.encode(self
._output
_charset
)
379 return tmsg
.encode(locale
.getpreferredencoding())
382 return self
._fallback
.lngettext(msgid1
, msgid2
, n
)
388 def ugettext(self
, message
):
390 tmsg
= self
._catalog
.get(message
, missing
)
393 return self
._fallback
.ugettext(message
)
394 return unicode(message
)
397 def ungettext(self
, msgid1
, msgid2
, n
):
399 tmsg
= self
._catalog
[(msgid1
, self
.plural(n
))]
402 return self
._fallback
.ungettext(msgid1
, msgid2
, n
)
404 tmsg
= unicode(msgid1
)
406 tmsg
= unicode(msgid2
)
410 # Locate a .mo file using the gettext strategy
411 def find(domain
, localedir
=None, languages
=None, all
=0):
412 # Get some reasonable defaults for arguments that were not supplied
413 if localedir
is None:
414 localedir
= _default_localedir
415 if languages
is None:
417 for envar
in ('LANGUAGE', 'LC_ALL', 'LC_MESSAGES', 'LANG'):
418 val
= os
.environ
.get(envar
)
420 languages
= val
.split(':')
422 if 'C' not in languages
:
423 languages
.append('C')
424 # now normalize and expand the languages
426 for lang
in languages
:
427 for nelang
in _expand_lang(lang
):
428 if nelang
not in nelangs
:
429 nelangs
.append(nelang
)
438 mofile
= os
.path
.join(localedir
, lang
, 'LC_MESSAGES', '%s.mo' % domain
)
439 if os
.path
.exists(mofile
):
441 result
.append(mofile
)
448 # a mapping between absolute .mo file path and Translation object
451 def translation(domain
, localedir
=None, languages
=None,
452 class_
=None, fallback
=False, codeset
=None):
454 class_
= GNUTranslations
455 mofiles
= find(domain
, localedir
, languages
, all
=1)
458 return NullTranslations()
459 raise IOError(ENOENT
, 'No translation file found for domain', domain
)
460 # TBD: do we need to worry about the file pointer getting collected?
461 # Avoid opening, reading, and parsing the .mo file after it's been done
464 for mofile
in mofiles
:
465 key
= os
.path
.abspath(mofile
)
466 t
= _translations
.get(key
)
468 t
= _translations
.setdefault(key
, class_(open(mofile
, 'rb')))
469 # Copy the translation object to allow setting fallbacks and
470 # output charset. All other instance data is shared with the
474 t
.set_output_charset(codeset
)
478 result
.add_fallback(t
)
482 def install(domain
, localedir
=None, unicode=False, codeset
=None):
483 t
= translation(domain
, localedir
, fallback
=True, codeset
=codeset
)
488 # a mapping b/w domains and locale directories
490 # a mapping b/w domains and codesets
492 # current global domain, `messages' used for compatibility w/ GNU gettext
493 _current_domain
= 'messages'
496 def textdomain(domain
=None):
497 global _current_domain
498 if domain
is not None:
499 _current_domain
= domain
500 return _current_domain
503 def bindtextdomain(domain
, localedir
=None):
505 if localedir
is not None:
506 _localedirs
[domain
] = localedir
507 return _localedirs
.get(domain
, _default_localedir
)
510 def bind_textdomain_codeset(domain
, codeset
=None):
511 global _localecodesets
512 if codeset
is not None:
513 _localecodesets
[domain
] = codeset
514 return _localecodesets
.get(domain
)
517 def dgettext(domain
, message
):
519 t
= translation(domain
, _localedirs
.get(domain
, None),
520 codeset
=_localecodesets
.get(domain
))
523 return t
.gettext(message
)
525 def ldgettext(domain
, message
):
527 t
= translation(domain
, _localedirs
.get(domain
, None),
528 codeset
=_localecodesets
.get(domain
))
531 return t
.lgettext(message
)
533 def dngettext(domain
, msgid1
, msgid2
, n
):
535 t
= translation(domain
, _localedirs
.get(domain
, None),
536 codeset
=_localecodesets
.get(domain
))
542 return t
.ngettext(msgid1
, msgid2
, n
)
544 def ldngettext(domain
, msgid1
, msgid2
, n
):
546 t
= translation(domain
, _localedirs
.get(domain
, None),
547 codeset
=_localecodesets
.get(domain
))
553 return t
.lngettext(msgid1
, msgid2
, n
)
555 def gettext(message
):
556 return dgettext(_current_domain
, message
)
558 def lgettext(message
):
559 return ldgettext(_current_domain
, message
)
561 def ngettext(msgid1
, msgid2
, n
):
562 return dngettext(_current_domain
, msgid1
, msgid2
, n
)
564 def lngettext(msgid1
, msgid2
, n
):
565 return ldngettext(_current_domain
, msgid1
, msgid2
, n
)
567 # dcgettext() has been deemed unnecessary and is not implemented.
569 # James Henstridge's Catalog constructor from GNOME gettext. Documented usage
573 # cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR)
575 # print _('Hello World')
577 # The resulting catalog object currently don't support access through a
578 # dictionary API, which was supported (but apparently unused) in GNOME
581 Catalog
= translation