Exceptions raised during renaming in rotating file handlers are now passed to handleE...
[python.git] / Lib / gettext.py
bloba20c6f15224d565b964b3f0c451eb0c928f9d154
1 """Internationalization and localization support.
3 This module provides internationalization (I18N) and localization (L10N)
4 support for your Python programs by providing an interface to the GNU gettext
5 message catalog library.
7 I18N refers to the operation by which a program is made aware of multiple
8 languages. L10N refers to the adaptation of your program, once
9 internationalized, to the local language and cultural habits.
11 """
13 # This module represents the integration of work, contributions, feedback, and
14 # suggestions from the following people:
16 # Martin von Loewis, who wrote the initial implementation of the underlying
17 # C-based libintlmodule (later renamed _gettext), along with a skeletal
18 # gettext.py implementation.
20 # Peter Funk, who wrote fintl.py, a fairly complete wrapper around intlmodule,
21 # which also included a pure-Python implementation to read .mo files if
22 # intlmodule wasn't available.
24 # James Henstridge, who also wrote a gettext.py module, which has some
25 # interesting, but currently unsupported experimental features: the notion of
26 # a Catalog class and instances, and the ability to add to a catalog file via
27 # a Python API.
29 # Barry Warsaw integrated these modules, wrote the .install() API and code,
30 # and conformed all C and Python code to Python's coding standards.
32 # Francois Pinard and Marc-Andre Lemburg also contributed valuably to this
33 # module.
35 # J. David Ibanez implemented plural forms. Bruno Haible fixed some bugs.
37 # TODO:
38 # - Lazy loading of .mo files. Currently the entire catalog is loaded into
39 # memory, but that's probably bad for large translated programs. Instead,
40 # the lexical sort of original strings in GNU .mo files should be exploited
41 # to do binary searches and lazy initializations. Or you might want to use
42 # the undocumented double-hash algorithm for .mo files with hash tables, but
43 # you'll need to study the GNU gettext code to do this.
45 # - Support Solaris .mo file formats. Unfortunately, we've been unable to
46 # find this format documented anywhere.
49 import locale, copy, os, re, struct, sys
50 from errno import ENOENT
53 __all__ = ['NullTranslations', 'GNUTranslations', 'Catalog',
54 'find', 'translation', 'install', 'textdomain', 'bindtextdomain',
55 'dgettext', 'dngettext', 'gettext', 'ngettext',
58 _default_localedir = os.path.join(sys.prefix, 'share', 'locale')
61 def test(condition, true, false):
62 """
63 Implements the C expression:
65 condition ? true : false
67 Required to correctly interpret plural forms.
68 """
69 if condition:
70 return true
71 else:
72 return false
75 def c2py(plural):
76 """Gets a C expression as used in PO files for plural forms and returns a
77 Python lambda function that implements an equivalent expression.
78 """
79 # Security check, allow only the "n" identifier
80 try:
81 from cStringIO import StringIO
82 except ImportError:
83 from StringIO import StringIO
84 import token, tokenize
85 tokens = tokenize.generate_tokens(StringIO(plural).readline)
86 try:
87 danger = [x for x in tokens if x[0] == token.NAME and x[1] != 'n']
88 except tokenize.TokenError:
89 raise ValueError, \
90 'plural forms expression error, maybe unbalanced parenthesis'
91 else:
92 if danger:
93 raise ValueError, 'plural forms expression could be dangerous'
95 # Replace some C operators by their Python equivalents
96 plural = plural.replace('&&', ' and ')
97 plural = plural.replace('||', ' or ')
99 expr = re.compile(r'\!([^=])')
100 plural = expr.sub(' not \\1', plural)
102 # Regular expression and replacement function used to transform
103 # "a?b:c" to "test(a,b,c)".
104 expr = re.compile(r'(.*?)\?(.*?):(.*)')
105 def repl(x):
106 return "test(%s, %s, %s)" % (x.group(1), x.group(2),
107 expr.sub(repl, x.group(3)))
109 # Code to transform the plural expression, taking care of parentheses
110 stack = ['']
111 for c in plural:
112 if c == '(':
113 stack.append('')
114 elif c == ')':
115 if len(stack) == 1:
116 # Actually, we never reach this code, because unbalanced
117 # parentheses get caught in the security check at the
118 # beginning.
119 raise ValueError, 'unbalanced parenthesis in plural form'
120 s = expr.sub(repl, stack.pop())
121 stack[-1] += '(%s)' % s
122 else:
123 stack[-1] += c
124 plural = expr.sub(repl, stack.pop())
126 return eval('lambda n: int(%s)' % plural)
130 def _expand_lang(locale):
131 from locale import normalize
132 locale = normalize(locale)
133 COMPONENT_CODESET = 1 << 0
134 COMPONENT_TERRITORY = 1 << 1
135 COMPONENT_MODIFIER = 1 << 2
136 # split up the locale into its base components
137 mask = 0
138 pos = locale.find('@')
139 if pos >= 0:
140 modifier = locale[pos:]
141 locale = locale[:pos]
142 mask |= COMPONENT_MODIFIER
143 else:
144 modifier = ''
145 pos = locale.find('.')
146 if pos >= 0:
147 codeset = locale[pos:]
148 locale = locale[:pos]
149 mask |= COMPONENT_CODESET
150 else:
151 codeset = ''
152 pos = locale.find('_')
153 if pos >= 0:
154 territory = locale[pos:]
155 locale = locale[:pos]
156 mask |= COMPONENT_TERRITORY
157 else:
158 territory = ''
159 language = locale
160 ret = []
161 for i in range(mask+1):
162 if not (i & ~mask): # if all components for this combo exist ...
163 val = language
164 if i & COMPONENT_TERRITORY: val += territory
165 if i & COMPONENT_CODESET: val += codeset
166 if i & COMPONENT_MODIFIER: val += modifier
167 ret.append(val)
168 ret.reverse()
169 return ret
173 class NullTranslations:
174 def __init__(self, fp=None):
175 self._info = {}
176 self._charset = None
177 self._output_charset = None
178 self._fallback = None
179 if fp is not None:
180 self._parse(fp)
182 def _parse(self, fp):
183 pass
185 def add_fallback(self, fallback):
186 if self._fallback:
187 self._fallback.add_fallback(fallback)
188 else:
189 self._fallback = fallback
191 def gettext(self, message):
192 if self._fallback:
193 return self._fallback.gettext(message)
194 return message
196 def lgettext(self, message):
197 if self._fallback:
198 return self._fallback.lgettext(message)
199 return message
201 def ngettext(self, msgid1, msgid2, n):
202 if self._fallback:
203 return self._fallback.ngettext(msgid1, msgid2, n)
204 if n == 1:
205 return msgid1
206 else:
207 return msgid2
209 def lngettext(self, msgid1, msgid2, n):
210 if self._fallback:
211 return self._fallback.lngettext(msgid1, msgid2, n)
212 if n == 1:
213 return msgid1
214 else:
215 return msgid2
217 def ugettext(self, message):
218 if self._fallback:
219 return self._fallback.ugettext(message)
220 return unicode(message)
222 def ungettext(self, msgid1, msgid2, n):
223 if self._fallback:
224 return self._fallback.ungettext(msgid1, msgid2, n)
225 if n == 1:
226 return unicode(msgid1)
227 else:
228 return unicode(msgid2)
230 def info(self):
231 return self._info
233 def charset(self):
234 return self._charset
236 def output_charset(self):
237 return self._output_charset
239 def set_output_charset(self, charset):
240 self._output_charset = charset
242 def install(self, unicode=False):
243 import __builtin__
244 __builtin__.__dict__['_'] = unicode and self.ugettext or self.gettext
247 class GNUTranslations(NullTranslations):
248 # Magic number of .mo files
249 LE_MAGIC = 0x950412deL
250 BE_MAGIC = 0xde120495L
252 def _parse(self, fp):
253 """Override this method to support alternative .mo formats."""
254 unpack = struct.unpack
255 filename = getattr(fp, 'name', '')
256 # Parse the .mo file header, which consists of 5 little endian 32
257 # bit words.
258 self._catalog = catalog = {}
259 self.plural = lambda n: int(n != 1) # germanic plural by default
260 buf = fp.read()
261 buflen = len(buf)
262 # Are we big endian or little endian?
263 magic = unpack('<I', buf[:4])[0]
264 if magic == self.LE_MAGIC:
265 version, msgcount, masteridx, transidx = unpack('<4I', buf[4:20])
266 ii = '<II'
267 elif magic == self.BE_MAGIC:
268 version, msgcount, masteridx, transidx = unpack('>4I', buf[4:20])
269 ii = '>II'
270 else:
271 raise IOError(0, 'Bad magic number', filename)
272 # Now put all messages from the .mo file buffer into the catalog
273 # dictionary.
274 for i in xrange(0, msgcount):
275 mlen, moff = unpack(ii, buf[masteridx:masteridx+8])
276 mend = moff + mlen
277 tlen, toff = unpack(ii, buf[transidx:transidx+8])
278 tend = toff + tlen
279 if mend < buflen and tend < buflen:
280 msg = buf[moff:mend]
281 tmsg = buf[toff:tend]
282 else:
283 raise IOError(0, 'File is corrupt', filename)
284 # See if we're looking at GNU .mo conventions for metadata
285 if mlen == 0:
286 # Catalog description
287 lastk = k = None
288 for item in tmsg.splitlines():
289 item = item.strip()
290 if not item:
291 continue
292 if ':' in item:
293 k, v = item.split(':', 1)
294 k = k.strip().lower()
295 v = v.strip()
296 self._info[k] = v
297 lastk = k
298 elif lastk:
299 self._info[lastk] += '\n' + item
300 if k == 'content-type':
301 self._charset = v.split('charset=')[1]
302 elif k == 'plural-forms':
303 v = v.split(';')
304 plural = v[1].split('plural=')[1]
305 self.plural = c2py(plural)
306 # Note: we unconditionally convert both msgids and msgstrs to
307 # Unicode using the character encoding specified in the charset
308 # parameter of the Content-Type header. The gettext documentation
309 # strongly encourages msgids to be us-ascii, but some appliations
310 # require alternative encodings (e.g. Zope's ZCML and ZPT). For
311 # traditional gettext applications, the msgid conversion will
312 # cause no problems since us-ascii should always be a subset of
313 # the charset encoding. We may want to fall back to 8-bit msgids
314 # if the Unicode conversion fails.
315 if '\x00' in msg:
316 # Plural forms
317 msgid1, msgid2 = msg.split('\x00')
318 tmsg = tmsg.split('\x00')
319 if self._charset:
320 msgid1 = unicode(msgid1, self._charset)
321 tmsg = [unicode(x, self._charset) for x in tmsg]
322 for i in range(len(tmsg)):
323 catalog[(msgid1, i)] = tmsg[i]
324 else:
325 if self._charset:
326 msg = unicode(msg, self._charset)
327 tmsg = unicode(tmsg, self._charset)
328 catalog[msg] = tmsg
329 # advance to next entry in the seek tables
330 masteridx += 8
331 transidx += 8
333 def gettext(self, message):
334 missing = object()
335 tmsg = self._catalog.get(message, missing)
336 if tmsg is missing:
337 if self._fallback:
338 return self._fallback.gettext(message)
339 return message
340 # Encode the Unicode tmsg back to an 8-bit string, if possible
341 if self._output_charset:
342 return tmsg.encode(self._output_charset)
343 elif self._charset:
344 return tmsg.encode(self._charset)
345 return tmsg
347 def lgettext(self, message):
348 missing = object()
349 tmsg = self._catalog.get(message, missing)
350 if tmsg is missing:
351 if self._fallback:
352 return self._fallback.lgettext(message)
353 return message
354 if self._output_charset:
355 return tmsg.encode(self._output_charset)
356 return tmsg.encode(locale.getpreferredencoding())
358 def ngettext(self, msgid1, msgid2, n):
359 try:
360 tmsg = self._catalog[(msgid1, self.plural(n))]
361 if self._output_charset:
362 return tmsg.encode(self._output_charset)
363 elif self._charset:
364 return tmsg.encode(self._charset)
365 return tmsg
366 except KeyError:
367 if self._fallback:
368 return self._fallback.ngettext(msgid1, msgid2, n)
369 if n == 1:
370 return msgid1
371 else:
372 return msgid2
374 def lngettext(self, msgid1, msgid2, n):
375 try:
376 tmsg = self._catalog[(msgid1, self.plural(n))]
377 if self._output_charset:
378 return tmsg.encode(self._output_charset)
379 return tmsg.encode(locale.getpreferredencoding())
380 except KeyError:
381 if self._fallback:
382 return self._fallback.lngettext(msgid1, msgid2, n)
383 if n == 1:
384 return msgid1
385 else:
386 return msgid2
388 def ugettext(self, message):
389 missing = object()
390 tmsg = self._catalog.get(message, missing)
391 if tmsg is missing:
392 if self._fallback:
393 return self._fallback.ugettext(message)
394 return unicode(message)
395 return tmsg
397 def ungettext(self, msgid1, msgid2, n):
398 try:
399 tmsg = self._catalog[(msgid1, self.plural(n))]
400 except KeyError:
401 if self._fallback:
402 return self._fallback.ungettext(msgid1, msgid2, n)
403 if n == 1:
404 tmsg = unicode(msgid1)
405 else:
406 tmsg = unicode(msgid2)
407 return tmsg
410 # Locate a .mo file using the gettext strategy
411 def find(domain, localedir=None, languages=None, all=0):
412 # Get some reasonable defaults for arguments that were not supplied
413 if localedir is None:
414 localedir = _default_localedir
415 if languages is None:
416 languages = []
417 for envar in ('LANGUAGE', 'LC_ALL', 'LC_MESSAGES', 'LANG'):
418 val = os.environ.get(envar)
419 if val:
420 languages = val.split(':')
421 break
422 if 'C' not in languages:
423 languages.append('C')
424 # now normalize and expand the languages
425 nelangs = []
426 for lang in languages:
427 for nelang in _expand_lang(lang):
428 if nelang not in nelangs:
429 nelangs.append(nelang)
430 # select a language
431 if all:
432 result = []
433 else:
434 result = None
435 for lang in nelangs:
436 if lang == 'C':
437 break
438 mofile = os.path.join(localedir, lang, 'LC_MESSAGES', '%s.mo' % domain)
439 if os.path.exists(mofile):
440 if all:
441 result.append(mofile)
442 else:
443 return mofile
444 return result
448 # a mapping between absolute .mo file path and Translation object
449 _translations = {}
451 def translation(domain, localedir=None, languages=None,
452 class_=None, fallback=False, codeset=None):
453 if class_ is None:
454 class_ = GNUTranslations
455 mofiles = find(domain, localedir, languages, all=1)
456 if not mofiles:
457 if fallback:
458 return NullTranslations()
459 raise IOError(ENOENT, 'No translation file found for domain', domain)
460 # TBD: do we need to worry about the file pointer getting collected?
461 # Avoid opening, reading, and parsing the .mo file after it's been done
462 # once.
463 result = None
464 for mofile in mofiles:
465 key = os.path.abspath(mofile)
466 t = _translations.get(key)
467 if t is None:
468 t = _translations.setdefault(key, class_(open(mofile, 'rb')))
469 # Copy the translation object to allow setting fallbacks and
470 # output charset. All other instance data is shared with the
471 # cached object.
472 t = copy.copy(t)
473 if codeset:
474 t.set_output_charset(codeset)
475 if result is None:
476 result = t
477 else:
478 result.add_fallback(t)
479 return result
482 def install(domain, localedir=None, unicode=False, codeset=None):
483 t = translation(domain, localedir, fallback=True, codeset=codeset)
484 t.install(unicode)
488 # a mapping b/w domains and locale directories
489 _localedirs = {}
490 # a mapping b/w domains and codesets
491 _localecodesets = {}
492 # current global domain, `messages' used for compatibility w/ GNU gettext
493 _current_domain = 'messages'
496 def textdomain(domain=None):
497 global _current_domain
498 if domain is not None:
499 _current_domain = domain
500 return _current_domain
503 def bindtextdomain(domain, localedir=None):
504 global _localedirs
505 if localedir is not None:
506 _localedirs[domain] = localedir
507 return _localedirs.get(domain, _default_localedir)
510 def bind_textdomain_codeset(domain, codeset=None):
511 global _localecodesets
512 if codeset is not None:
513 _localecodesets[domain] = codeset
514 return _localecodesets.get(domain)
517 def dgettext(domain, message):
518 try:
519 t = translation(domain, _localedirs.get(domain, None),
520 codeset=_localecodesets.get(domain))
521 except IOError:
522 return message
523 return t.gettext(message)
525 def ldgettext(domain, message):
526 try:
527 t = translation(domain, _localedirs.get(domain, None),
528 codeset=_localecodesets.get(domain))
529 except IOError:
530 return message
531 return t.lgettext(message)
533 def dngettext(domain, msgid1, msgid2, n):
534 try:
535 t = translation(domain, _localedirs.get(domain, None),
536 codeset=_localecodesets.get(domain))
537 except IOError:
538 if n == 1:
539 return msgid1
540 else:
541 return msgid2
542 return t.ngettext(msgid1, msgid2, n)
544 def ldngettext(domain, msgid1, msgid2, n):
545 try:
546 t = translation(domain, _localedirs.get(domain, None),
547 codeset=_localecodesets.get(domain))
548 except IOError:
549 if n == 1:
550 return msgid1
551 else:
552 return msgid2
553 return t.lngettext(msgid1, msgid2, n)
555 def gettext(message):
556 return dgettext(_current_domain, message)
558 def lgettext(message):
559 return ldgettext(_current_domain, message)
561 def ngettext(msgid1, msgid2, n):
562 return dngettext(_current_domain, msgid1, msgid2, n)
564 def lngettext(msgid1, msgid2, n):
565 return ldngettext(_current_domain, msgid1, msgid2, n)
567 # dcgettext() has been deemed unnecessary and is not implemented.
569 # James Henstridge's Catalog constructor from GNOME gettext. Documented usage
570 # was:
572 # import gettext
573 # cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR)
574 # _ = cat.gettext
575 # print _('Hello World')
577 # The resulting catalog object currently don't support access through a
578 # dictionary API, which was supported (but apparently unused) in GNOME
579 # gettext.
581 Catalog = translation