grantlee-extractor-pot-scripts/grantlee_strings_extractor.py

   1 #! /usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 ##
   5 # Copyright 2010,2011 Stephen Kelly <steveire@gmail.com>
   6 #
   7 # Redistribution and use in source and binary forms, with or without
   8 # modification, are permitted provided that the following conditions
   9 # are met:
  10 #
  11 # 1. Redistributions of source code must retain the above copyright
  12 #    notice, this list of conditions and the following disclaimer.
  13 # 2. Redistributions in binary form must reproduce the above copyright
  14 #    notice, this list of conditions and the following disclaimer in the
  15 #    documentation and/or other materials provided with the distribution.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18 # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19 # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20 # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22 # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27 ##
  28
  29 ## Parts of this file are reproduced from the Django framework. The Django licence appears below.
  30
  31 ##
  32 # Copyright (c) Django Software Foundation and individual contributors.
  33 # All rights reserved.
  34 #
  35 # Redistribution and use in source and binary forms, with or without modification,
  36 # are permitted provided that the following conditions are met:
  37 #
  38 #     1. Redistributions of source code must retain the above copyright notice,
  39 #        this list of conditions and the following disclaimer.
  40 #
  41 #     2. Redistributions in binary form must reproduce the above copyright
  42 #        notice, this list of conditions and the following disclaimer in the
  43 #        documentation and/or other materials provided with the distribution.
  44 #
  45 #     3. Neither the name of Django nor the names of its contributors may be used
  46 #        to endorse or promote products derived from this software without
  47 #        specific prior written permission.
  48 #
  49 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
  50 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  51 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  52 # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
  53 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  54 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  55 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
  56 # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  57 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  58 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  59 ##
  60
  61 import re
  62 import os.path
  63
  64 # == Introduction to the template syntax ==
  65 #
  66 # The template syntax looks like this:
  67 # (For more see here: http://grantlee.org/apidox/for_themers.html )
  68 #
  69 # This is plain text
  70 # This is text with a {{ value }} substitution
  71 # This is {% if condition_is_met %}a conditional{% endif %}
  72 # {# This is a comment #}
  73 # This is a {% comment %} multi-line
  74 # comment
  75 # {% endcomment %}
  76 #
  77 # That is, we have plain text.
  78 # We have value substitution with {{ }}
  79 # We have comments with {# #}
  80 # We have control tags with {% %}
  81 #
  82 # The first token inside {% %} syntax is called a tag name. Above, we have
  83 # an if tag and a comment tag.
  84 #
  85 # The 'value' in {{ value }} is called a filter expression. In the above case
  86 # the filter expression is a simple value which was inserted into the context.
  87 # In other cases it can be {{ value|upper }}, that is the value can be passed
  88 # through a filter called 'upper' with the '|', or filter expression can
  89 # be {{ value|join:"-" }}, that is it can be passed through the join filter
  90 # which takes an argument. In this case, the 'value' would actually be a list,
  91 # and the join filter would concatenate them with a dash. A filter can have
  92 # either no arguments, like upper, or it can take one argument, delimited by
  93 # a colon (';'). A filter expression can consist of a value followed by a
  94 # chain of filters, such as {{ value|join:"-"|upper }}. A filter expression
  95 # can appear one time inside {{ }} but may appear multiple times inside {% %}
  96 # For example {% cycle foo|upper bar|join:"-" bat %} contains 3 filter
  97 # expressions, 'foo|upper', 'bar|join:"-"' and 'bat'.
  98 #
  99 # Comments are ignored in the templates.
 100 #
 101 # == i18n in templates ==
 102 #
 103 # The purpose of this script is to extract translatable strings from templates
 104 # The aim is to allow template authors to write templates like this:
 105 #
 106 # This is a {{ _("translatable string") }} in the template.
 107 # This is a {% i18n "translatable string about %1" something %}
 108 # This is a {% i18nc "Some context information" "string about %1" something %}
 109 # This is a {% i18np "%1 string about %2" numthings something %}
 110 # This is a {% i18ncp "some context" "%1 string about %2" numthings something %}
 111 #
 112 # That is, simple translation with _(), and i18n* tags to allow for variable
 113 # substitution, context messages and plurals. Translatable strings may appear
 114 # in a filter expression, either as the value begin filtered, or as the argument
 115 # or both:
 116 #
 117 # {{ _("hello")|upper }}
 118 # {{ list|join:_("and") }}
 119 #
 120 # == How the strings are extracted ==
 121 #
 122 # The strings are extracted by parsing the template with regular expressions.
 123 # The tag_re regular expression breaks the template into a stream of tokens
 124 # containing plain text, {{ values }} and {% tags %}.
 125 # That work is done by the tokenize method with the create_token method.
 126 # Each token is then processed to extract the translatable strings from
 127 # the filter expressions.
 128
 129
 130 # The original context of much of this script is in the django template system:
 131 # http://code.djangoproject.com/browser/django/trunk/django/template/base.py
 132
 133
 134 TOKEN_TEXT = 0
 135 TOKEN_VAR = 1
 136 TOKEN_BLOCK = 2
 137 TOKEN_COMMENT = 3
 138
 139 # template syntax constants
 140 FILTER_SEPARATOR = '|'
 141 FILTER_ARGUMENT_SEPARATOR = ':'
 142 BLOCK_TAG_START = '{%'
 143 BLOCK_TAG_END = '%}'
 144 VARIABLE_TAG_START = '{{'
 145 VARIABLE_TAG_END = '}}'
 146 COMMENT_TAG_START = '{#'
 147 COMMENT_TAG_END = '#}'
 148
 149 # match a variable or block tag and capture the entire tag, including start/end delimiters
 150 tag_re = re.compile('(%s.*?%s|%s.*?%s)' % (re.escape(BLOCK_TAG_START), re.escape(BLOCK_TAG_END),
 151                                           re.escape(VARIABLE_TAG_START), re.escape(VARIABLE_TAG_END)))
 152
 153
 154 # Expression to match some_token and some_token="with spaces" (and similarly
 155 # for single-quoted strings).
 156 smart_split_re = re.compile(r"""
 157     ((?:
 158         [^\s'"]*
 159         (?:
 160             (?:"(?:[^"\\]|\\.)*" | '(?:[^'\\]|\\.)*')
 161             [^\s'"]*
 162         )+
 163     ) | \S+)
 164 """, re.VERBOSE)
 165
 166 def smart_split(text):
 167     r"""
 168     Generator that splits a string by spaces, leaving quoted phrases together.
 169     Supports both single and double quotes, and supports escaping quotes with
 170     backslashes. In the output, strings will keep their initial and trailing
 171     quote marks and escaped quotes will remain escaped (the results can then
 172     be further processed with unescape_string_literal()).
 173
 174     >>> list(smart_split(r'This is "a person\'s" test.'))
 175     [u'This', u'is', u'"a person\\\'s"', u'test.']
 176     >>> list(smart_split(r"Another 'person\'s' test."))
 177     [u'Another', u"'person\\'s'", u'test.']
 178     >>> list(smart_split(r'A "\"funky\" style" test.'))
 179     [u'A', u'"\\"funky\\" style"', u'test.']
 180     """
 181     for bit in smart_split_re.finditer(text):
 182         yield bit.group(0)
 183
 184
 185 # This only matches constant *strings* (things in quotes or marked for
 186 # translation).
 187
 188 constant_string = r"(?:%(strdq)s|%(strsq)s)" % {
 189     'strdq': r'"[^"\\]*(?:\\.[^"\\]*)*"', # double-quoted string
 190     'strsq': r"'[^'\\]*(?:\\.[^'\\]*)*'", # single-quoted string
 191     }
 192
 193 filter_raw_string = r"""^%(i18n_open)s(?P<l10nable>%(constant_string)s)%(i18n_close)s""" % {
 194     'constant_string': constant_string,
 195     'i18n_open' : re.escape("_("),
 196     'i18n_close' : re.escape(")"),
 197   }
 198
 199 filter_re = re.compile(filter_raw_string, re.UNICODE|re.VERBOSE)
 200
 201 class TemplateSyntaxError(Exception):
 202     pass
 203
 204 class TranslatableString:
 205     _string = ''
 206     context = ''
 207     plural = ''
 208
 209     def __repr__(self):
 210         return "String('%s', '%s', '%s')" % (self._string, self.context, self.plural)
 211
 212 class Token(object):
 213     def __init__(self, token_type, contents):
 214         # token_type must be TOKEN_TEXT, TOKEN_VAR, TOKEN_BLOCK or TOKEN_COMMENT.
 215         self.token_type, self.contents = token_type, contents
 216
 217     def __str__(self):
 218         return '<%s token: "%s...">' % \
 219             ({TOKEN_TEXT: 'Text', TOKEN_VAR: 'Var', TOKEN_BLOCK: 'Block', TOKEN_COMMENT: 'Comment'}[self.token_type],
 220             self.contents[:20].replace('\n', ''))
 221
 222 def create_token(token_string, in_tag):
 223     """
 224     Convert the given token string into a new Token object and return it.
 225     If in_tag is True, we are processing something that matched a tag,
 226     otherwise it should be treated as a literal string.
 227     """
 228     if in_tag:
 229         if token_string.startswith(VARIABLE_TAG_START):
 230             token = Token(TOKEN_VAR, token_string[len(VARIABLE_TAG_START):-len(VARIABLE_TAG_END)].strip())
 231         elif token_string.startswith(BLOCK_TAG_START):
 232             token = Token(TOKEN_BLOCK, token_string[len(BLOCK_TAG_START):-len(BLOCK_TAG_END)].strip())
 233         elif token_string.startswith(COMMENT_TAG_START):
 234             token = Token(TOKEN_COMMENT, '')
 235     else:
 236         token = Token(TOKEN_TEXT, token_string)
 237     return token
 238
 239 def tokenize(template_string):
 240
 241     in_tag = False
 242     result = []
 243     for bit in tag_re.split(template_string):
 244         if bit:
 245             result.append(create_token(bit, in_tag))
 246         in_tag = not in_tag
 247     return result
 248
 249 class TranslationOutputter:
 250     translatable_strings = []
 251
 252     def get_translatable_filter_args(self, token):
 253         """
 254         Find the filter expressions in token and extract the strings in it.
 255         """
 256         matches = filter_re.finditer(token)
 257         upto = 0
 258         var_obj = False
 259         for match in matches:
 260             l10nable = match.group("l10nable")
 261
 262             if l10nable:
 263                 # Make sure it's a quoted string
 264                 if l10nable.startswith('"') and l10nable.endswith('"') \
 265                         or l10nable.startswith("'") and l10nable.endswith("'"):
 266                     ts = TranslatableString()
 267                     ts._string = l10nable[1:-1]
 268                     self.translatable_strings.append(ts)
 269
 270     def get_contextual_strings(self, token):
 271         split = []
 272         _bits = smart_split(token.contents)
 273         _bit = _bits.next()
 274         if _bit =="i18n" or _bit == "i18n_var":
 275             # {% i18n "A one %1, a two %2, a three %3" var1 var2 var3 %}
 276             # {% i18n_var "A one %1, a two %2, a three %3" var1 var2 var3 as result %}
 277             _bit = _bits.next()
 278             if not _bit.startswith("'") and not _bit.startswith('"'):
 279                 return
 280
 281             sentinal = _bit[0]
 282             if not _bit.endswith(sentinal):
 283                 return
 284
 285             translatable_string = TranslatableString()
 286             translatable_string._string = _bit[1:-1]
 287             self.translatable_strings.append(translatable_string)
 288         elif _bit =="i18nc" or _bit == "i18nc_var":
 289             # {% i18nc "An email send operation failed." "%1 Failed!" var1 %}
 290             # {% i18nc_var "An email send operation failed." "%1 Failed!" var1 as result %}
 291             _bit = _bits.next()
 292             if not _bit.startswith("'") and not _bit.startswith('"'):
 293                 return
 294
 295             sentinal = _bit[0]
 296             if not _bit.endswith(sentinal):
 297                 return
 298
 299             translatable_string = TranslatableString()
 300             translatable_string.context = _bit[1:-1]
 301             _bit = _bits.next()
 302             translatable_string._string = _bit[1:-1]
 303             self.translatable_strings.append(translatable_string)
 304         elif _bit =="i18np" or _bit =="i18np_var":
 305             # {% i18np "An email send operation failed." "%1 email send operations failed. Error : % 2." count count errorMsg %}
 306             # {% i18np_var "An email send operation failed." "%1 email send operations failed. Error : % 2." count count errorMsg as result %}
 307             _bit = _bits.next()
 308             if not _bit.startswith("'") and not _bit.startswith('"'):
 309                 return
 310
 311             sentinal = _bit[0]
 312             if not _bit.endswith(sentinal):
 313                 return
 314
 315             translatable_string = TranslatableString()
 316             translatable_string._string = _bit[1:-1]
 317             _bit = _bits.next()
 318             translatable_string.plural = _bit[1:-1]
 319             self.translatable_strings.append(translatable_string)
 320         elif _bit =="i18ncp" or _bit =="i18ncp_var":
 321             # {% i18np "The user tried to send an email, but that failed." "An email send operation failed." "%1 email send operation failed." count count %}
 322             # {% i18np_var "The user tried to send an email, but that failed." "An email send operation failed." "%1 email send operation failed." count count as result %}
 323
 324             _bit = _bits.next()
 325             if not _bit.startswith("'") and not _bit.startswith('"'):
 326                 return
 327
 328             sentinal = _bit[0]
 329             if not _bit.endswith(sentinal):
 330                 return
 331
 332             translatable_string = TranslatableString()
 333             translatable_string.context = _bit[1:-1]
 334             _bit = _bits.next()
 335             translatable_string._string = _bit[1:-1]
 336             _bit = _bits.next()
 337             translatable_string.plural = _bit[1:-1]
 338             self.translatable_strings.append(translatable_string)
 339         else:
 340           return
 341
 342         for _bit in _bits:
 343
 344             if (_bit == "as"):
 345                 return
 346             self.get_translatable_filter_args(_bit)
 347
 348     def get_plain_strings(self, token):
 349         split = []
 350         bits = iter(smart_split(token.contents))
 351         for bit in bits:
 352             self.get_translatable_filter_args(bit)
 353
 354     def translate(self, template_file, outputfile):
 355         template_string = template_file.read()
 356         self.translatable_strings = []
 357         for token in tokenize(template_string):
 358             if token.token_type == TOKEN_VAR or token.token_type == TOKEN_BLOCK:
 359                 self.get_plain_strings(token)
 360             if token.token_type == TOKEN_BLOCK:
 361                 self.get_contextual_strings(token)
 362         self.createOutput(os.path.relpath(template_file.name), self.translatable_strings, outputfile)
 363
 364     def createOutput(self, template_filename, translatable_strings, outputfile):
 365       pass