Misc/Vim/vim_syntax.py

   1 from __future__ import with_statement
   2
   3 import keyword
   4 import exceptions
   5 import __builtin__
   6 from string import Template
   7 from sys import subversion
   8
   9 comment_header = '''" Auto-generated Vim syntax file for Python (%s: r%s).
  10 "
  11 " To use: copy or symlink to ~/.vim/syntax/python.vim'''
  12
  13 statement_header = """
  14 if exists("b:current_syntax")
  15   finish
  16 endif"""
  17
  18 statement_footer = '''
  19 " Uncomment the 'minlines' statement line and comment out the 'maxlines'
  20 " statement line; changes behaviour to look at least 2000 lines previously for
  21 " syntax matches instead of at most 200 lines
  22 syn sync match pythonSync grouphere NONE "):$"
  23 syn sync maxlines=200
  24 "syn sync minlines=2000
  25
  26 let b:current_syntax = "python"'''
  27
  28 looping = ('for', 'while')
  29 conditionals = ('if', 'elif', 'else')
  30 boolean_ops = ('and', 'in', 'is', 'not', 'or')
  31 import_stmts = ('import', 'from')
  32 object_defs = ('def', 'class')
  33
  34 exception_names = sorted(exc for exc in dir(exceptions)
  35                                 if not exc.startswith('__'))
  36
  37 # Need to include functions that start with '__' (e.g., __import__), but
  38 # nothing that comes with modules (e.g., __name__), so just exclude anything in
  39 # the 'exceptions' module since we want to ignore exceptions *and* what any
  40 # module would have
  41 builtin_names = sorted(builtin for builtin in dir(__builtin__)
  42                             if builtin not in dir(exceptions))
  43
  44 escapes = (r'+\\[abfnrtv\'"\\]+', r'"\\\o\{1,3}"', r'"\\x\x\{2}"',
  45             r'"\(\\u\x\{4}\|\\U\x\{8}\)"', r'"\\$"')
  46
  47 todos = ("TODO", "FIXME", "XXX")
  48
  49 # XXX codify?
  50 numbers = (r'"\<0x\x\+[Ll]\=\>"', r'"\<\d\+[LljJ]\=\>"',
  51             '"\.\d\+\([eE][+-]\=\d\+\)\=[jJ]\=\>"',
  52             '"\<\d\+\.\([eE][+-]\=\d\+\)\=[jJ]\=\>"',
  53             '"\<\d\+\.\d\+\([eE][+-]\=\d\+\)\=[jJ]\=\>"')
  54
  55 contained = lambda x: "%s contained" % x
  56
  57 def str_regexes():
  58     """Generator to yield various combinations of strings regexes"""
  59     regex_template = Template('matchgroup=Normal ' +
  60                                 'start=+[uU]\=${raw}${sep}+ ' +
  61                                 'end=+${sep}+ ' +
  62                                 '${skip} ' +
  63                                 '${contains}')
  64     skip_regex = Template(r'skip=+\\\\\|\\${sep}+')
  65     for raw in ('', '[rR]'):
  66         for separator in ("'", '"', '"""', "'''"):
  67             if len(separator) == 1:
  68                 skip = skip_regex.substitute(sep=separator)
  69             else:
  70                 skip = ''
  71             contains = 'contains=pythonEscape' if not raw else ''
  72             yield regex_template.substitute(raw=raw, sep=separator, skip=skip,
  73                                             contains = contains)
  74
  75 space_errors = (r'excludenl "\S\s\+$"ms=s+1', r'" \+\t"', r'"\t\+ "')
  76
  77 statements = (
  78                 ('',
  79                     # XXX Might need to change pythonStatement since have
  80                     # specific Repeat, Conditional, Operator, etc. for 'while',
  81                     # etc.
  82                     [("Statement", "pythonStatement", "keyword",
  83                         (kw for kw in keyword.kwlist
  84                             if kw not in (looping + conditionals + boolean_ops +
  85                                         import_stmts + object_defs))
  86                       ),
  87                      ("Statement", "pythonStatement", "keyword",
  88                          (' '.join(object_defs) +
  89                              ' nextgroup=pythonFunction skipwhite')),
  90                      ("Function","pythonFunction", "match",
  91                          contained('"[a-zA-Z_][a-zA-Z0-9_]*"')),
  92                      ("Repeat", "pythonRepeat", "keyword", looping),
  93                      ("Conditional", "pythonConditional", "keyword",
  94                          conditionals),
  95                      ("Operator", "pythonOperator", "keyword", boolean_ops),
  96                      ("PreCondit", "pythonPreCondit", "keyword", import_stmts),
  97                      ("Comment", "pythonComment", "match",
  98                          '"#.*$" contains=pythonTodo'),
  99                      ("Todo", "pythonTodo", "keyword",
 100                          contained(' '.join(todos))),
 101                      ("String", "pythonString", "region", str_regexes()),
 102                      ("Special", "pythonEscape", "match",
 103                          (contained(esc) for esc in escapes
 104                              if not '$' in esc)),
 105                      ("Special", "pythonEscape", "match", r'"\\$"'),
 106                     ]
 107                 ),
 108                 ("python_highlight_numbers",
 109                     [("Number", "pythonNumber", "match", numbers)]
 110                 ),
 111                 ("python_highlight_builtins",
 112                     [("Function", "pythonBuiltin", "keyword", builtin_names)]
 113                 ),
 114                 ("python_highlight_exceptions",
 115                     [("Exception", "pythonException", "keyword",
 116                         exception_names)]
 117                 ),
 118                 ("python_highlight_space_errors",
 119                     [("Error", "pythonSpaceError", "match",
 120                         ("display " + err for err in space_errors))]
 121                 )
 122              )
 123
 124 def syn_prefix(type_, kind):
 125     return 'syn %s %s    ' % (type_, kind)
 126
 127 def fill_stmt(iterable, fill_len):
 128     """Yield a string that fills at most fill_len characters with strings
 129     returned by 'iterable' and separated by a space"""
 130     # Deal with trailing char to handle ' '.join() calculation
 131     fill_len += 1
 132     overflow = None
 133     it = iter(iterable)
 134     while True:
 135         buffer_ = []
 136         total_len = 0
 137         if overflow:
 138             buffer_.append(overflow)
 139             total_len += len(overflow) + 1
 140             overflow = None
 141         while total_len < fill_len:
 142             try:
 143                 new_item = it.next()
 144                 buffer_.append(new_item)
 145                 total_len += len(new_item) + 1
 146             except StopIteration:
 147                 if buffer_:
 148                     break
 149                 if overflow:
 150                     yield overflow
 151                 return
 152         if total_len > fill_len:
 153             overflow = buffer_.pop()
 154             total_len -= len(overflow) - 1
 155         ret = ' '.join(buffer_)
 156         assert len(ret) <= fill_len
 157         yield ret
 158
 159 FILL = 80
 160
 161 def main(file_path):
 162     with open(file_path, 'w') as FILE:
 163         # Comment for file
 164         print>>FILE, comment_header % subversion[1:]
 165         print>>FILE, ''
 166         # Statements at start of file
 167         print>>FILE, statement_header
 168         print>>FILE, ''
 169         # Generate case for python_highlight_all
 170         print>>FILE, 'if exists("python_highlight_all")'
 171         for statement_var, statement_parts in statements:
 172             if statement_var:
 173                 print>>FILE, '  let %s = 1' % statement_var
 174         else:
 175             print>>FILE, 'endif'
 176             print>>FILE, ''
 177         # Generate Python groups
 178         for statement_var, statement_parts in statements:
 179             if statement_var:
 180                 print>>FILE, 'if exists("%s")' % statement_var
 181                 indent = '  '
 182             else:
 183                 indent = ''
 184             for colour_group, group, type_, arguments in statement_parts:
 185                 if not isinstance(arguments, basestring):
 186                     prefix = syn_prefix(type_, group)
 187                     if type_ == 'keyword':
 188                         stmt_iter = fill_stmt(arguments,
 189                                             FILL - len(prefix) - len(indent))
 190                         try:
 191                             while True:
 192                                 print>>FILE, indent + prefix + stmt_iter.next()
 193                         except StopIteration:
 194                             print>>FILE, ''
 195                     else:
 196                         for argument in arguments:
 197                             print>>FILE, indent + prefix + argument
 198                         else:
 199                             print>>FILE, ''
 200
 201                 else:
 202                     print>>FILE, indent + syn_prefix(type_, group) + arguments
 203                     print>>FILE, ''
 204             else:
 205                 if statement_var:
 206                     print>>FILE, 'endif'
 207                     print>>FILE, ''
 208             print>>FILE, ''
 209         # Associating Python group with Vim colour group
 210         for statement_var, statement_parts in statements:
 211             if statement_var:
 212                 print>>FILE, '  if exists("%s")' % statement_var
 213                 indent = '    '
 214             else:
 215                 indent = '  '
 216             for colour_group, group, type_, arguments in statement_parts:
 217                 print>>FILE, (indent + "hi def link %s %s" %
 218                                 (group, colour_group))
 219             else:
 220                 if statement_var:
 221                     print>>FILE, '  endif'
 222                 print>>FILE, ''
 223         # Statements at the end of the file
 224         print>>FILE, statement_footer
 225
 226 if __name__ == '__main__':
 227     main("python.vim")