config/Preprocessor.py

   1 """
   2 This is a very primitive line based preprocessor, for times when using
   3 a C preprocessor isn't an option.
   4 """
   5
   6 # This Source Code Form is subject to the terms of the Mozilla Public
   7 # License, v. 2.0. If a copy of the MPL was not distributed with this
   8 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   9
  10 import sys
  11 import os
  12 import os.path
  13 import re
  14 from optparse import OptionParser
  15
  16 # hack around win32 mangling our line endings
  17 # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65443
  18 if sys.platform == "win32":
  19   import msvcrt
  20   msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
  21   os.linesep = '\n'
  22
  23 import Expression
  24
  25 __all__ = ['Preprocessor', 'preprocess']
  26
  27
  28 class Preprocessor:
  29   """
  30   Class for preprocessing text files.
  31   """
  32   class Error(RuntimeError):
  33     def __init__(self, cpp, MSG, context):
  34       self.file = cpp.context['FILE']
  35       self.line = cpp.context['LINE']
  36       self.key = MSG
  37       RuntimeError.__init__(self, (self.file, self.line, self.key, context))
  38   def __init__(self):
  39     self.context = Expression.Context()
  40     for k,v in {'FILE': '',
  41                 'LINE': 0,
  42                 'DIRECTORY': os.path.abspath('.')}.iteritems():
  43       self.context[k] = v
  44     self.actionLevel = 0
  45     self.disableLevel = 0
  46     # ifStates can be
  47     #  0: hadTrue
  48     #  1: wantsTrue
  49     #  2: #else found
  50     self.ifStates = []
  51     self.checkLineNumbers = False
  52     self.writtenLines = 0
  53     self.filters = []
  54     self.cmds = {}
  55     for cmd, level in {'define': 0,
  56                        'undef': 0,
  57                        'if': sys.maxint,
  58                        'ifdef': sys.maxint,
  59                        'ifndef': sys.maxint,
  60                        'else': 1,
  61                        'elif': 1,
  62                        'elifdef': 1,
  63                        'elifndef': 1,
  64                        'endif': sys.maxint,
  65                        'expand': 0,
  66                        'literal': 0,
  67                        'filter': 0,
  68                        'unfilter': 0,
  69                        'include': 0,
  70                        'includesubst': 0,
  71                        'error': 0}.iteritems():
  72       self.cmds[cmd] = (level, getattr(self, 'do_' + cmd))
  73     self.out = sys.stdout
  74     self.setMarker('#')
  75     self.LE = '\n'
  76     self.varsubst = re.compile('@(?P<VAR>\w+)@', re.U)
  77
  78   def warnUnused(self, file):
  79     if self.actionLevel == 0:
  80       sys.stderr.write('%s: WARNING: no preprocessor directives found\n' % file)
  81     elif self.actionLevel == 1:
  82       sys.stderr.write('%s: WARNING: no useful preprocessor directives found\n' % file)
  83     pass
  84
  85   def setLineEndings(self, aLE):
  86     """
  87     Set the line endings to be used for output.
  88     """
  89     self.LE = {'cr': '\x0D', 'lf': '\x0A', 'crlf': '\x0D\x0A'}[aLE]
  90
  91   def setMarker(self, aMarker):
  92     """
  93     Set the marker to be used for processing directives.
  94     Used for handling CSS files, with pp.setMarker('%'), for example.
  95     The given marker may be None, in which case no markers are processed.
  96     """
  97     self.marker = aMarker
  98     if aMarker:
  99       self.instruction = re.compile('%s(?P<cmd>[a-z]+)(?:\s(?P<args>.*))?$'%aMarker, re.U)
 100       self.comment = re.compile(aMarker, re.U)
 101     else:
 102       class NoMatch(object):
 103         def match(self, *args):
 104           return False
 105       self.instruction = self.comment = NoMatch()
 106
 107   def clone(self):
 108     """
 109     Create a clone of the current processor, including line ending
 110     settings, marker, variable definitions, output stream.
 111     """
 112     rv = Preprocessor()
 113     rv.context.update(self.context)
 114     rv.setMarker(self.marker)
 115     rv.LE = self.LE
 116     rv.out = self.out
 117     return rv
 118
 119   def applyFilters(self, aLine):
 120     for f in self.filters:
 121       aLine = f[1](aLine)
 122     return aLine
 123
 124   def write(self, aLine):
 125     """
 126     Internal method for handling output.
 127     """
 128     if self.checkLineNumbers:
 129       self.writtenLines += 1
 130       ln = self.context['LINE']
 131       if self.writtenLines != ln:
 132         self.out.write('//@line %(line)d "%(file)s"%(le)s'%{'line': ln,
 133                                                             'file': self.context['FILE'],
 134                                                             'le': self.LE})
 135         self.writtenLines = ln
 136     filteredLine = self.applyFilters(aLine)
 137     if filteredLine != aLine:
 138       self.actionLevel = 2
 139     # ensure our line ending. Only need to handle \n, as we're reading
 140     # with universal line ending support, at least for files.
 141     filteredLine = re.sub('\n', self.LE, filteredLine)
 142     self.out.write(filteredLine)
 143
 144   def handleCommandLine(self, args, defaultToStdin = False):
 145     """
 146     Parse a commandline into this parser.
 147     Uses OptionParser internally, no args mean sys.argv[1:].
 148     """
 149     p = self.getCommandLineParser()
 150     (options, args) = p.parse_args(args=args)
 151     includes = options.I
 152     if defaultToStdin and len(args) == 0:
 153       args = [sys.stdin]
 154     includes.extend(args)
 155     if includes:
 156       for f in includes:
 157         self.do_include(f, False)
 158       self.warnUnused(f)
 159     pass
 160
 161   def getCommandLineParser(self, unescapeDefines = False):
 162     escapedValue = re.compile('".*"$')
 163     numberValue = re.compile('\d+$')
 164     def handleE(option, opt, value, parser):
 165       for k,v in os.environ.iteritems():
 166         self.context[k] = v
 167     def handleD(option, opt, value, parser):
 168       vals = value.split('=', 1)
 169       if len(vals) == 1:
 170         vals.append(1)
 171       elif unescapeDefines and escapedValue.match(vals[1]):
 172         # strip escaped string values
 173         vals[1] = vals[1][1:-1]
 174       elif numberValue.match(vals[1]):
 175         vals[1] = int(vals[1])
 176       self.context[vals[0]] = vals[1]
 177     def handleU(option, opt, value, parser):
 178       del self.context[value]
 179     def handleF(option, opt, value, parser):
 180       self.do_filter(value)
 181     def handleLE(option, opt, value, parser):
 182       self.setLineEndings(value)
 183     def handleMarker(option, opt, value, parser):
 184       self.setMarker(value)
 185     p = OptionParser()
 186     p.add_option('-I', action='append', type="string", default = [],
 187                  metavar="FILENAME", help='Include file')
 188     p.add_option('-E', action='callback', callback=handleE,
 189                  help='Import the environment into the defined variables')
 190     p.add_option('-D', action='callback', callback=handleD, type="string",
 191                  metavar="VAR[=VAL]", help='Define a variable')
 192     p.add_option('-U', action='callback', callback=handleU, type="string",
 193                  metavar="VAR", help='Undefine a variable')
 194     p.add_option('-F', action='callback', callback=handleF, type="string",
 195                  metavar="FILTER", help='Enable the specified filter')
 196     p.add_option('--line-endings', action='callback', callback=handleLE,
 197                  type="string", metavar="[cr|lr|crlf]",
 198                  help='Use the specified line endings [Default: OS dependent]')
 199     p.add_option('--marker', action='callback', callback=handleMarker,
 200                  type="string",
 201                  help='Use the specified marker instead of #')
 202     return p
 203
 204   def handleLine(self, aLine):
 205     """
 206     Handle a single line of input (internal).
 207     """
 208     if self.actionLevel == 0 and self.comment.match(aLine):
 209       self.actionLevel = 1
 210     m = self.instruction.match(aLine)
 211     if m:
 212       args = None
 213       cmd = m.group('cmd')
 214       try:
 215         args = m.group('args')
 216       except IndexError:
 217         pass
 218       if cmd not in self.cmds:
 219         raise Preprocessor.Error(self, 'INVALID_CMD', aLine)
 220       level, cmd = self.cmds[cmd]
 221       if (level >= self.disableLevel):
 222         cmd(args)
 223       if cmd != 'literal':
 224         self.actionLevel = 2
 225     elif self.disableLevel == 0 and not self.comment.match(aLine):
 226       self.write(aLine)
 227     pass
 228
 229   # Instruction handlers
 230   # These are named do_'instruction name' and take one argument
 231
 232   # Variables
 233   def do_define(self, args):
 234     m = re.match('(?P<name>\w+)(?:\s(?P<value>.*))?', args, re.U)
 235     if not m:
 236       raise Preprocessor.Error(self, 'SYNTAX_DEF', args)
 237     val = 1
 238     if m.group('value'):
 239       val = self.applyFilters(m.group('value'))
 240       try:
 241         val = int(val)
 242       except:
 243         pass
 244     self.context[m.group('name')] = val
 245   def do_undef(self, args):
 246     m = re.match('(?P<name>\w+)$', args, re.U)
 247     if not m:
 248       raise Preprocessor.Error(self, 'SYNTAX_DEF', args)
 249     if args in self.context:
 250       del self.context[args]
 251   # Logic
 252   def ensure_not_else(self):
 253     if len(self.ifStates) == 0 or self.ifStates[-1] == 2:
 254       sys.stderr.write('WARNING: bad nesting of #else\n')
 255   def do_if(self, args, replace=False):
 256     if self.disableLevel and not replace:
 257       self.disableLevel += 1
 258       return
 259     val = None
 260     try:
 261       e = Expression.Expression(args)
 262       val = e.evaluate(self.context)
 263     except Exception:
 264       # XXX do real error reporting
 265       raise Preprocessor.Error(self, 'SYNTAX_ERR', args)
 266     if type(val) == str:
 267       # we're looking for a number value, strings are false
 268       val = False
 269     if not val:
 270       self.disableLevel = 1
 271     if replace:
 272       if val:
 273         self.disableLevel = 0
 274       self.ifStates[-1] = self.disableLevel
 275     else:
 276       self.ifStates.append(self.disableLevel)
 277     pass
 278   def do_ifdef(self, args, replace=False):
 279     if self.disableLevel and not replace:
 280       self.disableLevel += 1
 281       return
 282     if re.match('\W', args, re.U):
 283       raise Preprocessor.Error(self, 'INVALID_VAR', args)
 284     if args not in self.context:
 285       self.disableLevel = 1
 286     if replace:
 287       if args in self.context:
 288         self.disableLevel = 0
 289       self.ifStates[-1] = self.disableLevel
 290     else:
 291       self.ifStates.append(self.disableLevel)
 292     pass
 293   def do_ifndef(self, args, replace=False):
 294     if self.disableLevel and not replace:
 295       self.disableLevel += 1
 296       return
 297     if re.match('\W', args, re.U):
 298       raise Preprocessor.Error(self, 'INVALID_VAR', args)
 299     if args in self.context:
 300       self.disableLevel = 1
 301     if replace:
 302       if args not in self.context:
 303         self.disableLevel = 0
 304       self.ifStates[-1] = self.disableLevel
 305     else:
 306       self.ifStates.append(self.disableLevel)
 307     pass
 308   def do_else(self, args, ifState = 2):
 309     self.ensure_not_else()
 310     hadTrue = self.ifStates[-1] == 0
 311     self.ifStates[-1] = ifState # in-else
 312     if hadTrue:
 313       self.disableLevel = 1
 314       return
 315     self.disableLevel = 0
 316   def do_elif(self, args):
 317     if self.disableLevel == 1:
 318       if self.ifStates[-1] == 1:
 319         self.do_if(args, replace=True)
 320     else:
 321       self.do_else(None, self.ifStates[-1])
 322   def do_elifdef(self, args):
 323     if self.disableLevel == 1:
 324       if self.ifStates[-1] == 1:
 325         self.do_ifdef(args, replace=True)
 326     else:
 327       self.do_else(None, self.ifStates[-1])
 328   def do_elifndef(self, args):
 329     if self.disableLevel == 1:
 330       if self.ifStates[-1] == 1:
 331         self.do_ifndef(args, replace=True)
 332     else:
 333       self.do_else(None, self.ifStates[-1])
 334   def do_endif(self, args):
 335     if self.disableLevel > 0:
 336       self.disableLevel -= 1
 337     if self.disableLevel == 0:
 338       self.ifStates.pop()
 339   # output processing
 340   def do_expand(self, args):
 341     lst = re.split('__(\w+)__', args, re.U)
 342     do_replace = False
 343     def vsubst(v):
 344       if v in self.context:
 345         return str(self.context[v])
 346       return ''
 347     for i in range(1, len(lst), 2):
 348       lst[i] = vsubst(lst[i])
 349     lst.append('\n') # add back the newline
 350     self.write(reduce(lambda x, y: x+y, lst, ''))
 351   def do_literal(self, args):
 352     self.write(args + self.LE)
 353   def do_filter(self, args):
 354     filters = [f for f in args.split(' ') if hasattr(self, 'filter_' + f)]
 355     if len(filters) == 0:
 356       return
 357     current = dict(self.filters)
 358     for f in filters:
 359       current[f] = getattr(self, 'filter_' + f)
 360     filterNames = current.keys()
 361     filterNames.sort()
 362     self.filters = [(fn, current[fn]) for fn in filterNames]
 363     return
 364   def do_unfilter(self, args):
 365     filters = args.split(' ')
 366     current = dict(self.filters)
 367     for f in filters:
 368       if f in current:
 369         del current[f]
 370     filterNames = current.keys()
 371     filterNames.sort()
 372     self.filters = [(fn, current[fn]) for fn in filterNames]
 373     return
 374   # Filters
 375   #
 376   # emptyLines
 377   #   Strips blank lines from the output.
 378   def filter_emptyLines(self, aLine):
 379     if aLine == '\n':
 380       return ''
 381     return aLine
 382   # slashslash
 383   #   Strips everything after //
 384   def filter_slashslash(self, aLine):
 385     [aLine, rest] = aLine.split('//', 1)
 386     if rest:
 387       aLine += '\n'
 388     return aLine
 389   # spaces
 390   #   Collapses sequences of spaces into a single space
 391   def filter_spaces(self, aLine):
 392     return re.sub(' +', ' ', aLine).strip(' ')
 393   # substition
 394   #   helper to be used by both substition and attemptSubstitution
 395   def filter_substitution(self, aLine, fatal=True):
 396     def repl(matchobj):
 397       varname = matchobj.group('VAR')
 398       if varname in self.context:
 399         return str(self.context[varname])
 400       if fatal:
 401         raise Preprocessor.Error(self, 'UNDEFINED_VAR', varname)
 402       return matchobj.group(0)
 403     return self.varsubst.sub(repl, aLine)
 404   def filter_attemptSubstitution(self, aLine):
 405     return self.filter_substitution(aLine, fatal=False)
 406   # File ops
 407   def do_include(self, args, filters=True):
 408     """
 409     Preprocess a given file.
 410     args can either be a file name, or a file-like object.
 411     Files should be opened, and will be closed after processing.
 412     """
 413     isName = type(args) == str or type(args) == unicode
 414     oldWrittenLines = self.writtenLines
 415     oldCheckLineNumbers = self.checkLineNumbers
 416     self.checkLineNumbers = False
 417     if isName:
 418       try:
 419         args = str(args)
 420         if filters:
 421           args = self.applyFilters(args)
 422         if not os.path.isabs(args):
 423           args = os.path.join(self.context['DIRECTORY'], args)
 424         args = open(args, 'rU')
 425       except Preprocessor.Error:
 426         raise
 427       except:
 428         raise Preprocessor.Error(self, 'FILE_NOT_FOUND', str(args))
 429     self.checkLineNumbers = bool(re.search('\.(js|java)(?:\.in)?$', args.name))
 430     oldFile = self.context['FILE']
 431     oldLine = self.context['LINE']
 432     oldDir = self.context['DIRECTORY']
 433     if args.isatty():
 434       # we're stdin, use '-' and '' for file and dir
 435       self.context['FILE'] = '-'
 436       self.context['DIRECTORY'] = ''
 437     else:
 438       abspath = os.path.abspath(args.name)
 439       self.context['FILE'] = abspath
 440       self.context['DIRECTORY'] = os.path.dirname(abspath)
 441     self.context['LINE'] = 0
 442     self.writtenLines = 0
 443     for l in args:
 444       self.context['LINE'] += 1
 445       self.handleLine(l)
 446     args.close()
 447     self.context['FILE'] = oldFile
 448     self.checkLineNumbers = oldCheckLineNumbers
 449     self.writtenLines = oldWrittenLines
 450     self.context['LINE'] = oldLine
 451     self.context['DIRECTORY'] = oldDir
 452   def do_includesubst(self, args):
 453     args = self.filter_substitution(args)
 454     self.do_include(args)
 455   def do_error(self, args):
 456     raise Preprocessor.Error(self, 'Error: ', str(args))
 457
 458 def main():
 459   pp = Preprocessor()
 460   pp.handleCommandLine(None, True)
 461   return
 462
 463 def preprocess(includes=[sys.stdin], defines={},
 464                output = sys.stdout,
 465                line_endings='\n', marker='#'):
 466   pp = Preprocessor()
 467   pp.context.update(defines)
 468   pp.setLineEndings(line_endings)
 469   pp.setMarker(marker)
 470   pp.out = output
 471   for f in includes:
 472     pp.do_include(f, False)
 473
 474 if __name__ == "__main__":
 475   main()