config/Preprocessor.py

   1 """
   2 This is a very primitive line based preprocessor, for times when using
   3 a C preprocessor isn't an option.
   4 """
   5
   6 # This Source Code Form is subject to the terms of the Mozilla Public
   7 # License, v. 2.0. If a copy of the MPL was not distributed with this
   8 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   9
  10 import sys
  11 import os
  12 import os.path
  13 import re
  14 from optparse import OptionParser
  15 import errno
  16
  17 # hack around win32 mangling our line endings
  18 # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65443
  19 if sys.platform == "win32":
  20   import msvcrt
  21   msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
  22   os.linesep = '\n'
  23
  24 import Expression
  25
  26 __all__ = ['Preprocessor', 'preprocess']
  27
  28
  29 class Preprocessor:
  30   """
  31   Class for preprocessing text files.
  32   """
  33   class Error(RuntimeError):
  34     def __init__(self, cpp, MSG, context):
  35       self.file = cpp.context['FILE']
  36       self.line = cpp.context['LINE']
  37       self.key = MSG
  38       RuntimeError.__init__(self, (self.file, self.line, self.key, context))
  39   def __init__(self):
  40     self.context = Expression.Context()
  41     for k,v in {'FILE': '',
  42                 'LINE': 0,
  43                 'DIRECTORY': os.path.abspath('.')}.iteritems():
  44       self.context[k] = v
  45     self.actionLevel = 0
  46     self.disableLevel = 0
  47     # ifStates can be
  48     #  0: hadTrue
  49     #  1: wantsTrue
  50     #  2: #else found
  51     self.ifStates = []
  52     self.checkLineNumbers = False
  53     self.writtenLines = 0
  54     self.filters = []
  55     self.cmds = {}
  56     for cmd, level in {'define': 0,
  57                        'undef': 0,
  58                        'if': sys.maxint,
  59                        'ifdef': sys.maxint,
  60                        'ifndef': sys.maxint,
  61                        'else': 1,
  62                        'elif': 1,
  63                        'elifdef': 1,
  64                        'elifndef': 1,
  65                        'endif': sys.maxint,
  66                        'expand': 0,
  67                        'literal': 0,
  68                        'filter': 0,
  69                        'unfilter': 0,
  70                        'include': 0,
  71                        'includesubst': 0,
  72                        'error': 0}.iteritems():
  73       self.cmds[cmd] = (level, getattr(self, 'do_' + cmd))
  74     self.out = sys.stdout
  75     self.setMarker('#')
  76     self.LE = '\n'
  77     self.varsubst = re.compile('@(?P<VAR>\w+)@', re.U)
  78
  79   def warnUnused(self, file):
  80     if self.actionLevel == 0:
  81       sys.stderr.write('{0}: WARNING: no preprocessor directives found\n'.format(file))
  82     elif self.actionLevel == 1:
  83       sys.stderr.write('{0}: WARNING: no useful preprocessor directives found\n'.format(file))
  84     pass
  85
  86   def setLineEndings(self, aLE):
  87     """
  88     Set the line endings to be used for output.
  89     """
  90     self.LE = {'cr': '\x0D', 'lf': '\x0A', 'crlf': '\x0D\x0A'}[aLE]
  91
  92   def setMarker(self, aMarker):
  93     """
  94     Set the marker to be used for processing directives.
  95     Used for handling CSS files, with pp.setMarker('%'), for example.
  96     The given marker may be None, in which case no markers are processed.
  97     """
  98     self.marker = aMarker
  99     if aMarker:
 100       self.instruction = re.compile('{0}(?P<cmd>[a-z]+)(?:\s(?P<args>.*))?$'
 101                                     .format(aMarker),
 102                                     re.U)
 103       self.comment = re.compile(aMarker, re.U)
 104     else:
 105       class NoMatch(object):
 106         def match(self, *args):
 107           return False
 108       self.instruction = self.comment = NoMatch()
 109
 110   def clone(self):
 111     """
 112     Create a clone of the current processor, including line ending
 113     settings, marker, variable definitions, output stream.
 114     """
 115     rv = Preprocessor()
 116     rv.context.update(self.context)
 117     rv.setMarker(self.marker)
 118     rv.LE = self.LE
 119     rv.out = self.out
 120     return rv
 121
 122   def applyFilters(self, aLine):
 123     for f in self.filters:
 124       aLine = f[1](aLine)
 125     return aLine
 126
 127   def write(self, aLine):
 128     """
 129     Internal method for handling output.
 130     """
 131     if self.checkLineNumbers:
 132       self.writtenLines += 1
 133       ln = self.context['LINE']
 134       if self.writtenLines != ln:
 135         self.out.write('//@line {line} "{file}"{le}'.format(line=ln,
 136                                                             file=self.context['FILE'],
 137                                                             le=self.LE))
 138         self.writtenLines = ln
 139     filteredLine = self.applyFilters(aLine)
 140     if filteredLine != aLine:
 141       self.actionLevel = 2
 142     # ensure our line ending. Only need to handle \n, as we're reading
 143     # with universal line ending support, at least for files.
 144     filteredLine = re.sub('\n', self.LE, filteredLine)
 145     self.out.write(filteredLine)
 146
 147   def handleCommandLine(self, args, defaultToStdin = False):
 148     """
 149     Parse a commandline into this parser.
 150     Uses OptionParser internally, no args mean sys.argv[1:].
 151     """
 152     p = self.getCommandLineParser()
 153     (options, args) = p.parse_args(args=args)
 154     includes = options.I
 155     if options.output:
 156       dir = os.path.dirname(options.output)
 157       if dir and not os.path.exists(dir):
 158         try:
 159           os.makedirs(dir)
 160         except OSError as error:
 161           if error.errno != errno.EEXIST:
 162             raise
 163       self.out = open(options.output, 'w')
 164     if defaultToStdin and len(args) == 0:
 165       args = [sys.stdin]
 166     includes.extend(args)
 167     if includes:
 168       for f in includes:
 169         self.do_include(f, False)
 170       self.warnUnused(f)
 171     pass
 172
 173   def getCommandLineParser(self, unescapeDefines = False):
 174     escapedValue = re.compile('".*"$')
 175     numberValue = re.compile('\d+$')
 176     def handleE(option, opt, value, parser):
 177       for k,v in os.environ.iteritems():
 178         self.context[k] = v
 179     def handleD(option, opt, value, parser):
 180       vals = value.split('=', 1)
 181       if len(vals) == 1:
 182         vals.append(1)
 183       elif unescapeDefines and escapedValue.match(vals[1]):
 184         # strip escaped string values
 185         vals[1] = vals[1][1:-1]
 186       elif numberValue.match(vals[1]):
 187         vals[1] = int(vals[1])
 188       self.context[vals[0]] = vals[1]
 189     def handleU(option, opt, value, parser):
 190       del self.context[value]
 191     def handleF(option, opt, value, parser):
 192       self.do_filter(value)
 193     def handleLE(option, opt, value, parser):
 194       self.setLineEndings(value)
 195     def handleMarker(option, opt, value, parser):
 196       self.setMarker(value)
 197     p = OptionParser()
 198     p.add_option('-I', action='append', type="string", default = [],
 199                  metavar="FILENAME", help='Include file')
 200     p.add_option('-E', action='callback', callback=handleE,
 201                  help='Import the environment into the defined variables')
 202     p.add_option('-D', action='callback', callback=handleD, type="string",
 203                  metavar="VAR[=VAL]", help='Define a variable')
 204     p.add_option('-U', action='callback', callback=handleU, type="string",
 205                  metavar="VAR", help='Undefine a variable')
 206     p.add_option('-F', action='callback', callback=handleF, type="string",
 207                  metavar="FILTER", help='Enable the specified filter')
 208     p.add_option('-o', '--output', type="string", default=None,
 209                  metavar="FILENAME", help='Output to the specified file '+
 210                  'instead of stdout')
 211     p.add_option('--line-endings', action='callback', callback=handleLE,
 212                  type="string", metavar="[cr|lr|crlf]",
 213                  help='Use the specified line endings [Default: OS dependent]')
 214     p.add_option('--marker', action='callback', callback=handleMarker,
 215                  type="string",
 216                  help='Use the specified marker instead of #')
 217     return p
 218
 219   def handleLine(self, aLine):
 220     """
 221     Handle a single line of input (internal).
 222     """
 223     if self.actionLevel == 0 and self.comment.match(aLine):
 224       self.actionLevel = 1
 225     m = self.instruction.match(aLine)
 226     if m:
 227       args = None
 228       cmd = m.group('cmd')
 229       try:
 230         args = m.group('args')
 231       except IndexError:
 232         pass
 233       if cmd not in self.cmds:
 234         raise Preprocessor.Error(self, 'INVALID_CMD', aLine)
 235       level, cmd = self.cmds[cmd]
 236       if (level >= self.disableLevel):
 237         cmd(args)
 238       if cmd != 'literal':
 239         self.actionLevel = 2
 240     elif self.disableLevel == 0 and not self.comment.match(aLine):
 241       self.write(aLine)
 242     pass
 243
 244   # Instruction handlers
 245   # These are named do_'instruction name' and take one argument
 246
 247   # Variables
 248   def do_define(self, args):
 249     m = re.match('(?P<name>\w+)(?:\s(?P<value>.*))?', args, re.U)
 250     if not m:
 251       raise Preprocessor.Error(self, 'SYNTAX_DEF', args)
 252     val = 1
 253     if m.group('value'):
 254       val = self.applyFilters(m.group('value'))
 255       try:
 256         val = int(val)
 257       except:
 258         pass
 259     self.context[m.group('name')] = val
 260   def do_undef(self, args):
 261     m = re.match('(?P<name>\w+)$', args, re.U)
 262     if not m:
 263       raise Preprocessor.Error(self, 'SYNTAX_DEF', args)
 264     if args in self.context:
 265       del self.context[args]
 266   # Logic
 267   def ensure_not_else(self):
 268     if len(self.ifStates) == 0 or self.ifStates[-1] == 2:
 269       sys.stderr.write('WARNING: bad nesting of #else\n')
 270   def do_if(self, args, replace=False):
 271     if self.disableLevel and not replace:
 272       self.disableLevel += 1
 273       return
 274     val = None
 275     try:
 276       e = Expression.Expression(args)
 277       val = e.evaluate(self.context)
 278     except Exception:
 279       # XXX do real error reporting
 280       raise Preprocessor.Error(self, 'SYNTAX_ERR', args)
 281     if type(val) == str:
 282       # we're looking for a number value, strings are false
 283       val = False
 284     if not val:
 285       self.disableLevel = 1
 286     if replace:
 287       if val:
 288         self.disableLevel = 0
 289       self.ifStates[-1] = self.disableLevel
 290     else:
 291       self.ifStates.append(self.disableLevel)
 292     pass
 293   def do_ifdef(self, args, replace=False):
 294     if self.disableLevel and not replace:
 295       self.disableLevel += 1
 296       return
 297     if re.match('\W', args, re.U):
 298       raise Preprocessor.Error(self, 'INVALID_VAR', args)
 299     if args not in self.context:
 300       self.disableLevel = 1
 301     if replace:
 302       if args in self.context:
 303         self.disableLevel = 0
 304       self.ifStates[-1] = self.disableLevel
 305     else:
 306       self.ifStates.append(self.disableLevel)
 307     pass
 308   def do_ifndef(self, args, replace=False):
 309     if self.disableLevel and not replace:
 310       self.disableLevel += 1
 311       return
 312     if re.match('\W', args, re.U):
 313       raise Preprocessor.Error(self, 'INVALID_VAR', args)
 314     if args in self.context:
 315       self.disableLevel = 1
 316     if replace:
 317       if args not in self.context:
 318         self.disableLevel = 0
 319       self.ifStates[-1] = self.disableLevel
 320     else:
 321       self.ifStates.append(self.disableLevel)
 322     pass
 323   def do_else(self, args, ifState = 2):
 324     self.ensure_not_else()
 325     hadTrue = self.ifStates[-1] == 0
 326     self.ifStates[-1] = ifState # in-else
 327     if hadTrue:
 328       self.disableLevel = 1
 329       return
 330     self.disableLevel = 0
 331   def do_elif(self, args):
 332     if self.disableLevel == 1:
 333       if self.ifStates[-1] == 1:
 334         self.do_if(args, replace=True)
 335     else:
 336       self.do_else(None, self.ifStates[-1])
 337   def do_elifdef(self, args):
 338     if self.disableLevel == 1:
 339       if self.ifStates[-1] == 1:
 340         self.do_ifdef(args, replace=True)
 341     else:
 342       self.do_else(None, self.ifStates[-1])
 343   def do_elifndef(self, args):
 344     if self.disableLevel == 1:
 345       if self.ifStates[-1] == 1:
 346         self.do_ifndef(args, replace=True)
 347     else:
 348       self.do_else(None, self.ifStates[-1])
 349   def do_endif(self, args):
 350     if self.disableLevel > 0:
 351       self.disableLevel -= 1
 352     if self.disableLevel == 0:
 353       self.ifStates.pop()
 354   # output processing
 355   def do_expand(self, args):
 356     lst = re.split('__(\w+)__', args, re.U)
 357     do_replace = False
 358     def vsubst(v):
 359       if v in self.context:
 360         return str(self.context[v])
 361       return ''
 362     for i in range(1, len(lst), 2):
 363       lst[i] = vsubst(lst[i])
 364     lst.append('\n') # add back the newline
 365     self.write(reduce(lambda x, y: x+y, lst, ''))
 366   def do_literal(self, args):
 367     self.write(args + self.LE)
 368   def do_filter(self, args):
 369     filters = [f for f in args.split(' ') if hasattr(self, 'filter_' + f)]
 370     if len(filters) == 0:
 371       return
 372     current = dict(self.filters)
 373     for f in filters:
 374       current[f] = getattr(self, 'filter_' + f)
 375     filterNames = current.keys()
 376     filterNames.sort()
 377     self.filters = [(fn, current[fn]) for fn in filterNames]
 378     return
 379   def do_unfilter(self, args):
 380     filters = args.split(' ')
 381     current = dict(self.filters)
 382     for f in filters:
 383       if f in current:
 384         del current[f]
 385     filterNames = current.keys()
 386     filterNames.sort()
 387     self.filters = [(fn, current[fn]) for fn in filterNames]
 388     return
 389   # Filters
 390   #
 391   # emptyLines
 392   #   Strips blank lines from the output.
 393   def filter_emptyLines(self, aLine):
 394     if aLine == '\n':
 395       return ''
 396     return aLine
 397   # slashslash
 398   #   Strips everything after //
 399   def filter_slashslash(self, aLine):
 400     if (aLine.find('//') == -1):
 401       return aLine
 402     [aLine, rest] = aLine.split('//', 1)
 403     if rest:
 404       aLine += '\n'
 405     return aLine
 406   # spaces
 407   #   Collapses sequences of spaces into a single space
 408   def filter_spaces(self, aLine):
 409     return re.sub(' +', ' ', aLine).strip(' ')
 410   # substition
 411   #   helper to be used by both substition and attemptSubstitution
 412   def filter_substitution(self, aLine, fatal=True):
 413     def repl(matchobj):
 414       varname = matchobj.group('VAR')
 415       if varname in self.context:
 416         return str(self.context[varname])
 417       if fatal:
 418         raise Preprocessor.Error(self, 'UNDEFINED_VAR', varname)
 419       return matchobj.group(0)
 420     return self.varsubst.sub(repl, aLine)
 421   def filter_attemptSubstitution(self, aLine):
 422     return self.filter_substitution(aLine, fatal=False)
 423   # File ops
 424   def do_include(self, args, filters=True):
 425     """
 426     Preprocess a given file.
 427     args can either be a file name, or a file-like object.
 428     Files should be opened, and will be closed after processing.
 429     """
 430     isName = type(args) == str or type(args) == unicode
 431     oldWrittenLines = self.writtenLines
 432     oldCheckLineNumbers = self.checkLineNumbers
 433     self.checkLineNumbers = False
 434     if isName:
 435       try:
 436         args = str(args)
 437         if filters:
 438           args = self.applyFilters(args)
 439         if not os.path.isabs(args):
 440           args = os.path.join(self.context['DIRECTORY'], args)
 441         args = open(args, 'rU')
 442       except Preprocessor.Error:
 443         raise
 444       except:
 445         raise Preprocessor.Error(self, 'FILE_NOT_FOUND', str(args))
 446     self.checkLineNumbers = bool(re.search('\.(js|jsm|java)(?:\.in)?$', args.name))
 447     oldFile = self.context['FILE']
 448     oldLine = self.context['LINE']
 449     oldDir = self.context['DIRECTORY']
 450     if args.isatty():
 451       # we're stdin, use '-' and '' for file and dir
 452       self.context['FILE'] = '-'
 453       self.context['DIRECTORY'] = ''
 454     else:
 455       abspath = os.path.abspath(args.name)
 456       self.context['FILE'] = abspath
 457       self.context['DIRECTORY'] = os.path.dirname(abspath)
 458     self.context['LINE'] = 0
 459     self.writtenLines = 0
 460     for l in args:
 461       self.context['LINE'] += 1
 462       self.handleLine(l)
 463     args.close()
 464     self.context['FILE'] = oldFile
 465     self.checkLineNumbers = oldCheckLineNumbers
 466     self.writtenLines = oldWrittenLines
 467     self.context['LINE'] = oldLine
 468     self.context['DIRECTORY'] = oldDir
 469   def do_includesubst(self, args):
 470     args = self.filter_substitution(args)
 471     self.do_include(args)
 472   def do_error(self, args):
 473     raise Preprocessor.Error(self, 'Error: ', str(args))
 474
 475 def main():
 476   pp = Preprocessor()
 477   pp.handleCommandLine(None, True)
 478   return
 479
 480 def preprocess(includes=[sys.stdin], defines={},
 481                output = sys.stdout,
 482                line_endings='\n', marker='#'):
 483   pp = Preprocessor()
 484   pp.context.update(defines)
 485   pp.setLineEndings(line_endings)
 486   pp.setMarker(marker)
 487   pp.out = output
 488   for f in includes:
 489     pp.do_include(f, False)
 490
 491 if __name__ == "__main__":
 492   main()