config/Preprocessor.py

   1 """
   2 This is a very primitive line based preprocessor, for times when using
   3 a C preprocessor isn't an option.
   4 """
   5
   6 # ***** BEGIN LICENSE BLOCK *****
   7 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   8 #
   9 # The contents of this file are subject to the Mozilla Public License Version
  10 # 1.1 (the "License"); you may not use this file except in compliance with
  11 # the License. You may obtain a copy of the License at
  12 # http://www.mozilla.org/MPL/
  13 #
  14 # Software distributed under the License is distributed on an "AS IS" basis,
  15 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  16 # for the specific language governing rights and limitations under the
  17 # License.
  18 #
  19 # The Original Code is Mozilla build system.
  20 #
  21 # The Initial Developer of the Original Code is
  22 # Mozilla Foundation.
  23 # Portions created by the Initial Developer are Copyright (C) 2007
  24 # the Initial Developer. All Rights Reserved.
  25 #
  26 # Contributor(s):
  27 #  Axel Hecht <axel@pike.org>
  28 #
  29 # Alternatively, the contents of this file may be used under the terms of
  30 # either the GNU General Public License Version 2 or later (the "GPL"), or
  31 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  32 # in which case the provisions of the GPL or the LGPL are applicable instead
  33 # of those above. If you wish to allow use of your version of this file only
  34 # under the terms of either the GPL or the LGPL, and not to allow others to
  35 # use your version of this file under the terms of the MPL, indicate your
  36 # decision by deleting the provisions above and replace them with the notice
  37 # and other provisions required by the GPL or the LGPL. If you do not delete
  38 # the provisions above, a recipient may use your version of this file under
  39 # the terms of any one of the MPL, the GPL or the LGPL.
  40 #
  41 # ***** END LICENSE BLOCK *****
  42
  43 import sys
  44 import os
  45 import os.path
  46 import re
  47 from optparse import OptionParser
  48
  49 # hack around win32 mangling our line endings
  50 # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65443
  51 if sys.platform == "win32":
  52   import msvcrt
  53   msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
  54   os.linesep = '\n'
  55
  56 import Expression
  57
  58 __all__ = ['Preprocessor', 'preprocess']
  59
  60
  61 class Preprocessor:
  62   """
  63   Class for preprocessing text files.
  64   """
  65   class Error(RuntimeError):
  66     def __init__(self, cpp, MSG, context):
  67       self.file = cpp.context['FILE']
  68       self.line = cpp.context['LINE']
  69       self.key = MSG
  70       RuntimeError.__init__(self, (self.file, self.line, self.key, context))
  71   def __init__(self):
  72     self.context = Expression.Context()
  73     for k,v in {'FILE': '',
  74                 'LINE': 0,
  75                 'DIRECTORY': os.path.abspath('.')}.iteritems():
  76       self.context[k] = v
  77     self.disableLevel = 0
  78     # ifStates can be
  79     #  0: hadTrue
  80     #  1: wantsTrue
  81     #  2: #else found
  82     self.ifStates = []
  83     self.checkLineNumbers = False
  84     self.writtenLines = 0
  85     self.filters = []
  86     self.cmds = {}
  87     for cmd, level in {'define': 0,
  88                        'undef': 0,
  89                        'if': sys.maxint,
  90                        'ifdef': sys.maxint,
  91                        'ifndef': sys.maxint,
  92                        'else': 1,
  93                        'elif': 1,
  94                        'elifdef': 1,
  95                        'elifndef': 1,
  96                        'endif': sys.maxint,
  97                        'expand': 0,
  98                        'literal': 0,
  99                        'filter': 0,
 100                        'unfilter': 0,
 101                        'include': 0,
 102                        'includesubst': 0,
 103                        'error': 0}.iteritems():
 104       self.cmds[cmd] = (level, getattr(self, 'do_' + cmd))
 105     self.out = sys.stdout
 106     self.setMarker('#')
 107     self.LE = '\n'
 108     self.varsubst = re.compile('@(?P<VAR>\w+)@', re.U)
 109
 110   def setLineEndings(self, aLE):
 111     """
 112     Set the line endings to be used for output.
 113     """
 114     self.LE = {'cr': '\x0D', 'lf': '\x0A', 'crlf': '\x0D\x0A'}[aLE]
 115
 116   def setMarker(self, aMarker):
 117     """
 118     Set the marker to be used for processing directives.
 119     Used for handling CSS files, with pp.setMarker('%'), for example.
 120     """
 121     self.marker = aMarker
 122     self.instruction = re.compile('%s(?P<cmd>[a-z]+)(?:\s(?P<args>.*))?$'%aMarker, re.U)
 123     self.comment = re.compile(aMarker, re.U)
 124
 125   def clone(self):
 126     """
 127     Create a clone of the current processor, including line ending
 128     settings, marker, variable definitions, output stream.
 129     """
 130     rv = Preprocessor()
 131     rv.context.update(self.context)
 132     rv.setMarker(self.marker)
 133     rv.LE = self.LE
 134     rv.out = self.out
 135     return rv
 136
 137   def write(self, aLine):
 138     """
 139     Internal method for handling output.
 140     """
 141     if self.checkLineNumbers:
 142       self.writtenLines += 1
 143       ln = self.context['LINE']
 144       if self.writtenLines != ln:
 145         self.out.write('//@line %(line)d "%(file)s"%(le)s'%{'line': ln,
 146                                                             'file': self.context['FILE'],
 147                                                             'le': self.LE})
 148         self.writtenLines = ln
 149     for f in self.filters:
 150       aLine = f[1](aLine)
 151     # ensure our line ending. Only need to handle \n, as we're reading
 152     # with universal line ending support, at least for files.
 153     aLine = re.sub('\n', self.LE, aLine)
 154     self.out.write(aLine)
 155
 156   def handleCommandLine(self, args, defaultToStdin = False):
 157     """
 158     Parse a commandline into this parser.
 159     Uses OptionParser internally, no args mean sys.argv[1:].
 160     """
 161     p = self.getCommandLineParser()
 162     (options, args) = p.parse_args(args=args)
 163     includes = options.I
 164     if defaultToStdin and len(args) == 0:
 165       args = [sys.stdin]
 166     includes.extend(args)
 167     for f in includes:
 168       self.do_include(f)
 169     pass
 170
 171   def getCommandLineParser(self, unescapeDefines = False):
 172     escapedValue = re.compile('".*"$')
 173     numberValue = re.compile('\d+$')
 174     def handleE(option, opt, value, parser):
 175       for k,v in os.environ.iteritems():
 176         self.context[k] = v
 177     def handleD(option, opt, value, parser):
 178       vals = value.split('=', 1)
 179       if len(vals) == 1:
 180         vals.append(1)
 181       elif unescapeDefines and escapedValue.match(vals[1]):
 182         # strip escaped string values
 183         vals[1] = vals[1][1:-1]
 184       elif numberValue.match(vals[1]):
 185         if vals[1][0] == '0':
 186           vals[1] = int(vals[1], 8)
 187         else:
 188           vals[1] = int(vals[1])
 189       self.context[vals[0]] = vals[1]
 190     def handleU(option, opt, value, parser):
 191       del self.context[value]
 192     def handleF(option, opt, value, parser):
 193       self.do_filter(value)
 194     def handleLE(option, opt, value, parser):
 195       self.setLineEndings(value)
 196     def handleMarker(option, opt, value, parser):
 197       self.setMarker(value)
 198     p = OptionParser()
 199     p.add_option('-I', action='append', type="string", default = [],
 200                  metavar="FILENAME", help='Include file')
 201     p.add_option('-E', action='callback', callback=handleE,
 202                  help='Import the environment into the defined variables')
 203     p.add_option('-D', action='callback', callback=handleD, type="string",
 204                  metavar="VAR[=VAL]", help='Define a variable')
 205     p.add_option('-U', action='callback', callback=handleU, type="string",
 206                  metavar="VAR", help='Undefine a variable')
 207     p.add_option('-F', action='callback', callback=handleF, type="string",
 208                  metavar="FILTER", help='Enable the specified filter')
 209     p.add_option('--line-endings', action='callback', callback=handleLE,
 210                  type="string", metavar="[cr|lr|crlf]",
 211                  help='Use the specified line endings [Default: OS dependent]')
 212     p.add_option('--marker', action='callback', callback=handleMarker,
 213                  type="string",
 214                  help='Use the specified marker instead of #')
 215     return p
 216
 217   def handleLine(self, aLine):
 218     """
 219     Handle a single line of input (internal).
 220     """
 221     m = self.instruction.match(aLine)
 222     if m:
 223       args = None
 224       cmd = m.group('cmd')
 225       try:
 226         args = m.group('args')
 227       except IndexError:
 228         pass
 229       if cmd not in self.cmds:
 230         raise Preprocessor.Error(self, 'INVALID_CMD', aLine)
 231       level, cmd = self.cmds[cmd]
 232       if (level >= self.disableLevel):
 233         cmd(args)
 234     elif self.disableLevel == 0 and not self.comment.match(aLine):
 235       self.write(aLine)
 236     pass
 237
 238   # Instruction handlers
 239   # These are named do_'instruction name' and take one argument
 240
 241   # Variables
 242   def do_define(self, args):
 243     m = re.match('(?P<name>\w+)(?:\s(?P<value>.*))?', args, re.U)
 244     if not m:
 245       raise Preprocessor.Error(self, 'SYNTAX_DEF', args)
 246     val = 1
 247     if m.group('value'):
 248       val = m.group('value')
 249       try:
 250         if val[0] == '0':
 251           val = int(val, 8)
 252         else:
 253           val = int(val)
 254       except:
 255         pass
 256     self.context[m.group('name')] = val
 257   def do_undef(self, args):
 258     m = re.match('(?P<name>\w+)$', args, re.U)
 259     if not m:
 260       raise Preprocessor.Error(self, 'SYNTAX_DEF', args)
 261     if args in self.context:
 262       del self.context[args]
 263   # Logic
 264   def ensure_not_else(self):
 265     if len(self.ifStates) == 0 or self.ifStates[-1] == 2:
 266       sys.stderr.write('WARNING: bad nesting of #else\n')
 267   def do_if(self, args, replace=False):
 268     if self.disableLevel and not replace:
 269       self.disableLevel += 1
 270       return
 271     val = None
 272     try:
 273       e = Expression.Expression(args)
 274       val = e.evaluate(self.context)
 275     except Exception:
 276       # XXX do real error reporting
 277       raise Preprocessor.Error(self, 'SYNTAX_ERR', args)
 278     if type(val) == str:
 279       # we're looking for a number value, strings are false
 280       val = False
 281     if not val:
 282       self.disableLevel = 1
 283     if replace:
 284       if val:
 285         self.disableLevel = 0
 286       self.ifStates[-1] = self.disableLevel
 287     else:
 288       self.ifStates.append(self.disableLevel)
 289     pass
 290   def do_ifdef(self, args, replace=False):
 291     if self.disableLevel and not replace:
 292       self.disableLevel += 1
 293       return
 294     if re.match('\W', args, re.U):
 295       raise Preprocessor.Error(self, 'INVALID_VAR', args)
 296     if args not in self.context:
 297       self.disableLevel = 1
 298     if replace:
 299       if args in self.context:
 300         self.disableLevel = 0
 301       self.ifStates[-1] = self.disableLevel
 302     else:
 303       self.ifStates.append(self.disableLevel)
 304     pass
 305   def do_ifndef(self, args, replace=False):
 306     if self.disableLevel and not replace:
 307       self.disableLevel += 1
 308       return
 309     if re.match('\W', args, re.U):
 310       raise Preprocessor.Error(self, 'INVALID_VAR', args)
 311     if args in self.context:
 312       self.disableLevel = 1
 313     if replace:
 314       if args not in self.context:
 315         self.disableLevel = 0
 316       self.ifStates[-1] = self.disableLevel
 317     else:
 318       self.ifStates.append(self.disableLevel)
 319     pass
 320   def do_else(self, args, ifState = 2):
 321     self.ensure_not_else()
 322     hadTrue = self.ifStates[-1] == 0
 323     self.ifStates[-1] = ifState # in-else
 324     if hadTrue:
 325       self.disableLevel = 1
 326       return
 327     self.disableLevel = 0
 328   def do_elif(self, args):
 329     if self.disableLevel == 1:
 330       if self.ifStates[-1] == 1:
 331         self.do_if(args, replace=True)
 332     else:
 333       self.do_else(None, self.ifStates[-1])
 334   def do_elifdef(self, args):
 335     if self.disableLevel == 1:
 336       if self.ifStates[-1] == 1:
 337         self.do_ifdef(args, replace=True)
 338     else:
 339       self.do_else(None, self.ifStates[-1])
 340   def do_elifndef(self, args):
 341     if self.disableLevel == 1:
 342       if self.ifStates[-1] == 1:
 343         self.do_ifndef(args, replace=True)
 344     else:
 345       self.do_else(None, self.ifStates[-1])
 346   def do_endif(self, args):
 347     if self.disableLevel > 0:
 348       self.disableLevel -= 1
 349     if self.disableLevel == 0:
 350       self.ifStates.pop()
 351   # output processing
 352   def do_expand(self, args):
 353     lst = re.split('__(\w+)__', args, re.U)
 354     do_replace = False
 355     def vsubst(v):
 356       if v in self.context:
 357         return str(self.context[v])
 358       return ''
 359     for i in range(1, len(lst), 2):
 360       lst[i] = vsubst(lst[i])
 361     lst.append('\n') # add back the newline
 362     self.write(reduce(lambda x, y: x+y, lst, ''))
 363   def do_literal(self, args):
 364     self.write(args + self.LE)
 365   def do_filter(self, args):
 366     filters = [f for f in args.split(' ') if hasattr(self, 'filter_' + f)]
 367     if len(filters) == 0:
 368       return
 369     current = dict(self.filters)
 370     for f in filters:
 371       current[f] = getattr(self, 'filter_' + f)
 372     filterNames = current.keys()
 373     filterNames.sort()
 374     self.filters = [(fn, current[fn]) for fn in filterNames]
 375     return
 376   def do_unfilter(self, args):
 377     filters = args.split(' ')
 378     current = dict(self.filters)
 379     for f in filters:
 380       if f in current:
 381         del current[f]
 382     filterNames = current.keys()
 383     filterNames.sort()
 384     self.filters = [(fn, current[fn]) for fn in filterNames]
 385     return
 386   # Filters
 387   #
 388   # emptyLines
 389   #   Strips blank lines from the output.
 390   def filter_emptyLines(self, aLine):
 391     if aLine == '\n':
 392       return ''
 393     return aLine
 394   # slashslash
 395   #   Strips everything after //
 396   def filter_slashslash(self, aLine):
 397     [aLine, rest] = aLine.split('//', 1)
 398     if rest:
 399       aLine += '\n'
 400     return aLine
 401   # spaces
 402   #   Collapses sequences of spaces into a single space
 403   def filter_spaces(self, aLine):
 404     return re.sub(' +', ' ', aLine).strip(' ')
 405   # substition
 406   #   helper to be used by both substition and attemptSubstitution
 407   def filter_substitution(self, aLine, fatal=True):
 408     def repl(matchobj):
 409       varname = matchobj.group('VAR')
 410       if varname in self.context:
 411         return str(self.context[varname])
 412       if fatal:
 413         raise Preprocessor.Error(self, 'UNDEFINED_VAR', varname)
 414       return ''
 415     return self.varsubst.sub(repl, aLine)
 416   def filter_attemptSubstitution(self, aLine):
 417     return self.filter_substitution(aLine, fatal=False)
 418   # File ops
 419   def do_include(self, args):
 420     """
 421     Preprocess a given file.
 422     args can either be a file name, or a file-like object.
 423     Files should be opened, and will be closed after processing.
 424     """
 425     isName = type(args) == str or type(args) == unicode
 426     oldWrittenLines = self.writtenLines
 427     oldCheckLineNumbers = self.checkLineNumbers
 428     self.checkLineNumbers = False
 429     if isName:
 430       try:
 431         args = str(args)
 432         if not os.path.isabs(args):
 433           args = os.path.join(self.context['DIRECTORY'], args)
 434         args = open(args, 'rU')
 435       except:
 436         raise Preprocessor.Error(self, 'FILE_NOT_FOUND', str(args))
 437     self.checkLineNumbers = bool(re.search('\.js(?:\.in)?$', args.name))
 438     oldFile = self.context['FILE']
 439     oldLine = self.context['LINE']
 440     oldDir = self.context['DIRECTORY']
 441     if args.isatty():
 442       # we're stdin, use '-' and '' for file and dir
 443       self.context['FILE'] = '-'
 444       self.context['DIRECTORY'] = ''
 445     else:
 446       abspath = os.path.abspath(args.name)
 447       self.context['FILE'] = abspath
 448       self.context['DIRECTORY'] = os.path.dirname(abspath)
 449     self.context['LINE'] = 0
 450     self.writtenLines = 0
 451     for l in args:
 452       self.context['LINE'] += 1
 453       self.handleLine(l)
 454     args.close()
 455     self.context['FILE'] = oldFile
 456     self.checkLineNumbers = oldCheckLineNumbers
 457     self.writtenLines = oldWrittenLines
 458     self.context['LINE'] = oldLine
 459     self.context['DIRECTORY'] = oldDir
 460   def do_includesubst(self, args):
 461     args = self.filter_substitution(args)
 462     self.do_include(args)
 463   def do_error(self, args):
 464     raise Preprocessor.Error(self, 'Error: ', str(args))
 465
 466 def main():
 467   pp = Preprocessor()
 468   pp.handleCommandLine(None, True)
 469   return
 470
 471 def preprocess(includes=[sys.stdin], defines={},
 472                output = sys.stdout,
 473                line_endings='\n', marker='#'):
 474   pp = Preprocessor()
 475   pp.context.update(defines)
 476   pp.setLineEndings(line_endings)
 477   pp.setMarker(marker)
 478   pp.out = output
 479   for f in includes:
 480     pp.do_include(f)
 481
 482 if __name__ == "__main__":
 483   main()