config/Preprocessor.py

   1 """
   2 This is a very primitive line based preprocessor, for times when using
   3 a C preprocessor isn't an option.
   4 """
   5
   6 # ***** BEGIN LICENSE BLOCK *****
   7 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   8 #
   9 # The contents of this file are subject to the Mozilla Public License Version
  10 # 1.1 (the "License"); you may not use this file except in compliance with
  11 # the License. You may obtain a copy of the License at
  12 # http://www.mozilla.org/MPL/
  13 #
  14 # Software distributed under the License is distributed on an "AS IS" basis,
  15 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  16 # for the specific language governing rights and limitations under the
  17 # License.
  18 #
  19 # The Original Code is Mozilla build system.
  20 #
  21 # The Initial Developer of the Original Code is
  22 # Mozilla Foundation.
  23 # Portions created by the Initial Developer are Copyright (C) 2007
  24 # the Initial Developer. All Rights Reserved.
  25 #
  26 # Contributor(s):
  27 #  Axel Hecht <axel@pike.org>
  28 #
  29 # Alternatively, the contents of this file may be used under the terms of
  30 # either the GNU General Public License Version 2 or later (the "GPL"), or
  31 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  32 # in which case the provisions of the GPL or the LGPL are applicable instead
  33 # of those above. If you wish to allow use of your version of this file only
  34 # under the terms of either the GPL or the LGPL, and not to allow others to
  35 # use your version of this file under the terms of the MPL, indicate your
  36 # decision by deleting the provisions above and replace them with the notice
  37 # and other provisions required by the GPL or the LGPL. If you do not delete
  38 # the provisions above, a recipient may use your version of this file under
  39 # the terms of any one of the MPL, the GPL or the LGPL.
  40 #
  41 # ***** END LICENSE BLOCK *****
  42
  43 import sys
  44 import os
  45 import os.path
  46 import re
  47 from optparse import OptionParser
  48
  49 # hack around win32 mangling our line endings
  50 # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65443
  51 if sys.platform == "win32":
  52   import os, msvcrt
  53   msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
  54
  55 import Expression
  56
  57 __all__ = ['Preprocessor', 'preprocess']
  58
  59
  60 class Preprocessor:
  61   """
  62   Class for preprocessing text files.
  63   """
  64   class Error(RuntimeError):
  65     def __init__(self, cpp, MSG, context):
  66       self.file = cpp.context['FILE']
  67       self.line = cpp.context['LINE']
  68       self.key = MSG
  69       RuntimeError.__init__(self, (self.file, self.line, self.key, context))
  70   def __init__(self):
  71     self.context = Expression.Context()
  72     for k,v in {'FILE': '',
  73                 'LINE': 0,
  74                 'DIRECTORY': os.path.abspath('.')}.iteritems():
  75       self.context[k] = v
  76     self.disableLevel = 0
  77     # ifStates can be
  78     #  0: hadTrue
  79     #  1: wantsTrue
  80     #  2: #else found
  81     self.ifStates = []
  82     self.checkLineNumbers = False
  83     self.writtenLines = 0
  84     self.filters = []
  85     self.cmds = {}
  86     for cmd, level in {'define': 0,
  87                        'undef': 0,
  88                        'if': sys.maxint,
  89                        'ifdef': sys.maxint,
  90                        'ifndef': sys.maxint,
  91                        'else': 1,
  92                        'elif': 1,
  93                        'elifdef': 1,
  94                        'elifndef': 1,
  95                        'endif': sys.maxint,
  96                        'expand': 0,
  97                        'literal': 0,
  98                        'filter': 0,
  99                        'unfilter': 0,
 100                        'include': 0,
 101                        'includesubst': 0,
 102                        'error': 0}.iteritems():
 103       self.cmds[cmd] = (level, getattr(self, 'do_' + cmd))
 104     self.out = sys.stdout
 105     self.setMarker('#')
 106     self.LE = '\n'
 107     self.varsubst = re.compile('@(?P<VAR>\w+)@', re.U)
 108
 109   def setLineEndings(self, aLE):
 110     """
 111     Set the line endings to be used for output.
 112     """
 113     self.LE = {'cr': '\x0D', 'lf': '\x0A', 'crlf': '\x0D\x0A'}[aLE]
 114
 115   def setMarker(self, aMarker):
 116     """
 117     Set the marker to be used for processing directives.
 118     Used for handling CSS files, with pp.setMarker('%'), for example.
 119     """
 120     self.marker = aMarker
 121     self.instruction = re.compile('%s(?P<cmd>[a-z]+)(?:\s(?P<args>.*))?$'%aMarker, re.U)
 122     self.comment = re.compile(aMarker, re.U)
 123
 124   def clone(self):
 125     """
 126     Create a clone of the current processor, including line ending
 127     settings, marker, variable definitions, output stream.
 128     """
 129     rv = Preprocessor()
 130     rv.context.update(self.context)
 131     rv.setMarker(self.marker)
 132     rv.LE = self.LE
 133     rv.out = self.out
 134     return rv
 135
 136   def write(self, aLine):
 137     """
 138     Internal method for handling output.
 139     """
 140     if self.checkLineNumbers:
 141       self.writtenLines += 1
 142       ln = self.context['LINE']
 143       if self.writtenLines != ln:
 144         self.out.write('//@line %(line)d "%(file)s"%(le)s'%{'line': ln,
 145                                                             'file': self.context['FILE'],
 146                                                             'le': self.LE})
 147         self.writtenLines = ln
 148     for f in self.filters:
 149       aLine = f[1](aLine)
 150     # ensure our line ending. Only need to handle \n, as we're reading
 151     # with universal line ending support, at least for files.
 152     aLine = re.sub('\n', self.LE, aLine)
 153     self.out.write(aLine)
 154
 155   def handleCommandLine(self, args, defaultToStdin = False):
 156     """
 157     Parse a commandline into this parser.
 158     Uses OptionParser internally, no args mean sys.argv[1:].
 159     """
 160     includes = []
 161     def handleI(option, opt, value, parser):
 162       includes.append(value)
 163     def handleE(option, opt, value, parser):
 164       for k,v in os.environ.iteritems():
 165         self.context[k] = v
 166     def handleD(option, opt, value, parser):
 167       vals = value.split('=')
 168       assert len(vals) < 3
 169       if len(vals) == 1:
 170         vals.append(1)
 171       self.context[vals[0]] = vals[1]
 172     def handleU(option, opt, value, parser):
 173       del self.context[value]
 174     def handleF(option, opt, value, parser):
 175       self.do_filter(value)
 176     def handleLE(option, opt, value, parser):
 177       self.setLineEndings(value)
 178     def handleMarker(option, opt, value, parser):
 179       self.setMarker(value)
 180     p = OptionParser()
 181     p.add_option('-I', action='callback', callback=handleI, type="string",
 182                  metavar="FILENAME", help='Include file')
 183     p.add_option('-E', action='callback', callback=handleE,
 184                  help='Import the environment into the defined variables')
 185     p.add_option('-D', action='callback', callback=handleD, type="string",
 186                  metavar="VAR[=VAL]", help='Define a variable')
 187     p.add_option('-U', action='callback', callback=handleU, type="string",
 188                  metavar="VAR", help='Undefine a variable')
 189     p.add_option('-F', action='callback', callback=handleF, type="string",
 190                  metavar="FILTER", help='Enabble the specified filter')
 191     p.add_option('--line-endings', action='callback', callback=handleLE,
 192                  type="string", metavar="[cr|lr|crlf]",
 193                  help='Use the specified line endings [Default: OS dependent]')
 194     p.add_option('--marker', action='callback', callback=handleMarker,
 195                  type="string",
 196                  help='Use the specified marker instead of #')
 197     (options, args) = p.parse_args(args=args)
 198     if defaultToStdin and len(args) == 0:
 199       args = [sys.stdin]
 200     includes.extend(args)
 201     for f in includes:
 202       self.do_include(f)
 203     pass
 204
 205   def handleLine(self, aLine):
 206     """
 207     Handle a single line of input (internal).
 208     """
 209     m = self.instruction.match(aLine)
 210     if m:
 211       args = None
 212       cmd = m.group('cmd')
 213       try:
 214         args = m.group('args')
 215       except IndexError:
 216         pass
 217       if cmd not in self.cmds:
 218         raise Preprocessor.Error(self, 'INVALID_CMD', aLine)
 219       level, cmd = self.cmds[cmd]
 220       if (level >= self.disableLevel):
 221         cmd(args)
 222     elif self.disableLevel == 0 and not self.comment.match(aLine):
 223       self.write(aLine)
 224     pass
 225
 226   # Instruction handlers
 227   # These are named do_'instruction name' and take one argument
 228
 229   # Variables
 230   def do_define(self, args):
 231     m = re.match('(?P<name>\w+)(?:\s(?P<value>.*))?', args, re.U)
 232     if not m:
 233       raise Preprocessor.Error(self, 'SYNTAX_DEF', args)
 234     val = 1
 235     if m.group('value'):
 236       val = m.group('value')
 237       try:
 238         if val[0] == '0':
 239           val = int(val, 8)
 240         else:
 241           val = int(val)
 242       except:
 243         pass
 244     self.context[m.group('name')] = val
 245   def do_undef(self, args):
 246     m = re.match('(?P<name>\w+)$', args, re.U)
 247     if not m:
 248       raise Preprocessor.Error(self, 'SYNTAX_DEF', args)
 249     if args in self.context:
 250       del self.context[args]
 251   # Logic
 252   def ensure_not_else(self):
 253     if len(self.ifStates) == 0 or self.ifStates[-1] == 2:
 254       sys.stderr.write('WARNING: bad nesting of #else\n')
 255   def do_if(self, args, replace=False):
 256     if self.disableLevel and not replace:
 257       self.disableLevel += 1
 258       return
 259     val = None
 260     try:
 261       e = Expression.Expression(args)
 262       val = e.evaluate(self.context)
 263     except Exception:
 264       # XXX do real error reporting
 265       raise Preprocessor.Error(self, 'SYNTAX_ERR', args)
 266     if type(val) == str:
 267       # we're looking for a number value, strings are false
 268       val = False
 269     if not val:
 270       self.disableLevel = 1
 271     if replace:
 272       if val:
 273         self.disableLevel = 0
 274       self.ifStates[-1] = self.disableLevel
 275     else:
 276       self.ifStates.append(self.disableLevel)
 277     pass
 278   def do_ifdef(self, args, replace=False):
 279     if self.disableLevel and not replace:
 280       self.disableLevel += 1
 281       return
 282     if re.match('\W', args, re.U):
 283       raise Preprocessor.Error(self, 'INVALID_VAR', args)
 284     if args not in self.context:
 285       self.disableLevel = 1
 286     if replace:
 287       if args in self.context:
 288         self.disableLevel = 0
 289       self.ifStates[-1] = self.disableLevel
 290     else:
 291       self.ifStates.append(self.disableLevel)
 292     pass
 293   def do_ifndef(self, args, replace=False):
 294     if self.disableLevel and not replace:
 295       self.disableLevel += 1
 296       return
 297     if re.match('\W', args, re.U):
 298       raise Preprocessor.Error(self, 'INVALID_VAR', args)
 299     if args in self.context:
 300       self.disableLevel = 1
 301     if replace:
 302       if args not in self.context:
 303         self.disableLevel = 0
 304       self.ifStates[-1] = self.disableLevel
 305     else:
 306       self.ifStates.append(self.disableLevel)
 307     pass
 308   def do_else(self, args, ifState = 2):
 309     self.ensure_not_else()
 310     hadTrue = self.ifStates[-1] == 0
 311     self.ifStates[-1] = ifState # in-else
 312     if hadTrue:
 313       self.disableLevel = 1
 314       return
 315     self.disableLevel = 0
 316   def do_elif(self, args):
 317     if self.disableLevel == 1:
 318       if self.ifStates[-1] == 1:
 319         self.do_if(args, replace=True)
 320     else:
 321       self.do_else(None, self.ifStates[-1])
 322   def do_elifdef(self, args):
 323     if self.disableLevel == 1:
 324       if self.ifStates[-1] == 1:
 325         self.do_ifdef(args, replace=True)
 326     else:
 327       self.do_else(None, self.ifStates[-1])
 328   def do_elifndef(self, args):
 329     if self.disableLevel == 1:
 330       if self.ifStates[-1] == 1:
 331         self.do_ifndef(args, replace=True)
 332     else:
 333       self.do_else(None, self.ifStates[-1])
 334   def do_endif(self, args):
 335     if self.disableLevel > 0:
 336       self.disableLevel -= 1
 337     if self.disableLevel == 0:
 338       self.ifStates.pop()
 339   # output processing
 340   def do_expand(self, args):
 341     lst = re.split('__(\w+)__', args, re.U)
 342     do_replace = False
 343     def vsubst(v):
 344       if v in self.context:
 345         return str(self.context[v])
 346       return ''
 347     for i in range(1, len(lst), 2):
 348       lst[i] = vsubst(lst[i])
 349     lst.append('\n') # add back the newline
 350     self.write(reduce(lambda x, y: x+y, lst, ''))
 351   def do_literal(self, args):
 352     self.write(args + self.LE)
 353   def do_filter(self, args):
 354     filters = [f for f in args.split(' ') if hasattr(self, 'filter_' + f)]
 355     if len(filters) == 0:
 356       return
 357     current = dict(self.filters)
 358     for f in filters:
 359       current[f] = getattr(self, 'filter_' + f)
 360     filterNames = current.keys()
 361     filterNames.sort()
 362     self.filters = [(fn, current[fn]) for fn in filterNames]
 363     return
 364   def do_unfilter(self, args):
 365     filters = args.split(' ')
 366     current = dict(self.filters)
 367     for f in filters:
 368       if f in current:
 369         del current[f]
 370     filterNames = current.keys()
 371     filterNames.sort()
 372     self.filters = [(fn, current[fn]) for fn in filterNames]
 373     return
 374   # Filters
 375   #
 376   # emptyLines
 377   #   Strips blank lines from the output.
 378   def filter_emptyLines(self, aLine):
 379     if aLine == '\n':
 380       return ''
 381     return aLine
 382   # slashslash
 383   #   Strips everything after //
 384   def filter_slashslash(self, aLine):
 385     [aLine, rest] = aLine.split('//', 1)
 386     if rest:
 387       aLine += '\n'
 388     return aLine
 389   # spaces
 390   #   Collapses sequences of spaces into a single space
 391   def filter_spaces(self, aLine):
 392     return re.sub(' +', ' ', aLine).strip(' ')
 393   # substition
 394   #   helper to be used by both substition and attemptSubstitution
 395   def filter_substitution(self, aLine, fatal=True):
 396     def repl(matchobj):
 397       varname = matchobj.group('VAR')
 398       if varname in self.context:
 399         return str(self.context[varname])
 400       if fatal:
 401         raise Preprocessor.Error(self, 'UNDEFINED_VAR', varname)
 402       return ''
 403     return self.varsubst.sub(repl, aLine)
 404   def filter_attemptSubstitution(self, aLine):
 405     return self.filter_substitution(aLine, fatal=False)
 406   # File ops
 407   def do_include(self, args):
 408     """
 409     Preprocess a given file.
 410     args can either be a file name, or a file-like object.
 411     Files should be opened, and will be closed after processing.
 412     """
 413     isName = type(args) == str or type(args) == unicode
 414     oldWrittenLines = self.writtenLines
 415     oldCheckLineNumbers = self.checkLineNumbers
 416     self.checkLineNumbers = False
 417     if isName:
 418       try:
 419         args = str(args)
 420         if not os.path.isabs(args):
 421           args = os.path.join(self.context['DIRECTORY'], args)
 422         args = open(args, 'rU')
 423       except:
 424         raise Preprocessor.Error(self, 'FILE_NOT_FOUND', str(args))
 425     self.checkLineNumbers = bool(re.search('\.js(?:\.in)?$', args.name))
 426     oldFile = self.context['FILE']
 427     oldLine = self.context['LINE']
 428     oldDir = self.context['DIRECTORY']
 429     if args.isatty():
 430       # we're stdin, use '-' and '' for file and dir
 431       self.context['FILE'] = '-'
 432       self.context['DIRECTORY'] = ''
 433     else:
 434       abspath = os.path.abspath(args.name)
 435       self.context['FILE'] = abspath
 436       self.context['DIRECTORY'] = os.path.dirname(abspath)
 437     self.context['LINE'] = 0
 438     self.writtenLines = 0
 439     for l in args:
 440       self.context['LINE'] += 1
 441       self.handleLine(l)
 442     args.close()
 443     self.context['FILE'] = oldFile
 444     self.checkLineNumbers = oldCheckLineNumbers
 445     self.writtenLines = oldWrittenLines
 446     self.context['LINE'] = oldLine
 447     self.context['DIRECTORY'] = oldDir
 448   def do_includesubst(self, args):
 449     args = self.filter_substitution(args)
 450     self.do_include(args)
 451   def do_error(self, args):
 452     raise Preprocessor.Error(self, 'Error: ', str(args))
 453
 454 def main():
 455   pp = Preprocessor()
 456   pp.handleCommandLine(None, True)
 457   return
 458
 459 def preprocess(includes=[sys.stdin], defines={},
 460                output = sys.stdout,
 461                line_endings='\n', marker='#'):
 462   pp = Preprocessor()
 463   pp.context.update(defines)
 464   pp.setLineEndings(line_endings)
 465   pp.setMarker(marker)
 466   pp.out = output
 467   for f in includes:
 468     pp.do_include(f)
 469
 470 if __name__ == "__main__":
 471   main()