config/Preprocessor.py

   1 """
   2 This is a very primitive line based preprocessor, for times when using
   3 a C preprocessor isn't an option.
   4 """
   5
   6 # ***** BEGIN LICENSE BLOCK *****
   7 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   8 #
   9 # The contents of this file are subject to the Mozilla Public License Version
  10 # 1.1 (the "License"); you may not use this file except in compliance with
  11 # the License. You may obtain a copy of the License at
  12 # http://www.mozilla.org/MPL/
  13 #
  14 # Software distributed under the License is distributed on an "AS IS" basis,
  15 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  16 # for the specific language governing rights and limitations under the
  17 # License.
  18 #
  19 # The Original Code is Mozilla build system.
  20 #
  21 # The Initial Developer of the Original Code is
  22 # Mozilla Foundation.
  23 # Portions created by the Initial Developer are Copyright (C) 2007
  24 # the Initial Developer. All Rights Reserved.
  25 #
  26 # Contributor(s):
  27 #  Axel Hecht <axel@pike.org>
  28 #
  29 # Alternatively, the contents of this file may be used under the terms of
  30 # either the GNU General Public License Version 2 or later (the "GPL"), or
  31 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  32 # in which case the provisions of the GPL or the LGPL are applicable instead
  33 # of those above. If you wish to allow use of your version of this file only
  34 # under the terms of either the GPL or the LGPL, and not to allow others to
  35 # use your version of this file under the terms of the MPL, indicate your
  36 # decision by deleting the provisions above and replace them with the notice
  37 # and other provisions required by the GPL or the LGPL. If you do not delete
  38 # the provisions above, a recipient may use your version of this file under
  39 # the terms of any one of the MPL, the GPL or the LGPL.
  40 #
  41 # ***** END LICENSE BLOCK *****
  42
  43 import sys
  44 import os
  45 import os.path
  46 import re
  47 from optparse import OptionParser
  48
  49 # hack around win32 mangling our line endings
  50 # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65443
  51 if sys.platform == "win32":
  52   import os, msvcrt
  53   msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
  54
  55 import Expression
  56
  57 __all__ = ['Preprocessor', 'preprocess']
  58
  59
  60 class Preprocessor:
  61   """
  62   Class for preprocessing text files.
  63   """
  64   class Error(RuntimeError):
  65     def __init__(self, cpp, MSG, context):
  66       self.file = cpp.context['FILE']
  67       self.line = cpp.context['LINE']
  68       self.key = MSG
  69       RuntimeError.__init__(self, (self.file, self.line, self.key, context))
  70   def __init__(self):
  71     self.context = Expression.Context()
  72     for k,v in {'FILE': '',
  73                 'LINE': 0,
  74                 'DIRECTORY': os.path.abspath('.')}.iteritems():
  75       self.context[k] = v
  76     self.disableLevel = 0
  77     # ifStates can be
  78     #  0: hadTrue
  79     #  1: wantsTrue
  80     #  2: #else found
  81     self.ifStates = []
  82     self.checkLineNumbers = False
  83     self.writtenLines = 0
  84     self.filters = []
  85     self.cmds = {}
  86     for cmd, level in {'define': 0,
  87                        'undef': 0,
  88                        'if': sys.maxint,
  89                        'ifdef': sys.maxint,
  90                        'ifndef': sys.maxint,
  91                        'else': 1,
  92                        'elif': 1,
  93                        'elifdef': 1,
  94                        'elifndef': 1,
  95                        'endif': sys.maxint,
  96                        'expand': 0,
  97                        'literal': 0,
  98                        'filter': 0,
  99                        'unfilter': 0,
 100                        'include': 0,
 101                        'includesubst': 0,
 102                        'error': 0}.iteritems():
 103       self.cmds[cmd] = (level, getattr(self, 'do_' + cmd))
 104     self.out = sys.stdout
 105     self.setMarker('#')
 106     self.LE = '\n'
 107     self.varsubst = re.compile('@(?P<VAR>\w+)@', re.U)
 108
 109   def setLineEndings(self, aLE):
 110     """
 111     Set the line endings to be used for output.
 112     """
 113     self.LE = {'cr': '\x0D', 'lf': '\x0A', 'crlf': '\x0D\x0A'}[aLE]
 114
 115   def setMarker(self, aMarker):
 116     """
 117     Set the marker to be used for processing directives.
 118     Used for handling CSS files, with pp.setMarker('%'), for example.
 119     """
 120     self.marker = aMarker
 121     self.instruction = re.compile('%s(?P<cmd>[a-z]+)(?:\s(?P<args>.*))?$'%aMarker, re.U)
 122     self.comment = re.compile(aMarker, re.U)
 123
 124   def clone(self):
 125     """
 126     Create a clone of the current processor, including line ending
 127     settings, marker, variable definitions, output stream.
 128     """
 129     rv = Preprocessor()
 130     rv.context.update(self.context)
 131     rv.setMarker(self.marker)
 132     rv.LE = self.LE
 133     rv.out = self.out
 134     return rv
 135
 136   def write(self, aLine):
 137     """
 138     Internal method for handling output.
 139     """
 140     if self.checkLineNumbers:
 141       self.writtenLines += 1
 142       ln = self.context['LINE']
 143       if self.writtenLines != ln:
 144         self.out.write('//@line %(line)d "%(file)s"%(le)s'%{'line': ln,
 145                                                             'file': self.context['FILE'],
 146                                                             'le': self.LE})
 147         self.writtenLines = ln
 148     for f in self.filters:
 149       aLine = f[1](aLine)
 150     aLine = aLine.rstrip('\r\n') + self.LE
 151     self.out.write(aLine)
 152
 153   def handleCommandLine(self, args, defaultToStdin = False):
 154     """
 155     Parse a commandline into this parser.
 156     Uses OptionParser internally, no args mean sys.argv[1:].
 157     """
 158     includes = []
 159     def handleI(option, opt, value, parser):
 160       includes.append(value)
 161     def handleE(option, opt, value, parser):
 162       for k,v in os.environ.iteritems():
 163         self.context[k] = v
 164     def handleD(option, opt, value, parser):
 165       vals = value.split('=')
 166       assert len(vals) < 3
 167       if len(vals) == 1:
 168         vals.append(1)
 169       self.context[vals[0]] = vals[1]
 170     def handleU(option, opt, value, parser):
 171       del self.context[value]
 172     def handleF(option, opt, value, parser):
 173       self.do_filter(value)
 174     def handleLE(option, opt, value, parser):
 175       self.setLineEndings(value)
 176     def handleMarker(option, opt, value, parser):
 177       self.setMarker(value)
 178     p = OptionParser()
 179     p.add_option('-I', action='callback', callback=handleI, type="string",
 180                  metavar="FILENAME", help='Include file')
 181     p.add_option('-E', action='callback', callback=handleE,
 182                  help='Import the environment into the defined variables')
 183     p.add_option('-D', action='callback', callback=handleD, type="string",
 184                  metavar="VAR[=VAL]", help='Define a variable')
 185     p.add_option('-U', action='callback', callback=handleU, type="string",
 186                  metavar="VAR", help='Undefine a variable')
 187     p.add_option('-F', action='callback', callback=handleF, type="string",
 188                  metavar="FILTER", help='Enabble the specified filter')
 189     p.add_option('--line-endings', action='callback', callback=handleLE,
 190                  type="string", metavar="[cr|lr|crlf]",
 191                  help='Use the specified line endings [Default: OS dependent]')
 192     p.add_option('--marker', action='callback', callback=handleMarker,
 193                  type="string",
 194                  help='Use the specified marker instead of #')
 195     (options, args) = p.parse_args(args=args)
 196     if defaultToStdin and len(args) == 0:
 197       args = [sys.stdin]
 198     includes.extend(args)
 199     for f in includes:
 200       self.do_include(f)
 201     pass
 202
 203   def handleLine(self, aLine):
 204     """
 205     Handle a single line of input (internal).
 206     """
 207     m = self.instruction.match(aLine)
 208     if m:
 209       args = None
 210       cmd = m.group('cmd')
 211       try:
 212         args = m.group('args')
 213       except IndexError:
 214         pass
 215       if cmd not in self.cmds:
 216         raise Preprocessor.Error(self, 'INVALID_CMD', aLine)
 217       level, cmd = self.cmds[cmd]
 218       if (level >= self.disableLevel):
 219         cmd(args)
 220     elif self.disableLevel == 0 and not self.comment.match(aLine):
 221       self.write(aLine)
 222     pass
 223
 224   # Instruction handlers
 225   # These are named do_'instruction name' and take one argument
 226
 227   # Variables
 228   def do_define(self, args):
 229     m = re.match('(?P<name>\w+)(?:\s(?P<value>.*))?', args, re.U)
 230     if not m:
 231       raise Preprocessor.Error(self, 'SYNTAX_DEF', args)
 232     val = 1
 233     if m.group('value'):
 234       val = m.group('value')
 235       try:
 236         if val[0] == '0':
 237           val = int(val, 8)
 238         else:
 239           val = int(val)
 240       except:
 241         pass
 242     self.context[m.group('name')] = val
 243   def do_undef(self, args):
 244     m = re.match('(?P<name>\w+)$', args, re.U)
 245     if not m:
 246       raise Preprocessor.Error(self, 'SYNTAX_DEF', args)
 247     if args in self.context:
 248       del self.context[args]
 249   # Logic
 250   def ensure_not_else(self):
 251     if len(self.ifStates) == 0 or self.ifStates[-1] == 2:
 252       sys.stderr.write('WARNING: bad nesting of #else\n')
 253   def do_if(self, args, replace=False):
 254     if self.disableLevel and not replace:
 255       self.disableLevel += 1
 256       return
 257     val = None
 258     try:
 259       e = Expression.Expression(args)
 260       val = e.evaluate(self.context)
 261     except Exception:
 262       # XXX do real error reporting
 263       raise Preprocessor.Error(self, 'SYNTAX_ERR', args)
 264     if type(val) == str:
 265       # we're looking for a number value, strings are false
 266       val = False
 267     if not val:
 268       self.disableLevel = 1
 269     if replace:
 270       if val:
 271         self.disableLevel = 0
 272       self.ifStates[-1] = self.disableLevel
 273     else:
 274       self.ifStates.append(self.disableLevel)
 275     pass
 276   def do_ifdef(self, args, replace=False):
 277     if self.disableLevel and not replace:
 278       self.disableLevel += 1
 279       return
 280     if re.match('\W', args, re.U):
 281       raise Preprocessor.Error(self, 'INVALID_VAR', args)
 282     if args not in self.context:
 283       self.disableLevel = 1
 284     if replace:
 285       if args in self.context:
 286         self.disableLevel = 0
 287       self.ifStates[-1] = self.disableLevel
 288     else:
 289       self.ifStates.append(self.disableLevel)
 290     pass
 291   def do_ifndef(self, args, replace=False):
 292     if self.disableLevel and not replace:
 293       self.disableLevel += 1
 294       return
 295     if re.match('\W', args, re.U):
 296       raise Preprocessor.Error(self, 'INVALID_VAR', args)
 297     if args in self.context:
 298       self.disableLevel = 1
 299     if replace:
 300       if args not in self.context:
 301         self.disableLevel = 0
 302       self.ifStates[-1] = self.disableLevel
 303     else:
 304       self.ifStates.append(self.disableLevel)
 305     pass
 306   def do_else(self, args, ifState = 2):
 307     self.ensure_not_else()
 308     hadTrue = self.ifStates[-1] == 0
 309     self.ifStates[-1] = ifState # in-else
 310     if hadTrue:
 311       self.disableLevel = 1
 312       return
 313     self.disableLevel = 0
 314   def do_elif(self, args):
 315     if self.disableLevel == 1:
 316       if self.ifStates[-1] == 1:
 317         self.do_if(args, replace=True)
 318     else:
 319       self.do_else(None, self.ifStates[-1])
 320   def do_elifdef(self, args):
 321     if self.disableLevel == 1:
 322       if self.ifStates[-1] == 1:
 323         self.do_ifdef(args, replace=True)
 324     else:
 325       self.do_else(None, self.ifStates[-1])
 326   def do_elifndef(self, args):
 327     if self.disableLevel == 1:
 328       if self.ifStates[-1] == 1:
 329         self.do_ifndef(args, replace=True)
 330     else:
 331       self.do_else(None, self.ifStates[-1])
 332   def do_endif(self, args):
 333     if self.disableLevel > 0:
 334       self.disableLevel -= 1
 335     if self.disableLevel == 0:
 336       self.ifStates.pop()
 337   # output processing
 338   def do_expand(self, args):
 339     lst = re.split('__(\w+)__', args, re.U)
 340     do_replace = False
 341     def vsubst(v):
 342       if v in self.context:
 343         return str(self.context[v])
 344       return ''
 345     for i in range(1, len(lst), 2):
 346       lst[i] = vsubst(lst[i])
 347     lst.append('\n') # add back the newline
 348     self.write(reduce(lambda x, y: x+y, lst, ''))
 349   def do_literal(self, args):
 350     self.write(args)
 351   def do_filter(self, args):
 352     filters = [f for f in args.split(' ') if hasattr(self, 'filter_' + f)]
 353     if len(filters) == 0:
 354       return
 355     current = dict(self.filters)
 356     for f in filters:
 357       current[f] = getattr(self, 'filter_' + f)
 358     filterNames = current.keys()
 359     filterNames.sort()
 360     self.filters = [(fn, current[fn]) for fn in filterNames]
 361     return
 362   def do_unfilter(self, args):
 363     filters = args.split(' ')
 364     current = dict(self.filters)
 365     for f in filters:
 366       if f in current:
 367         del current[f]
 368     filterNames = current.keys()
 369     filterNames.sort()
 370     self.filters = [(fn, current[fn]) for fn in filterNames]
 371     return
 372   # Filters
 373   #
 374   # emptyLines
 375   #   Strips blank lines from the output.
 376   def filter_emptyLines(self, aLine):
 377     if aLine == '\n':
 378       return ''
 379     return aLine
 380   # slashslash
 381   #   Strips everything after //
 382   def filter_slashslash(self, aLine):
 383     [aLine, rest] = aLine.split('//', 1)
 384     if rest:
 385       aLine += '\n'
 386     return aLine
 387   # spaces
 388   #   Collapses sequences of spaces into a single space
 389   def filter_spaces(self, aLine):
 390     return re.sub(' +', ' ', aLine).strip(' ')
 391   # substition
 392   #   helper to be used by both substition and attemptSubstitution
 393   def filter_substitution(self, aLine, fatal=True):
 394     def repl(matchobj):
 395       varname = matchobj.group('VAR')
 396       if varname in self.context:
 397         return str(self.context[varname])
 398       if fatal:
 399         raise Preprocessor.Error(self, 'UNDEFINED_VAR', varname)
 400       return ''
 401     return self.varsubst.sub(repl, aLine)
 402   def filter_attemptSubstitution(self, aLine):
 403     return self.filter_substitution(aLine, fatal=False)
 404   # File ops
 405   def do_include(self, args):
 406     """
 407     Preprocess a given file.
 408     args can either be a file name, or a file-like object.
 409     Files should be opened, and will be closed after processing.
 410     """
 411     isName = type(args) == str or type(args) == unicode
 412     oldWrittenLines = self.writtenLines
 413     oldCheckLineNumbers = self.checkLineNumbers
 414     self.checkLineNumbers = False
 415     if isName:
 416       try:
 417         args = str(args)
 418         if not os.path.isabs(args):
 419           args = os.path.join(self.context['DIRECTORY'], args)
 420         args = open(args)
 421       except:
 422         raise Preprocessor.Error(self, 'FILE_NOT_FOUND', str(args))
 423     self.checkLineNumbers = bool(re.search('\.js(?:\.in)?$', args.name))
 424     oldFile = self.context['FILE']
 425     oldLine = self.context['LINE']
 426     oldDir = self.context['DIRECTORY']
 427     if args.isatty():
 428       # we're stdin, use '-' and '' for file and dir
 429       self.context['FILE'] = '-'
 430       self.context['DIRECTORY'] = ''
 431     else:
 432       abspath = os.path.abspath(args.name)
 433       self.context['FILE'] = abspath
 434       self.context['DIRECTORY'] = os.path.dirname(abspath)
 435     self.context['LINE'] = 0
 436     self.writtenLines = 0
 437     for l in args:
 438       self.context['LINE'] += 1
 439       self.handleLine(l)
 440     args.close()
 441     self.context['FILE'] = oldFile
 442     self.checkLineNumbers = oldCheckLineNumbers
 443     self.writtenLines = oldWrittenLines
 444     self.context['LINE'] = oldLine
 445     self.context['DIRECTORY'] = oldDir
 446   def do_includesubst(self, args):
 447     args = self.filter_substitution(args)
 448     self.do_include(args)
 449   def do_error(self, args):
 450     raise Preprocessor.Error(self, 'Error: ', str(args))
 451
 452 def main():
 453   pp = Preprocessor()
 454   pp.handleCommandLine(None, True)
 455   return
 456
 457 def preprocess(includes=[sys.stdin], defines={},
 458                output = sys.stdout,
 459                line_endings='\n', marker='#'):
 460   pp = Preprocessor()
 461   pp.context.update(defines)
 462   pp.setLineEndings(line_endings)
 463   pp.setMarker(marker)
 464   pp.out = output
 465   for f in includes:
 466     pp.do_include(f)
 467
 468 if __name__ == "__main__":
 469   main()