config/Preprocessor.py

   1 """
   2 This is a very primitive line based preprocessor, for times when using
   3 a C preprocessor isn't an option.
   4 """
   5
   6 # ***** BEGIN LICENSE BLOCK *****
   7 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   8 #
   9 # The contents of this file are subject to the Mozilla Public License Version
  10 # 1.1 (the "License"); you may not use this file except in compliance with
  11 # the License. You may obtain a copy of the License at
  12 # http://www.mozilla.org/MPL/
  13 #
  14 # Software distributed under the License is distributed on an "AS IS" basis,
  15 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  16 # for the specific language governing rights and limitations under the
  17 # License.
  18 #
  19 # The Original Code is Mozilla build system.
  20 #
  21 # The Initial Developer of the Original Code is
  22 # Mozilla Foundation.
  23 # Portions created by the Initial Developer are Copyright (C) 2007
  24 # the Initial Developer. All Rights Reserved.
  25 #
  26 # Contributor(s):
  27 #  Axel Hecht <axel@pike.org>
  28 #
  29 # Alternatively, the contents of this file may be used under the terms of
  30 # either the GNU General Public License Version 2 or later (the "GPL"), or
  31 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  32 # in which case the provisions of the GPL or the LGPL are applicable instead
  33 # of those above. If you wish to allow use of your version of this file only
  34 # under the terms of either the GPL or the LGPL, and not to allow others to
  35 # use your version of this file under the terms of the MPL, indicate your
  36 # decision by deleting the provisions above and replace them with the notice
  37 # and other provisions required by the GPL or the LGPL. If you do not delete
  38 # the provisions above, a recipient may use your version of this file under
  39 # the terms of any one of the MPL, the GPL or the LGPL.
  40 #
  41 # ***** END LICENSE BLOCK *****
  42
  43 import sys
  44 import os
  45 import os.path
  46 import re
  47 from optparse import OptionParser
  48
  49 # hack around win32 mangling our line endings
  50 # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65443
  51 if sys.platform == "win32":
  52   import msvcrt
  53   msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
  54   os.linesep = '\n'
  55
  56 import Expression
  57
  58 __all__ = ['Preprocessor', 'preprocess']
  59
  60
  61 class Preprocessor:
  62   """
  63   Class for preprocessing text files.
  64   """
  65   class Error(RuntimeError):
  66     def __init__(self, cpp, MSG, context):
  67       self.file = cpp.context['FILE']
  68       self.line = cpp.context['LINE']
  69       self.key = MSG
  70       RuntimeError.__init__(self, (self.file, self.line, self.key, context))
  71   def __init__(self):
  72     self.context = Expression.Context()
  73     for k,v in {'FILE': '',
  74                 'LINE': 0,
  75                 'DIRECTORY': os.path.abspath('.')}.iteritems():
  76       self.context[k] = v
  77     self.disableLevel = 0
  78     # ifStates can be
  79     #  0: hadTrue
  80     #  1: wantsTrue
  81     #  2: #else found
  82     self.ifStates = []
  83     self.checkLineNumbers = False
  84     self.writtenLines = 0
  85     self.filters = []
  86     self.cmds = {}
  87     for cmd, level in {'define': 0,
  88                        'undef': 0,
  89                        'if': sys.maxint,
  90                        'ifdef': sys.maxint,
  91                        'ifndef': sys.maxint,
  92                        'else': 1,
  93                        'elif': 1,
  94                        'elifdef': 1,
  95                        'elifndef': 1,
  96                        'endif': sys.maxint,
  97                        'expand': 0,
  98                        'literal': 0,
  99                        'filter': 0,
 100                        'unfilter': 0,
 101                        'include': 0,
 102                        'includesubst': 0,
 103                        'error': 0}.iteritems():
 104       self.cmds[cmd] = (level, getattr(self, 'do_' + cmd))
 105     self.out = sys.stdout
 106     self.setMarker('#')
 107     self.LE = '\n'
 108     self.varsubst = re.compile('@(?P<VAR>\w+)@', re.U)
 109
 110   def setLineEndings(self, aLE):
 111     """
 112     Set the line endings to be used for output.
 113     """
 114     self.LE = {'cr': '\x0D', 'lf': '\x0A', 'crlf': '\x0D\x0A'}[aLE]
 115
 116   def setMarker(self, aMarker):
 117     """
 118     Set the marker to be used for processing directives.
 119     Used for handling CSS files, with pp.setMarker('%'), for example.
 120     """
 121     self.marker = aMarker
 122     self.instruction = re.compile('%s(?P<cmd>[a-z]+)(?:\s(?P<args>.*))?$'%aMarker, re.U)
 123     self.comment = re.compile(aMarker, re.U)
 124
 125   def clone(self):
 126     """
 127     Create a clone of the current processor, including line ending
 128     settings, marker, variable definitions, output stream.
 129     """
 130     rv = Preprocessor()
 131     rv.context.update(self.context)
 132     rv.setMarker(self.marker)
 133     rv.LE = self.LE
 134     rv.out = self.out
 135     return rv
 136
 137   def write(self, aLine):
 138     """
 139     Internal method for handling output.
 140     """
 141     if self.checkLineNumbers:
 142       self.writtenLines += 1
 143       ln = self.context['LINE']
 144       if self.writtenLines != ln:
 145         self.out.write('//@line %(line)d "%(file)s"%(le)s'%{'line': ln,
 146                                                             'file': self.context['FILE'],
 147                                                             'le': self.LE})
 148         self.writtenLines = ln
 149     for f in self.filters:
 150       aLine = f[1](aLine)
 151     # ensure our line ending. Only need to handle \n, as we're reading
 152     # with universal line ending support, at least for files.
 153     aLine = re.sub('\n', self.LE, aLine)
 154     self.out.write(aLine)
 155
 156   def handleCommandLine(self, args, defaultToStdin = False):
 157     """
 158     Parse a commandline into this parser.
 159     Uses OptionParser internally, no args mean sys.argv[1:].
 160     """
 161     p = self.getCommandLineParser()
 162     (options, args) = p.parse_args(args=args)
 163     includes = options.I
 164     if defaultToStdin and len(args) == 0:
 165       args = [sys.stdin]
 166     includes.extend(args)
 167     for f in includes:
 168       self.do_include(f)
 169     pass
 170
 171   def getCommandLineParser(self, unescapeDefines = False):
 172     escapedValue = re.compile('".*"$')
 173     numberValue = re.compile('\d+$')
 174     def handleE(option, opt, value, parser):
 175       for k,v in os.environ.iteritems():
 176         self.context[k] = v
 177     def handleD(option, opt, value, parser):
 178       vals = value.split('=', 1)
 179       if len(vals) == 1:
 180         vals.append(1)
 181       elif unescapeDefines and escapedValue.match(vals[1]):
 182         # strip escaped string values
 183         vals[1] = vals[1][1:-1]
 184       elif numberValue.match(vals[1]):
 185         vals[1] = int(vals[1])
 186       self.context[vals[0]] = vals[1]
 187     def handleU(option, opt, value, parser):
 188       del self.context[value]
 189     def handleF(option, opt, value, parser):
 190       self.do_filter(value)
 191     def handleLE(option, opt, value, parser):
 192       self.setLineEndings(value)
 193     def handleMarker(option, opt, value, parser):
 194       self.setMarker(value)
 195     p = OptionParser()
 196     p.add_option('-I', action='append', type="string", default = [],
 197                  metavar="FILENAME", help='Include file')
 198     p.add_option('-E', action='callback', callback=handleE,
 199                  help='Import the environment into the defined variables')
 200     p.add_option('-D', action='callback', callback=handleD, type="string",
 201                  metavar="VAR[=VAL]", help='Define a variable')
 202     p.add_option('-U', action='callback', callback=handleU, type="string",
 203                  metavar="VAR", help='Undefine a variable')
 204     p.add_option('-F', action='callback', callback=handleF, type="string",
 205                  metavar="FILTER", help='Enable the specified filter')
 206     p.add_option('--line-endings', action='callback', callback=handleLE,
 207                  type="string", metavar="[cr|lr|crlf]",
 208                  help='Use the specified line endings [Default: OS dependent]')
 209     p.add_option('--marker', action='callback', callback=handleMarker,
 210                  type="string",
 211                  help='Use the specified marker instead of #')
 212     return p
 213
 214   def handleLine(self, aLine):
 215     """
 216     Handle a single line of input (internal).
 217     """
 218     m = self.instruction.match(aLine)
 219     if m:
 220       args = None
 221       cmd = m.group('cmd')
 222       try:
 223         args = m.group('args')
 224       except IndexError:
 225         pass
 226       if cmd not in self.cmds:
 227         raise Preprocessor.Error(self, 'INVALID_CMD', aLine)
 228       level, cmd = self.cmds[cmd]
 229       if (level >= self.disableLevel):
 230         cmd(args)
 231     elif self.disableLevel == 0 and not self.comment.match(aLine):
 232       self.write(aLine)
 233     pass
 234
 235   # Instruction handlers
 236   # These are named do_'instruction name' and take one argument
 237
 238   # Variables
 239   def do_define(self, args):
 240     m = re.match('(?P<name>\w+)(?:\s(?P<value>.*))?', args, re.U)
 241     if not m:
 242       raise Preprocessor.Error(self, 'SYNTAX_DEF', args)
 243     val = 1
 244     if m.group('value'):
 245       val = m.group('value')
 246       try:
 247         val = int(val)
 248       except:
 249         pass
 250     self.context[m.group('name')] = val
 251   def do_undef(self, args):
 252     m = re.match('(?P<name>\w+)$', args, re.U)
 253     if not m:
 254       raise Preprocessor.Error(self, 'SYNTAX_DEF', args)
 255     if args in self.context:
 256       del self.context[args]
 257   # Logic
 258   def ensure_not_else(self):
 259     if len(self.ifStates) == 0 or self.ifStates[-1] == 2:
 260       sys.stderr.write('WARNING: bad nesting of #else\n')
 261   def do_if(self, args, replace=False):
 262     if self.disableLevel and not replace:
 263       self.disableLevel += 1
 264       return
 265     val = None
 266     try:
 267       e = Expression.Expression(args)
 268       val = e.evaluate(self.context)
 269     except Exception:
 270       # XXX do real error reporting
 271       raise Preprocessor.Error(self, 'SYNTAX_ERR', args)
 272     if type(val) == str:
 273       # we're looking for a number value, strings are false
 274       val = False
 275     if not val:
 276       self.disableLevel = 1
 277     if replace:
 278       if val:
 279         self.disableLevel = 0
 280       self.ifStates[-1] = self.disableLevel
 281     else:
 282       self.ifStates.append(self.disableLevel)
 283     pass
 284   def do_ifdef(self, args, replace=False):
 285     if self.disableLevel and not replace:
 286       self.disableLevel += 1
 287       return
 288     if re.match('\W', args, re.U):
 289       raise Preprocessor.Error(self, 'INVALID_VAR', args)
 290     if args not in self.context:
 291       self.disableLevel = 1
 292     if replace:
 293       if args in self.context:
 294         self.disableLevel = 0
 295       self.ifStates[-1] = self.disableLevel
 296     else:
 297       self.ifStates.append(self.disableLevel)
 298     pass
 299   def do_ifndef(self, args, replace=False):
 300     if self.disableLevel and not replace:
 301       self.disableLevel += 1
 302       return
 303     if re.match('\W', args, re.U):
 304       raise Preprocessor.Error(self, 'INVALID_VAR', args)
 305     if args in self.context:
 306       self.disableLevel = 1
 307     if replace:
 308       if args not in self.context:
 309         self.disableLevel = 0
 310       self.ifStates[-1] = self.disableLevel
 311     else:
 312       self.ifStates.append(self.disableLevel)
 313     pass
 314   def do_else(self, args, ifState = 2):
 315     self.ensure_not_else()
 316     hadTrue = self.ifStates[-1] == 0
 317     self.ifStates[-1] = ifState # in-else
 318     if hadTrue:
 319       self.disableLevel = 1
 320       return
 321     self.disableLevel = 0
 322   def do_elif(self, args):
 323     if self.disableLevel == 1:
 324       if self.ifStates[-1] == 1:
 325         self.do_if(args, replace=True)
 326     else:
 327       self.do_else(None, self.ifStates[-1])
 328   def do_elifdef(self, args):
 329     if self.disableLevel == 1:
 330       if self.ifStates[-1] == 1:
 331         self.do_ifdef(args, replace=True)
 332     else:
 333       self.do_else(None, self.ifStates[-1])
 334   def do_elifndef(self, args):
 335     if self.disableLevel == 1:
 336       if self.ifStates[-1] == 1:
 337         self.do_ifndef(args, replace=True)
 338     else:
 339       self.do_else(None, self.ifStates[-1])
 340   def do_endif(self, args):
 341     if self.disableLevel > 0:
 342       self.disableLevel -= 1
 343     if self.disableLevel == 0:
 344       self.ifStates.pop()
 345   # output processing
 346   def do_expand(self, args):
 347     lst = re.split('__(\w+)__', args, re.U)
 348     do_replace = False
 349     def vsubst(v):
 350       if v in self.context:
 351         return str(self.context[v])
 352       return ''
 353     for i in range(1, len(lst), 2):
 354       lst[i] = vsubst(lst[i])
 355     lst.append('\n') # add back the newline
 356     self.write(reduce(lambda x, y: x+y, lst, ''))
 357   def do_literal(self, args):
 358     self.write(args + self.LE)
 359   def do_filter(self, args):
 360     filters = [f for f in args.split(' ') if hasattr(self, 'filter_' + f)]
 361     if len(filters) == 0:
 362       return
 363     current = dict(self.filters)
 364     for f in filters:
 365       current[f] = getattr(self, 'filter_' + f)
 366     filterNames = current.keys()
 367     filterNames.sort()
 368     self.filters = [(fn, current[fn]) for fn in filterNames]
 369     return
 370   def do_unfilter(self, args):
 371     filters = args.split(' ')
 372     current = dict(self.filters)
 373     for f in filters:
 374       if f in current:
 375         del current[f]
 376     filterNames = current.keys()
 377     filterNames.sort()
 378     self.filters = [(fn, current[fn]) for fn in filterNames]
 379     return
 380   # Filters
 381   #
 382   # emptyLines
 383   #   Strips blank lines from the output.
 384   def filter_emptyLines(self, aLine):
 385     if aLine == '\n':
 386       return ''
 387     return aLine
 388   # slashslash
 389   #   Strips everything after //
 390   def filter_slashslash(self, aLine):
 391     [aLine, rest] = aLine.split('//', 1)
 392     if rest:
 393       aLine += '\n'
 394     return aLine
 395   # spaces
 396   #   Collapses sequences of spaces into a single space
 397   def filter_spaces(self, aLine):
 398     return re.sub(' +', ' ', aLine).strip(' ')
 399   # substition
 400   #   helper to be used by both substition and attemptSubstitution
 401   def filter_substitution(self, aLine, fatal=True):
 402     def repl(matchobj):
 403       varname = matchobj.group('VAR')
 404       if varname in self.context:
 405         return str(self.context[varname])
 406       if fatal:
 407         raise Preprocessor.Error(self, 'UNDEFINED_VAR', varname)
 408       return ''
 409     return self.varsubst.sub(repl, aLine)
 410   def filter_attemptSubstitution(self, aLine):
 411     return self.filter_substitution(aLine, fatal=False)
 412   # File ops
 413   def do_include(self, args):
 414     """
 415     Preprocess a given file.
 416     args can either be a file name, or a file-like object.
 417     Files should be opened, and will be closed after processing.
 418     """
 419     isName = type(args) == str or type(args) == unicode
 420     oldWrittenLines = self.writtenLines
 421     oldCheckLineNumbers = self.checkLineNumbers
 422     self.checkLineNumbers = False
 423     if isName:
 424       try:
 425         args = str(args)
 426         if not os.path.isabs(args):
 427           args = os.path.join(self.context['DIRECTORY'], args)
 428         args = open(args, 'rU')
 429       except:
 430         raise Preprocessor.Error(self, 'FILE_NOT_FOUND', str(args))
 431     self.checkLineNumbers = bool(re.search('\.js(?:\.in)?$', args.name))
 432     oldFile = self.context['FILE']
 433     oldLine = self.context['LINE']
 434     oldDir = self.context['DIRECTORY']
 435     if args.isatty():
 436       # we're stdin, use '-' and '' for file and dir
 437       self.context['FILE'] = '-'
 438       self.context['DIRECTORY'] = ''
 439     else:
 440       abspath = os.path.abspath(args.name)
 441       self.context['FILE'] = abspath
 442       self.context['DIRECTORY'] = os.path.dirname(abspath)
 443     self.context['LINE'] = 0
 444     self.writtenLines = 0
 445     for l in args:
 446       self.context['LINE'] += 1
 447       self.handleLine(l)
 448     args.close()
 449     self.context['FILE'] = oldFile
 450     self.checkLineNumbers = oldCheckLineNumbers
 451     self.writtenLines = oldWrittenLines
 452     self.context['LINE'] = oldLine
 453     self.context['DIRECTORY'] = oldDir
 454   def do_includesubst(self, args):
 455     args = self.filter_substitution(args)
 456     self.do_include(args)
 457   def do_error(self, args):
 458     raise Preprocessor.Error(self, 'Error: ', str(args))
 459
 460 def main():
 461   pp = Preprocessor()
 462   pp.handleCommandLine(None, True)
 463   return
 464
 465 def preprocess(includes=[sys.stdin], defines={},
 466                output = sys.stdout,
 467                line_endings='\n', marker='#'):
 468   pp = Preprocessor()
 469   pp.context.update(defines)
 470   pp.setLineEndings(line_endings)
 471   pp.setMarker(marker)
 472   pp.out = output
 473   for f in includes:
 474     pp.do_include(f)
 475
 476 if __name__ == "__main__":
 477   main()