2 This is a very primitive line based preprocessor, for times when using
3 a C preprocessor isn't an option.
6 # ***** BEGIN LICENSE BLOCK *****
7 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
9 # The contents of this file are subject to the Mozilla Public License Version
10 # 1.1 (the "License"); you may not use this file except in compliance with
11 # the License. You may obtain a copy of the License at
12 # http://www.mozilla.org/MPL/
14 # Software distributed under the License is distributed on an "AS IS" basis,
15 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
16 # for the specific language governing rights and limitations under the
19 # The Original Code is Mozilla build system.
21 # The Initial Developer of the Original Code is
23 # Portions created by the Initial Developer are Copyright (C) 2007
24 # the Initial Developer. All Rights Reserved.
27 # Axel Hecht <axel@pike.org>
29 # Alternatively, the contents of this file may be used under the terms of
30 # either the GNU General Public License Version 2 or later (the "GPL"), or
31 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
32 # in which case the provisions of the GPL or the LGPL are applicable instead
33 # of those above. If you wish to allow use of your version of this file only
34 # under the terms of either the GPL or the LGPL, and not to allow others to
35 # use your version of this file under the terms of the MPL, indicate your
36 # decision by deleting the provisions above and replace them with the notice
37 # and other provisions required by the GPL or the LGPL. If you do not delete
38 # the provisions above, a recipient may use your version of this file under
39 # the terms of any one of the MPL, the GPL or the LGPL.
41 # ***** END LICENSE BLOCK *****
47 from optparse
import OptionParser
49 # hack around win32 mangling our line endings
50 # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65443
51 if sys
.platform
== "win32":
53 msvcrt
.setmode(sys
.stdout
.fileno(), os
.O_BINARY
)
57 __all__
= ['Preprocessor', 'preprocess']
62 Class for preprocessing text files.
64 class Error(RuntimeError):
65 def __init__(self
, cpp
, MSG
, context
):
66 self
.file = cpp
.context
['FILE']
67 self
.line
= cpp
.context
['LINE']
69 RuntimeError.__init
__(self
, (self
.file, self
.line
, self
.key
, context
))
71 self
.context
= Expression
.Context()
72 for k
,v
in {'FILE': '',
74 'DIRECTORY': os
.path
.abspath('.')}.iteritems():
82 self
.checkLineNumbers
= False
86 for cmd
, level
in {'define': 0,
102 'error': 0}.iteritems():
103 self
.cmds
[cmd
] = (level
, getattr(self
, 'do_' + cmd
))
104 self
.out
= sys
.stdout
107 self
.varsubst
= re
.compile('@(?P<VAR>\w+)@', re
.U
)
109 def setLineEndings(self
, aLE
):
111 Set the line endings to be used for output.
113 self
.LE
= {'cr': '\x0D', 'lf': '\x0A', 'crlf': '\x0D\x0A'}[aLE
]
115 def setMarker(self
, aMarker
):
117 Set the marker to be used for processing directives.
118 Used for handling CSS files, with pp.setMarker('%'), for example.
120 self
.marker
= aMarker
121 self
.instruction
= re
.compile('%s(?P<cmd>[a-z]+)(?:\s(?P<args>.*))?$'%aMarker
, re
.U
)
122 self
.comment
= re
.compile(aMarker
, re
.U
)
126 Create a clone of the current processor, including line ending
127 settings, marker, variable definitions, output stream.
130 rv
.context
.update(self
.context
)
131 rv
.setMarker(self
.marker
)
136 def write(self
, aLine
):
138 Internal method for handling output.
140 if self
.checkLineNumbers
:
141 self
.writtenLines
+= 1
142 ln
= self
.context
['LINE']
143 if self
.writtenLines
!= ln
:
144 self
.out
.write('//@line %(line)d "%(file)s"%(le)s'%{'line': ln
,
145 'file': self
.context
['FILE'],
147 self
.writtenLines
= ln
148 for f
in self
.filters
:
150 aLine
= aLine
.rstrip('\r\n') + self
.LE
151 self
.out
.write(aLine
)
153 def handleCommandLine(self
, args
, defaultToStdin
= False):
155 Parse a commandline into this parser.
156 Uses OptionParser internally, no args mean sys.argv[1:].
159 def handleI(option
, opt
, value
, parser
):
160 includes
.append(value
)
161 def handleE(option
, opt
, value
, parser
):
162 for k
,v
in os
.environ
.iteritems():
164 def handleD(option
, opt
, value
, parser
):
165 vals
= value
.split('=')
169 self
.context
[vals
[0]] = vals
[1]
170 def handleU(option
, opt
, value
, parser
):
171 del self
.context
[value
]
172 def handleF(option
, opt
, value
, parser
):
173 self
.do_filter(value
)
174 def handleLE(option
, opt
, value
, parser
):
175 self
.setLineEndings(value
)
176 def handleMarker(option
, opt
, value
, parser
):
177 self
.setMarker(value
)
179 p
.add_option('-I', action
='callback', callback
=handleI
, type="string",
180 metavar
="FILENAME", help='Include file')
181 p
.add_option('-E', action
='callback', callback
=handleE
,
182 help='Import the environment into the defined variables')
183 p
.add_option('-D', action
='callback', callback
=handleD
, type="string",
184 metavar
="VAR[=VAL]", help='Define a variable')
185 p
.add_option('-U', action
='callback', callback
=handleU
, type="string",
186 metavar
="VAR", help='Undefine a variable')
187 p
.add_option('-F', action
='callback', callback
=handleF
, type="string",
188 metavar
="FILTER", help='Enabble the specified filter')
189 p
.add_option('--line-endings', action
='callback', callback
=handleLE
,
190 type="string", metavar
="[cr|lr|crlf]",
191 help='Use the specified line endings [Default: OS dependent]')
192 p
.add_option('--marker', action
='callback', callback
=handleMarker
,
194 help='Use the specified marker instead of #')
195 (options
, args
) = p
.parse_args(args
=args
)
196 if defaultToStdin
and len(args
) == 0:
198 includes
.extend(args
)
203 def handleLine(self
, aLine
):
205 Handle a single line of input (internal).
207 m
= self
.instruction
.match(aLine
)
212 args
= m
.group('args')
215 if cmd
not in self
.cmds
:
216 raise Preprocessor
.Error(self
, 'INVALID_CMD', aLine
)
217 level
, cmd
= self
.cmds
[cmd
]
218 if (level
>= self
.disableLevel
):
220 elif self
.disableLevel
== 0 and not self
.comment
.match(aLine
):
224 # Instruction handlers
225 # These are named do_'instruction name' and take one argument
228 def do_define(self
, args
):
229 m
= re
.match('(?P<name>\w+)(?:\s(?P<value>.*))?', args
, re
.U
)
231 raise Preprocessor
.Error(self
, 'SYNTAX_DEF', args
)
234 val
= m
.group('value')
242 self
.context
[m
.group('name')] = val
243 def do_undef(self
, args
):
244 m
= re
.match('(?P<name>\w+)$', args
, re
.U
)
246 raise Preprocessor
.Error(self
, 'SYNTAX_DEF', args
)
247 if args
in self
.context
:
248 del self
.context
[args
]
250 def ensure_not_else(self
):
251 if len(self
.ifStates
) == 0 or self
.ifStates
[-1] == 2:
252 sys
.stderr
.write('WARNING: bad nesting of #else\n')
253 def do_if(self
, args
, replace
=False):
254 if self
.disableLevel
and not replace
:
255 self
.disableLevel
+= 1
259 e
= Expression
.Expression(args
)
260 val
= e
.evaluate(self
.context
)
262 # XXX do real error reporting
263 raise Preprocessor
.Error(self
, 'SYNTAX_ERR', args
)
265 # we're looking for a number value, strings are false
268 self
.disableLevel
= 1
271 self
.disableLevel
= 0
272 self
.ifStates
[-1] = self
.disableLevel
274 self
.ifStates
.append(self
.disableLevel
)
276 def do_ifdef(self
, args
, replace
=False):
277 if self
.disableLevel
and not replace
:
278 self
.disableLevel
+= 1
280 if re
.match('\W', args
, re
.U
):
281 raise Preprocessor
.Error(self
, 'INVALID_VAR', args
)
282 if args
not in self
.context
:
283 self
.disableLevel
= 1
285 if args
in self
.context
:
286 self
.disableLevel
= 0
287 self
.ifStates
[-1] = self
.disableLevel
289 self
.ifStates
.append(self
.disableLevel
)
291 def do_ifndef(self
, args
, replace
=False):
292 if self
.disableLevel
and not replace
:
293 self
.disableLevel
+= 1
295 if re
.match('\W', args
, re
.U
):
296 raise Preprocessor
.Error(self
, 'INVALID_VAR', args
)
297 if args
in self
.context
:
298 self
.disableLevel
= 1
300 if args
not in self
.context
:
301 self
.disableLevel
= 0
302 self
.ifStates
[-1] = self
.disableLevel
304 self
.ifStates
.append(self
.disableLevel
)
306 def do_else(self
, args
, ifState
= 2):
307 self
.ensure_not_else()
308 hadTrue
= self
.ifStates
[-1] == 0
309 self
.ifStates
[-1] = ifState
# in-else
311 self
.disableLevel
= 1
313 self
.disableLevel
= 0
314 def do_elif(self
, args
):
315 if self
.disableLevel
== 1:
316 if self
.ifStates
[-1] == 1:
317 self
.do_if(args
, replace
=True)
319 self
.do_else(None, self
.ifStates
[-1])
320 def do_elifdef(self
, args
):
321 if self
.disableLevel
== 1:
322 if self
.ifStates
[-1] == 1:
323 self
.do_ifdef(args
, replace
=True)
325 self
.do_else(None, self
.ifStates
[-1])
326 def do_elifndef(self
, args
):
327 if self
.disableLevel
== 1:
328 if self
.ifStates
[-1] == 1:
329 self
.do_ifndef(args
, replace
=True)
331 self
.do_else(None, self
.ifStates
[-1])
332 def do_endif(self
, args
):
333 if self
.disableLevel
> 0:
334 self
.disableLevel
-= 1
335 if self
.disableLevel
== 0:
338 def do_expand(self
, args
):
339 lst
= re
.split('__(\w+)__', args
, re
.U
)
342 if v
in self
.context
:
343 return str(self
.context
[v
])
345 for i
in range(1, len(lst
), 2):
346 lst
[i
] = vsubst(lst
[i
])
347 lst
.append('\n') # add back the newline
348 self
.write(reduce(lambda x
, y
: x
+y
, lst
, ''))
349 def do_literal(self
, args
):
351 def do_filter(self
, args
):
352 filters
= [f
for f
in args
.split(' ') if hasattr(self
, 'filter_' + f
)]
353 if len(filters
) == 0:
355 current
= dict(self
.filters
)
357 current
[f
] = getattr(self
, 'filter_' + f
)
358 filterNames
= current
.keys()
360 self
.filters
= [(fn
, current
[fn
]) for fn
in filterNames
]
362 def do_unfilter(self
, args
):
363 filters
= args
.split(' ')
364 current
= dict(self
.filters
)
368 filterNames
= current
.keys()
370 self
.filters
= [(fn
, current
[fn
]) for fn
in filterNames
]
375 # Strips blank lines from the output.
376 def filter_emptyLines(self
, aLine
):
381 # Strips everything after //
382 def filter_slashslash(self
, aLine
):
383 [aLine
, rest
] = aLine
.split('//', 1)
388 # Collapses sequences of spaces into a single space
389 def filter_spaces(self
, aLine
):
390 return re
.sub(' +', ' ', aLine
).strip(' ')
392 # helper to be used by both substition and attemptSubstitution
393 def filter_substitution(self
, aLine
, fatal
=True):
395 varname
= matchobj
.group('VAR')
396 if varname
in self
.context
:
397 return str(self
.context
[varname
])
399 raise Preprocessor
.Error(self
, 'UNDEFINED_VAR', varname
)
401 return self
.varsubst
.sub(repl
, aLine
)
402 def filter_attemptSubstitution(self
, aLine
):
403 return self
.filter_substitution(aLine
, fatal
=False)
405 def do_include(self
, args
):
407 Preprocess a given file.
408 args can either be a file name, or a file-like object.
409 Files should be opened, and will be closed after processing.
411 isName
= type(args
) == str or type(args
) == unicode
412 oldWrittenLines
= self
.writtenLines
413 oldCheckLineNumbers
= self
.checkLineNumbers
414 self
.checkLineNumbers
= False
418 if not os
.path
.isabs(args
):
419 args
= os
.path
.join(self
.context
['DIRECTORY'], args
)
422 raise Preprocessor
.Error(self
, 'FILE_NOT_FOUND', str(args
))
423 self
.checkLineNumbers
= bool(re
.search('\.js(?:\.in)?$', args
.name
))
424 oldFile
= self
.context
['FILE']
425 oldLine
= self
.context
['LINE']
426 oldDir
= self
.context
['DIRECTORY']
428 # we're stdin, use '-' and '' for file and dir
429 self
.context
['FILE'] = '-'
430 self
.context
['DIRECTORY'] = ''
432 abspath
= os
.path
.abspath(args
.name
)
433 self
.context
['FILE'] = abspath
434 self
.context
['DIRECTORY'] = os
.path
.dirname(abspath
)
435 self
.context
['LINE'] = 0
436 self
.writtenLines
= 0
438 self
.context
['LINE'] += 1
441 self
.context
['FILE'] = oldFile
442 self
.checkLineNumbers
= oldCheckLineNumbers
443 self
.writtenLines
= oldWrittenLines
444 self
.context
['LINE'] = oldLine
445 self
.context
['DIRECTORY'] = oldDir
446 def do_includesubst(self
, args
):
447 args
= self
.filter_substitution(args
)
448 self
.do_include(args
)
449 def do_error(self
, args
):
450 raise Preprocessor
.Error(self
, 'Error: ', str(args
))
454 pp
.handleCommandLine(None, True)
457 def preprocess(includes
=[sys
.stdin
], defines
={},
459 line_endings
='\n', marker
='#'):
461 pp
.context
.update(defines
)
462 pp
.setLineEndings(line_endings
)
468 if __name__
== "__main__":