Using FS mtime to reload non recursive cache.
[pyTivo.git] / Cheetah / Parser.py
bloba72e7027fca62ea56548ca51396b705dc7d4ab75
1 #!/usr/bin/env python
2 # $Id: Parser.py,v 1.130 2006/06/21 23:49:14 tavis_rudd Exp $
3 """Parser classes for Cheetah's Compiler
5 Classes:
6 ParseError( Exception )
7 _LowLevelParser( Cheetah.SourceReader.SourceReader ), basically a lexer
8 _HighLevelParser( _LowLevelParser )
9 Parser === _HighLevelParser (an alias)
11 Meta-Data
12 ================================================================================
13 Author: Tavis Rudd <tavis@damnsimple.com>
14 Version: $Revision: 1.130 $
15 Start Date: 2001/08/01
16 Last Revision Date: $Date: 2006/06/21 23:49:14 $
17 """
18 __author__ = "Tavis Rudd <tavis@damnsimple.com>"
19 __revision__ = "$Revision: 1.130 $"[11:-2]
21 import os
22 import sys
23 import re
24 from re import DOTALL, MULTILINE
25 from types import StringType, ListType, TupleType, ClassType, TypeType
26 import time
27 from tokenize import pseudoprog
28 import inspect
29 import new
30 import traceback
32 from Cheetah.SourceReader import SourceReader
33 from Cheetah import Filters
34 from Cheetah import ErrorCatchers
35 from Cheetah.Unspecified import Unspecified
37 # re tools
38 def escapeRegexChars(txt,
39 escapeRE=re.compile(r'([\$\^\*\+\.\?\{\}\[\]\(\)\|\\])')):
41 """Return a txt with all special regular expressions chars escaped."""
43 return escapeRE.sub(r'\\\1' , txt)
45 def group(*choices): return '(' + '|'.join(choices) + ')'
46 def nongroup(*choices): return '(?:' + '|'.join(choices) + ')'
47 def namedGroup(name, *choices): return '(P:<' + name +'>' + '|'.join(choices) + ')'
48 def any(*choices): return apply(group, choices) + '*'
49 def maybe(*choices): return apply(group, choices) + '?'
51 ##################################################
52 ## CONSTANTS & GLOBALS ##
54 NO_CACHE = 0
55 STATIC_CACHE = 1
56 REFRESH_CACHE = 2
58 SET_LOCAL = 0
59 SET_GLOBAL = 1
60 SET_MODULE = 2
62 ##################################################
63 ## Tokens for the parser ##
65 #generic
66 identchars = "abcdefghijklmnopqrstuvwxyz" \
67 "ABCDEFGHIJKLMNOPQRSTUVWXYZ_"
68 namechars = identchars + "0123456789"
70 #operators
71 powerOp = '**'
72 unaryArithOps = ('+', '-', '~')
73 binaryArithOps = ('+', '-', '/', '//','%')
74 shiftOps = ('>>','<<')
75 bitwiseOps = ('&','|','^')
76 assignOp = '='
77 augAssignOps = ('+=','-=','/=','*=', '**=','^=','%=',
78 '>>=','<<=','&=','|=', )
79 assignmentOps = (assignOp,) + augAssignOps
81 compOps = ('<','>','==','!=','<=','>=', '<>', 'is', 'in',)
82 booleanOps = ('and','or','not')
83 operators = (powerOp,) + unaryArithOps + binaryArithOps \
84 + shiftOps + bitwiseOps + assignmentOps \
85 + compOps + booleanOps
87 delimeters = ('(',')','{','}','[',']',
88 ',','.',':',';','=','`') + augAssignOps
91 keywords = ('and', 'del', 'for', 'is', 'raise',
92 'assert', 'elif', 'from', 'lambda', 'return',
93 'break', 'else', 'global', 'not', 'try',
94 'class', 'except', 'if', 'or', 'while',
95 'continue', 'exec', 'import', 'pass',
96 'def', 'finally', 'in', 'print',
99 single3 = "'''"
100 double3 = '"""'
102 tripleQuotedStringStarts = ("'''", '"""',
103 "r'''", 'r"""', "R'''", 'R"""',
104 "u'''", 'u"""', "U'''", 'U"""',
105 "ur'''", 'ur"""', "Ur'''", 'Ur"""',
106 "uR'''", 'uR"""', "UR'''", 'UR"""')
108 tripleQuotedStringPairs = {"'''": single3, '"""': double3,
109 "r'''": single3, 'r"""': double3,
110 "u'''": single3, 'u"""': double3,
111 "ur'''": single3, 'ur"""': double3,
112 "R'''": single3, 'R"""': double3,
113 "U'''": single3, 'U"""': double3,
114 "uR'''": single3, 'uR"""': double3,
115 "Ur'''": single3, 'Ur"""': double3,
116 "UR'''": single3, 'UR"""': double3,
119 closurePairs= {')':'(',']':'[','}':'{'}
120 closurePairsRev= {'(':')','[':']','{':'}'}
122 ##################################################
123 ## Regex chunks for the parser ##
125 tripleQuotedStringREs = {}
126 def makeTripleQuoteRe(start, end):
127 start = escapeRegexChars(start)
128 end = escapeRegexChars(end)
129 return re.compile(r'(?:' + start + r').*?' + r'(?:' + end + r')', re.DOTALL)
131 for start, end in tripleQuotedStringPairs.items():
132 tripleQuotedStringREs[start] = makeTripleQuoteRe(start, end)
134 WS = r'[ \f\t]*'
135 EOL = r'\r\n|\n|\r'
136 EOLZ = EOL + r'|\Z'
137 escCharLookBehind = nongroup(r'(?<=\A)',r'(?<!\\)')
138 nameCharLookAhead = r'(?=[A-Za-z_])'
139 identRE=re.compile(r'[a-zA-Z_][a-zA-Z_0-9]*')
140 EOLre=re.compile(r'(?:\r\n|\r|\n)')
142 specialVarRE=re.compile(r'([a-zA-z_]+)@') # for matching specialVar comments
143 # e.g. ##author@ Tavis Rudd
145 directiveNamesAndParsers = {
146 # importing and inheritance
147 'import':None,
148 'from':None,
149 'extends': 'eatExtends',
150 'implements': 'eatImplements',
152 # output, filtering, and caching
153 'slurp': 'eatSlurp',
154 'raw': 'eatRaw',
155 'include': 'eatInclude',
156 'cache': 'eatCache',
157 'filter': 'eatFilter',
158 'echo': None,
159 'silent': None,
161 'call': 'eatCall',
162 'arg': 'eatCallArg',
164 'capture': 'eatCapture',
166 # declaration, assignment, and deletion
167 'attr': 'eatAttr',
168 'def': 'eatDef',
169 'block': 'eatBlock',
170 '@': 'eatDecorator',
171 'defmacro': 'eatDefMacro',
173 'closure': 'eatClosure',
175 'set': 'eatSet',
176 'del': None,
178 # flow control
179 'if': 'eatIf',
180 'while': None,
181 'for': None,
182 'else': None,
183 'elif': None,
184 'pass': None,
185 'break': None,
186 'continue': None,
187 'stop': None,
188 'return': None,
189 'yield': None,
191 # little wrappers
192 'repeat': None,
193 'unless': None,
195 # error handling
196 'assert': None,
197 'raise': None,
198 'try': None,
199 'except': None,
200 'finally': None,
201 'errorCatcher': 'eatErrorCatcher',
203 # intructions to the parser and compiler
204 'breakpoint': 'eatBreakPoint',
205 'compiler': 'eatCompiler',
206 'compiler-settings': 'eatCompilerSettings',
208 # misc
209 'shBang': 'eatShbang',
210 'encoding': 'eatEncoding',
212 'end': 'eatEndDirective',
215 endDirectiveNamesAndHandlers = {
216 'def': 'handleEndDef', # has short-form
217 'block': None, # has short-form
218 'closure': None, # has short-form
219 'cache': None, # has short-form
220 'call': None, # has short-form
221 'capture': None, # has short-form
222 'filter': None,
223 'errorCatcher':None,
224 'while': None, # has short-form
225 'for': None, # has short-form
226 'if': None, # has short-form
227 'try': None, # has short-form
228 'repeat': None, # has short-form
229 'unless': None, # has short-form
232 ##################################################
233 ## CLASSES ##
235 # @@TR: SyntaxError doesn't call exception.__str__ for some reason!
236 #class ParseError(SyntaxError):
237 class ParseError(ValueError):
238 def __init__(self, stream, msg='Invalid Syntax', extMsg='', lineno=None, col=None):
239 self.stream = stream
240 if stream.pos() >= len(stream):
241 stream.setPos(len(stream) -1)
242 self.msg = msg
243 self.extMsg = extMsg
244 self.lineno = lineno
245 self.col = col
247 def __str__(self):
248 return self.report()
250 def report(self):
251 stream = self.stream
252 if stream.filename():
253 f = " in file %s" % stream.filename()
254 else:
255 f = ''
256 report = ''
257 if self.lineno:
258 lineno = self.lineno
259 row, col, line = (lineno, (self.col or 0),
260 self.stream.splitlines()[lineno-1])
261 else:
262 row, col, line = self.stream.getRowColLine()
264 ## get the surrounding lines
265 lines = stream.splitlines()
266 prevLines = [] # (rowNum, content)
267 for i in range(1,4):
268 if row-1-i <=0:
269 break
270 prevLines.append( (row-i,lines[row-1-i]) )
272 nextLines = [] # (rowNum, content)
273 for i in range(1,4):
274 if not row-1+i < len(lines):
275 break
276 nextLines.append( (row+i,lines[row-1+i]) )
277 nextLines.reverse()
279 ## print the main message
280 report += "\n\n%s\n" %self.msg
281 report += "Line %i, column %i%s\n\n" % (row, col, f)
282 report += 'Line|Cheetah Code\n'
283 report += '----|-------------------------------------------------------------\n'
284 while prevLines:
285 lineInfo = prevLines.pop()
286 report += "%(row)-4d|%(line)s\n"% {'row':lineInfo[0], 'line':lineInfo[1]}
287 report += "%(row)-4d|%(line)s\n"% {'row':row, 'line':line}
288 report += ' '*5 +' '*(col-1) + "^\n"
290 while nextLines:
291 lineInfo = nextLines.pop()
292 report += "%(row)-4d|%(line)s\n"% {'row':lineInfo[0], 'line':lineInfo[1]}
293 ## add the extra msg
294 if self.extMsg:
295 report += self.extMsg + '\n'
297 return report
299 class ForbiddenSyntax(ParseError): pass
300 class ForbiddenExpression(ForbiddenSyntax): pass
301 class ForbiddenDirective(ForbiddenSyntax): pass
303 class CheetahVariable:
304 def __init__(self, nameChunks, useNameMapper=True, cacheToken=None,
305 rawSource=None):
306 self.nameChunks = nameChunks
307 self.useNameMapper = useNameMapper
308 self.cacheToken = cacheToken
309 self.rawSource = rawSource
311 class Placeholder(CheetahVariable): pass
313 class ArgList:
314 """Used by _LowLevelParser.getArgList()"""
316 def __init__(self):
317 self.argNames = []
318 self.defVals = []
319 self.i = 0
321 def addArgName(self, name):
322 self.argNames.append( name )
323 self.defVals.append( None )
325 def next(self):
326 self.i += 1
328 def addToDefVal(self, token):
329 i = self.i
330 if self.defVals[i] == None:
331 self.defVals[i] = ''
332 self.defVals[i] += token
334 def merge(self):
335 defVals = self.defVals
336 for i in range(len(defVals)):
337 if type(defVals[i]) == StringType:
338 defVals[i] = defVals[i].strip()
340 return map(None, [i.strip() for i in self.argNames], defVals)
342 def __str__(self):
343 return str(self.merge())
345 class _LowLevelParser(SourceReader):
346 """This class implements the methods to match or extract ('get*') the basic
347 elements of Cheetah's grammar. It does NOT handle any code generation or
348 state management.
351 _settingsManager = None
353 def setSettingsManager(self, settingsManager):
354 self._settingsManager = settingsManager
356 def setting(self, key, default=Unspecified):
357 if default is Unspecified:
358 return self._settingsManager.setting(key)
359 else:
360 return self._settingsManager.setting(key, default=default)
362 def setSetting(self, key, val):
363 self._settingsManager.setSetting(key, val)
365 def settings(self):
366 return self._settingsManager.settings()
368 def updateSettings(self, settings):
369 self._settingsManager.updateSettings(settings)
371 def _initializeSettings(self):
372 self._settingsManager._initializeSettings()
374 def configureParser(self):
375 """Is called by the Compiler instance after the parser has had a
376 settingsManager assigned with self.setSettingsManager()
378 self._makeCheetahVarREs()
379 self._makeCommentREs()
380 self._makeDirectiveREs()
381 self._makePspREs()
382 self._possibleNonStrConstantChars = (
383 self.setting('commentStartToken')[0] +
384 self.setting('multiLineCommentStartToken')[0] +
385 self.setting('cheetahVarStartToken')[0] +
386 self.setting('directiveStartToken')[0] +
387 self.setting('PSPStartToken')[0])
388 self._nonStrConstMatchers = [
389 self.matchCommentStartToken,
390 self.matchMultiLineCommentStartToken,
391 self.matchVariablePlaceholderStart,
392 self.matchExpressionPlaceholderStart,
393 self.matchDirective,
394 self.matchPSPStartToken,
395 self.matchEOLSlurpToken,
398 ## regex setup ##
400 def _makeCheetahVarREs(self):
402 """Setup the regexs for Cheetah $var parsing."""
404 num = r'[0-9\.]+'
405 interval = (r'(?P<interval>' +
406 num + r's|' +
407 num + r'm|' +
408 num + r'h|' +
409 num + r'd|' +
410 num + r'w|' +
411 num + ')'
414 cacheToken = (r'(?:' +
415 r'(?P<REFRESH_CACHE>\*' + interval + '\*)'+
416 '|' +
417 r'(?P<STATIC_CACHE>\*)' +
418 '|' +
419 r'(?P<NO_CACHE>)' +
420 ')')
421 self.cacheTokenRE = re.compile(cacheToken)
423 silentPlaceholderToken = (r'(?:' +
424 r'(?P<SILENT>' +escapeRegexChars('!')+')'+
425 '|' +
426 r'(?P<NOT_SILENT>)' +
427 ')')
428 self.silentPlaceholderTokenRE = re.compile(silentPlaceholderToken)
430 self.cheetahVarStartRE = re.compile(
431 escCharLookBehind +
432 r'(?P<startToken>'+escapeRegexChars(self.setting('cheetahVarStartToken'))+')'+
433 r'(?P<silenceToken>'+silentPlaceholderToken+')'+
434 r'(?P<cacheToken>'+cacheToken+')'+
435 r'(?P<enclosure>|(?:(?:\{|\(|\[)[ \t\f]*))' + # allow WS after enclosure
436 r'(?=[A-Za-z_])')
437 validCharsLookAhead = r'(?=[A-Za-z_\*!\{\(\[])'
438 self.cheetahVarStartToken = self.setting('cheetahVarStartToken')
439 self.cheetahVarStartTokenRE = re.compile(
440 escCharLookBehind +
441 escapeRegexChars(self.setting('cheetahVarStartToken'))
442 +validCharsLookAhead
445 self.cheetahVarInExpressionStartTokenRE = re.compile(
446 escapeRegexChars(self.setting('cheetahVarStartToken'))
447 +r'(?=[A-Za-z_])'
450 self.expressionPlaceholderStartRE = re.compile(
451 escCharLookBehind +
452 r'(?P<startToken>' + escapeRegexChars(self.setting('cheetahVarStartToken')) + ')' +
453 r'(?P<cacheToken>' + cacheToken + ')' +
454 #r'\[[ \t\f]*'
455 r'(?:\{|\(|\[)[ \t\f]*'
456 + r'(?=[^\)\}\]])'
459 if self.setting('EOLSlurpToken'):
460 self.EOLSlurpRE = re.compile(
461 escapeRegexChars(self.setting('EOLSlurpToken'))
462 + r'[ \t\f]*'
463 + r'(?:'+EOL+')'
465 else:
466 self.EOLSlurpRE = None
469 def _makeCommentREs(self):
470 """Construct the regex bits that are used in comment parsing."""
471 startTokenEsc = escapeRegexChars(self.setting('commentStartToken'))
472 self.commentStartTokenRE = re.compile(escCharLookBehind + startTokenEsc)
473 del startTokenEsc
475 startTokenEsc = escapeRegexChars(
476 self.setting('multiLineCommentStartToken'))
477 endTokenEsc = escapeRegexChars(
478 self.setting('multiLineCommentEndToken'))
479 self.multiLineCommentTokenStartRE = re.compile(escCharLookBehind +
480 startTokenEsc)
481 self.multiLineCommentEndTokenRE = re.compile(escCharLookBehind +
482 endTokenEsc)
484 def _makeDirectiveREs(self):
485 """Construct the regexs that are used in directive parsing."""
486 startToken = self.setting('directiveStartToken')
487 endToken = self.setting('directiveEndToken')
488 startTokenEsc = escapeRegexChars(startToken)
489 endTokenEsc = escapeRegexChars(endToken)
490 validSecondCharsLookAhead = r'(?=[A-Za-z_@])'
491 reParts = [escCharLookBehind, startTokenEsc]
492 if self.setting('allowWhitespaceAfterDirectiveStartToken'):
493 reParts.append('[ \t]*')
494 reParts.append(validSecondCharsLookAhead)
495 self.directiveStartTokenRE = re.compile(''.join(reParts))
496 self.directiveEndTokenRE = re.compile(escCharLookBehind + endTokenEsc)
498 def _makePspREs(self):
499 """Setup the regexs for PSP parsing."""
500 startToken = self.setting('PSPStartToken')
501 startTokenEsc = escapeRegexChars(startToken)
502 self.PSPStartTokenRE = re.compile(escCharLookBehind + startTokenEsc)
503 endToken = self.setting('PSPEndToken')
504 endTokenEsc = escapeRegexChars(endToken)
505 self.PSPEndTokenRE = re.compile(escCharLookBehind + endTokenEsc)
508 def isLineClearToStartToken(self, pos=None):
509 return self.isLineClearToPos(pos)
511 def matchTopLevelToken(self):
512 """Returns the first match found from the following methods:
513 self.matchCommentStartToken
514 self.matchMultiLineCommentStartToken
515 self.matchVariablePlaceholderStart
516 self.matchExpressionPlaceholderStart
517 self.matchDirective
518 self.matchPSPStartToken
519 self.matchEOLSlurpToken
521 Returns None if no match.
523 match = None
524 if self.peek() in self._possibleNonStrConstantChars:
525 for matcher in self._nonStrConstMatchers:
526 match = matcher()
527 if match:
528 break
529 return match
531 def matchPyToken(self):
532 match = pseudoprog.match(self.src(), self.pos())
534 if match and match.group() in tripleQuotedStringStarts:
535 TQSmatch = tripleQuotedStringREs[match.group()].match(self.src(), self.pos())
536 if TQSmatch:
537 return TQSmatch
538 return match
540 def getPyToken(self):
541 match = self.matchPyToken()
542 if match is None:
543 raise ParseError(self)
544 elif match.group() in tripleQuotedStringStarts:
545 raise ParseError(self, msg='Malformed triple-quoted string')
546 return self.readTo(match.end())
548 def matchEOLSlurpToken(self):
549 if self.EOLSlurpRE:
550 return self.EOLSlurpRE.match(self.src(), self.pos())
552 def getEOLSlurpToken(self):
553 match = self.matchEOLSlurpToken()
554 if not match:
555 raise ParseError(self, msg='Invalid EOL slurp token')
556 return self.readTo(match.end())
558 def matchCommentStartToken(self):
559 return self.commentStartTokenRE.match(self.src(), self.pos())
561 def getCommentStartToken(self):
562 match = self.matchCommentStartToken()
563 if not match:
564 raise ParseError(self, msg='Invalid single-line comment start token')
565 return self.readTo(match.end())
567 def matchMultiLineCommentStartToken(self):
568 return self.multiLineCommentTokenStartRE.match(self.src(), self.pos())
570 def getMultiLineCommentStartToken(self):
571 match = self.matchMultiLineCommentStartToken()
572 if not match:
573 raise ParseError(self, msg='Invalid multi-line comment start token')
574 return self.readTo(match.end())
576 def matchMultiLineCommentEndToken(self):
577 return self.multiLineCommentEndTokenRE.match(self.src(), self.pos())
579 def getMultiLineCommentEndToken(self):
580 match = self.matchMultiLineCommentEndToken()
581 if not match:
582 raise ParseError(self, msg='Invalid multi-line comment end token')
583 return self.readTo(match.end())
585 def getDottedName(self):
586 srcLen = len(self)
587 nameChunks = []
589 if not self.peek() in identchars:
590 raise ParseError(self)
592 while self.pos() < srcLen:
593 c = self.peek()
594 if c in namechars:
595 nameChunk = self.getIdentifier()
596 nameChunks.append(nameChunk)
597 elif c == '.':
598 if self.pos()+1 <srcLen and self.peek(1) in identchars:
599 nameChunks.append(self.getc())
600 else:
601 break
602 else:
603 break
605 return ''.join(nameChunks)
607 def matchIdentifier(self):
608 return identRE.match(self.src(), self.pos())
610 def getIdentifier(self):
611 match = self.matchIdentifier()
612 if not match:
613 raise ParseError(self, msg='Invalid identifier')
614 return self.readTo(match.end())
616 def matchOperator(self):
617 match = self.matchPyToken()
618 if match and match.group() not in operators:
619 match = None
620 return match
622 def getOperator(self):
623 match = self.matchOperator()
624 if not match:
625 raise ParseError(self, msg='Expected operator')
626 return self.readTo( match.end() )
628 def matchAssignmentOperator(self):
629 match = self.matchPyToken()
630 if match and match.group() not in assignmentOps:
631 match = None
632 return match
634 def getAssignmentOperator(self):
635 match = self.matchAssignmentOperator()
636 if not match:
637 raise ParseError(self, msg='Expected assignment operator')
638 return self.readTo( match.end() )
640 def matchDirective(self):
641 """Returns False or the name of the directive matched.
643 startPos = self.pos()
644 if not self.matchDirectiveStartToken():
645 return False
646 self.getDirectiveStartToken()
647 directiveName = self.matchDirectiveName()
648 self.setPos(startPos)
649 return directiveName
651 def matchDirectiveName(self, directiveNameChars=identchars+'0123456789-@'):
652 startPos = self.pos()
653 directives = self._directiveNamesAndParsers.keys()
654 possibleMatches = []
655 name = ''
656 while not self.atEnd():
657 c = self.getc()
658 if not c in directiveNameChars:
659 break
660 name += c
661 if name in directives:
662 possibleMatches.append(name)
664 possibleMatches.sort()
665 possibleMatches.reverse() # longest match first
667 directiveName = False
668 if possibleMatches:
669 directiveName = possibleMatches[0]
671 self.setPos(startPos)
672 return directiveName
674 def matchDirectiveStartToken(self):
675 return self.directiveStartTokenRE.match(self.src(), self.pos())
677 def getDirectiveStartToken(self):
678 match = self.matchDirectiveStartToken()
679 if not match:
680 raise ParseError(self, msg='Invalid directive start token')
681 return self.readTo(match.end())
683 def matchDirectiveEndToken(self):
684 return self.directiveEndTokenRE.match(self.src(), self.pos())
686 def getDirectiveEndToken(self):
687 match = self.matchDirectiveEndToken()
688 if not match:
689 raise ParseError(self, msg='Invalid directive end token')
690 return self.readTo(match.end())
693 def matchColonForSingleLineShortFormDirective(self):
694 if not self.atEnd() and self.peek()==':':
695 restOfLine = self[self.pos()+1:self.findEOL()]
696 restOfLine = restOfLine.strip()
697 if not restOfLine:
698 return False
699 elif self.commentStartTokenRE.match(restOfLine):
700 return False
701 else: # non-whitespace, non-commment chars found
702 return True
703 return False
705 def matchPSPStartToken(self):
706 return self.PSPStartTokenRE.match(self.src(), self.pos())
708 def matchPSPEndToken(self):
709 return self.PSPEndTokenRE.match(self.src(), self.pos())
711 def getPSPStartToken(self):
712 match = self.matchPSPStartToken()
713 if not match:
714 raise ParseError(self, msg='Invalid psp start token')
715 return self.readTo(match.end())
717 def getPSPEndToken(self):
718 match = self.matchPSPEndToken()
719 if not match:
720 raise ParseError(self, msg='Invalid psp end token')
721 return self.readTo(match.end())
723 def matchCheetahVarStart(self):
724 """includes the enclosure and cache token"""
725 return self.cheetahVarStartRE.match(self.src(), self.pos())
727 def matchCheetahVarStartToken(self):
728 """includes the enclosure and cache token"""
729 return self.cheetahVarStartTokenRE.match(self.src(), self.pos())
731 def matchCheetahVarInExpressionStartToken(self):
732 """no enclosures or cache tokens allowed"""
733 return self.cheetahVarInExpressionStartTokenRE.match(self.src(), self.pos())
735 def matchVariablePlaceholderStart(self):
736 """includes the enclosure and cache token"""
737 return self.cheetahVarStartRE.match(self.src(), self.pos())
739 def matchExpressionPlaceholderStart(self):
740 """includes the enclosure and cache token"""
741 return self.expressionPlaceholderStartRE.match(self.src(), self.pos())
743 def getCheetahVarStartToken(self):
744 """just the start token, not the enclosure or cache token"""
745 match = self.matchCheetahVarStartToken()
746 if not match:
747 raise ParseError(self, msg='Expected Cheetah $var start token')
748 return self.readTo( match.end() )
751 def getCacheToken(self):
752 try:
753 token = self.cacheTokenRE.match(self.src(), self.pos())
754 self.setPos( token.end() )
755 return token.group()
756 except:
757 raise ParseError(self, msg='Expected cache token')
759 def getSilentPlaceholderToken(self):
760 try:
761 token = self.silentPlaceholderTokenRE.match(self.src(), self.pos())
762 self.setPos( token.end() )
763 return token.group()
764 except:
765 raise ParseError(self, msg='Expected silent placeholder token')
769 def getTargetVarsList(self):
770 varnames = []
771 while not self.atEnd():
772 if self.peek() in ' \t\f':
773 self.getWhiteSpace()
774 elif self.peek() in '\r\n':
775 break
776 elif self.startswith(','):
777 self.advance()
778 elif self.startswith('in ') or self.startswith('in\t'):
779 break
780 #elif self.matchCheetahVarStart():
781 elif self.matchCheetahVarInExpressionStartToken():
782 self.getCheetahVarStartToken()
783 self.getSilentPlaceholderToken()
784 self.getCacheToken()
785 varnames.append( self.getDottedName() )
786 elif self.matchIdentifier():
787 varnames.append( self.getDottedName() )
788 else:
789 break
790 return varnames
792 def getCheetahVar(self, plain=False, skipStartToken=False):
793 """This is called when parsing inside expressions. Cache tokens are only
794 valid in placeholders so this method discards any cache tokens found.
796 if not skipStartToken:
797 self.getCheetahVarStartToken()
798 self.getSilentPlaceholderToken()
799 self.getCacheToken()
800 return self.getCheetahVarBody(plain=plain)
802 def getCheetahVarBody(self, plain=False):
803 # @@TR: this should be in the compiler
804 return self._compiler.genCheetahVar(self.getCheetahVarNameChunks(), plain=plain)
806 def getCheetahVarNameChunks(self):
809 nameChunks = list of Cheetah $var subcomponents represented as tuples
810 [ (namemapperPart,autoCall,restOfName),
812 where:
813 namemapperPart = the dottedName base
814 autocall = where NameMapper should use autocalling on namemapperPart
815 restOfName = any arglist, index, or slice
817 If restOfName contains a call arglist (e.g. '(1234)') then autocall is
818 False, otherwise it defaults to True.
820 EXAMPLE
821 ------------------------------------------------------------------------
823 if the raw CheetahVar is
824 $a.b.c[1].d().x.y.z
826 nameChunks is the list
827 [ ('a.b.c',True,'[1]'),
828 ('d',False,'()'),
829 ('x.y.z',True,''),
834 chunks = []
835 while self.pos() < len(self):
836 rest = ''
837 autoCall = True
838 if not self.peek() in identchars + '.':
839 break
840 elif self.peek() == '.':
842 if self.pos()+1 < len(self) and self.peek(1) in identchars:
843 self.advance() # discard the period as it isn't needed with NameMapper
844 else:
845 break
847 dottedName = self.getDottedName()
848 if not self.atEnd() and self.peek() in '([':
849 if self.peek() == '(':
850 rest = self.getCallArgString()
851 else:
852 rest = self.getExpression(enclosed=True)
854 period = max(dottedName.rfind('.'), 0)
855 if period:
856 chunks.append( (dottedName[:period], autoCall, '') )
857 dottedName = dottedName[period+1:]
858 if rest and rest[0]=='(':
859 autoCall = False
860 chunks.append( (dottedName, autoCall, rest) )
862 return chunks
865 def getCallArgString(self,
866 enclosures=[], # list of tuples (char, pos), where char is ({ or [
867 useNameMapper=Unspecified):
869 """ Get a method/function call argument string.
871 This method understands *arg, and **kw
874 # @@TR: this settings mangling should be removed
875 if useNameMapper is not Unspecified:
876 useNameMapper_orig = self.setting('useNameMapper')
877 self.setSetting('useNameMapper', useNameMapper)
879 if enclosures:
880 pass
881 else:
882 if not self.peek() == '(':
883 raise ParseError(self, msg="Expected '('")
884 startPos = self.pos()
885 self.getc()
886 enclosures = [('(', startPos),
889 argStringBits = ['(']
890 addBit = argStringBits.append
892 while 1:
893 if self.atEnd():
894 open = enclosures[-1][0]
895 close = closurePairsRev[open]
896 self.setPos(enclosures[-1][1])
897 raise ParseError(
898 self, msg="EOF was reached before a matching '" + close +
899 "' was found for the '" + open + "'")
901 c = self.peek()
902 if c in ")}]": # get the ending enclosure and break
903 if not enclosures:
904 raise ParseError(self)
905 c = self.getc()
906 open = closurePairs[c]
907 if enclosures[-1][0] == open:
908 enclosures.pop()
909 addBit(')')
910 break
911 else:
912 raise ParseError(self)
913 elif c in " \t\f\r\n":
914 addBit(self.getc())
915 elif self.matchCheetahVarInExpressionStartToken():
916 startPos = self.pos()
917 codeFor1stToken = self.getCheetahVar()
918 WS = self.getWhiteSpace()
919 if not self.atEnd() and self.peek() == '=':
920 nextToken = self.getPyToken()
921 if nextToken == '=':
922 endPos = self.pos()
923 self.setPos(startPos)
924 codeFor1stToken = self.getCheetahVar(plain=True)
925 self.setPos(endPos)
927 ## finally
928 addBit( codeFor1stToken + WS + nextToken )
929 else:
930 addBit( codeFor1stToken + WS)
931 elif self.matchCheetahVarStart():
932 # it has syntax that is only valid at the top level
933 self._raiseErrorAboutInvalidCheetahVarSyntaxInExpr()
934 else:
935 beforeTokenPos = self.pos()
936 token = self.getPyToken()
937 if token in ('{','(','['):
938 self.rev()
939 token = self.getExpression(enclosed=True)
940 token = self.transformToken(token, beforeTokenPos)
941 addBit(token)
943 if useNameMapper is not Unspecified:
944 self.setSetting('useNameMapper', useNameMapper_orig) # @@TR: see comment above
946 return ''.join(argStringBits)
948 def getDefArgList(self, exitPos=None, useNameMapper=False):
950 """ Get an argument list. Can be used for method/function definition
951 argument lists or for #directive argument lists. Returns a list of
952 tuples in the form (argName, defVal=None) with one tuple for each arg
953 name.
955 These defVals are always strings, so (argName, defVal=None) is safe even
956 with a case like (arg1, arg2=None, arg3=1234*2), which would be returned as
957 [('arg1', None),
958 ('arg2', 'None'),
959 ('arg3', '1234*2'),
962 This method understands *arg, and **kw
966 if self.peek() == '(':
967 self.advance()
968 else:
969 exitPos = self.findEOL() # it's a directive so break at the EOL
970 argList = ArgList()
971 onDefVal = False
973 # @@TR: this settings mangling should be removed
974 useNameMapper_orig = self.setting('useNameMapper')
975 self.setSetting('useNameMapper', useNameMapper)
977 while 1:
978 if self.atEnd():
979 self.setPos(enclosures[-1][1])
980 raise ParseError(
981 self, msg="EOF was reached before a matching ')'"+
982 " was found for the '('")
984 if self.pos() == exitPos:
985 break
987 c = self.peek()
988 if c == ")" or self.matchDirectiveEndToken():
989 break
990 elif c == ":":
991 break
992 elif c in " \t\f\r\n":
993 if onDefVal:
994 argList.addToDefVal(c)
995 self.advance()
996 elif c == '=':
997 onDefVal = True
998 self.advance()
999 elif c == ",":
1000 argList.next()
1001 onDefVal = False
1002 self.advance()
1003 elif self.startswith(self.cheetahVarStartToken) and not onDefVal:
1004 self.advance(len(self.cheetahVarStartToken))
1005 elif self.matchIdentifier() and not onDefVal:
1006 argList.addArgName( self.getIdentifier() )
1007 elif onDefVal:
1008 if self.matchCheetahVarInExpressionStartToken():
1009 token = self.getCheetahVar()
1010 elif self.matchCheetahVarStart():
1011 # it has syntax that is only valid at the top level
1012 self._raiseErrorAboutInvalidCheetahVarSyntaxInExpr()
1013 else:
1014 beforeTokenPos = self.pos()
1015 token = self.getPyToken()
1016 if token in ('{','(','['):
1017 self.rev()
1018 token = self.getExpression(enclosed=True)
1019 token = self.transformToken(token, beforeTokenPos)
1020 argList.addToDefVal(token)
1021 elif c == '*' and not onDefVal:
1022 varName = self.getc()
1023 if self.peek() == '*':
1024 varName += self.getc()
1025 if not self.matchIdentifier():
1026 raise ParseError(self)
1027 varName += self.getIdentifier()
1028 argList.addArgName(varName)
1029 else:
1030 raise ParseError(self)
1033 self.setSetting('useNameMapper', useNameMapper_orig) # @@TR: see comment above
1034 return argList.merge()
1036 def getExpressionParts(self,
1037 enclosed=False,
1038 enclosures=None, # list of tuples (char, pos), where char is ({ or [
1039 pyTokensToBreakAt=None, # only works if not enclosed
1040 useNameMapper=Unspecified,
1043 """ Get a Cheetah expression that includes $CheetahVars and break at
1044 directive end tokens, the end of an enclosure, or at a specified
1045 pyToken.
1048 if useNameMapper is not Unspecified:
1049 useNameMapper_orig = self.setting('useNameMapper')
1050 self.setSetting('useNameMapper', useNameMapper)
1052 if enclosures is None:
1053 enclosures = []
1055 srcLen = len(self)
1056 exprBits = []
1057 while 1:
1058 if self.atEnd():
1059 if enclosures:
1060 open = enclosures[-1][0]
1061 close = closurePairsRev[open]
1062 self.setPos(enclosures[-1][1])
1063 raise ParseError(
1064 self, msg="EOF was reached before a matching '" + close +
1065 "' was found for the '" + open + "'")
1066 else:
1067 break
1069 c = self.peek()
1070 if c in "{([":
1071 exprBits.append(c)
1072 enclosures.append( (c, self.pos()) )
1073 self.advance()
1074 elif enclosed and not enclosures:
1075 break
1076 elif c in "])}":
1077 if not enclosures:
1078 raise ParseError(self)
1079 open = closurePairs[c]
1080 if enclosures[-1][0] == open:
1081 enclosures.pop()
1082 exprBits.append(c)
1083 else:
1084 open = enclosures[-1][0]
1085 close = closurePairsRev[open]
1086 row, col = self.getRowCol()
1087 self.setPos(enclosures[-1][1])
1088 raise ParseError(
1089 self, msg= "A '" + c + "' was found at line " + str(row) +
1090 ", col " + str(col) +
1091 " before a matching '" + close +
1092 "' was found\nfor the '" + open + "'")
1093 self.advance()
1095 elif c in " \f\t":
1096 exprBits.append(self.getWhiteSpace())
1097 elif self.matchDirectiveEndToken() and not enclosures:
1098 break
1099 elif c == "\\" and self.pos()+1 < srcLen:
1100 eolMatch = EOLre.match(self.src(), self.pos()+1)
1101 if not eolMatch:
1102 self.advance()
1103 raise ParseError(self, msg='Line ending expected')
1104 self.setPos( eolMatch.end() )
1105 elif c in '\r\n':
1106 if enclosures:
1107 self.advance()
1108 else:
1109 break
1110 elif self.matchCheetahVarInExpressionStartToken():
1111 expr = self.getCheetahVar()
1112 exprBits.append(expr)
1113 elif self.matchCheetahVarStart():
1114 # it has syntax that is only valid at the top level
1115 self._raiseErrorAboutInvalidCheetahVarSyntaxInExpr()
1116 else:
1117 beforeTokenPos = self.pos()
1118 token = self.getPyToken()
1119 if (not enclosures
1120 and pyTokensToBreakAt
1121 and token in pyTokensToBreakAt):
1123 self.setPos(beforeTokenPos)
1124 break
1126 token = self.transformToken(token, beforeTokenPos)
1128 exprBits.append(token)
1129 if identRE.match(token):
1130 if token == 'for':
1131 expr = self.getExpression(useNameMapper=False, pyTokensToBreakAt=['in'])
1132 exprBits.append(expr)
1133 else:
1134 exprBits.append(self.getWhiteSpace())
1135 if not self.atEnd() and self.peek() == '(':
1136 exprBits.append(self.getCallArgString())
1138 if useNameMapper is not Unspecified:
1139 self.setSetting('useNameMapper', useNameMapper_orig) # @@TR: see comment above
1140 return exprBits
1142 def getExpression(self,
1143 enclosed=False,
1144 enclosures=None, # list of tuples (char, pos), where # char is ({ or [
1145 pyTokensToBreakAt=None,
1146 useNameMapper=Unspecified,
1148 """Returns the output of self.getExpressionParts() as a concatenated
1149 string rather than as a list.
1151 return ''.join(self.getExpressionParts(
1152 enclosed=enclosed, enclosures=enclosures,
1153 pyTokensToBreakAt=pyTokensToBreakAt,
1154 useNameMapper=useNameMapper))
1157 def transformToken(self, token, beforeTokenPos):
1158 """Takes a token from the expression being parsed and performs and
1159 special transformations required by Cheetah.
1161 At the moment only Cheetah's c'$placeholder strings' are transformed.
1163 if token=='c' and not self.atEnd() and self.peek() in '\'"':
1164 nextToken = self.getPyToken()
1165 token = nextToken.upper()
1166 theStr = eval(token)
1167 endPos = self.pos()
1168 if not theStr:
1169 return
1171 if token.startswith(single3) or token.startswith(double3):
1172 startPosIdx = 3
1173 else:
1174 startPosIdx = 1
1175 #print 'CHEETAH STRING', nextToken, theStr, startPosIdx
1176 self.setPos(beforeTokenPos+startPosIdx+1)
1177 outputExprs = []
1178 strConst = ''
1179 while self.pos() < (endPos-startPosIdx):
1180 if self.matchCheetahVarStart() or self.matchExpressionPlaceholderStart():
1181 if strConst:
1182 outputExprs.append(repr(strConst))
1183 strConst = ''
1184 placeholderExpr = self.getPlaceholder()
1185 outputExprs.append('str('+placeholderExpr+')')
1186 else:
1187 strConst += self.getc()
1188 self.setPos(endPos)
1189 if strConst:
1190 outputExprs.append(repr(strConst))
1191 #if not self.atEnd() and self.matches('.join('):
1192 # print 'DEBUG***'
1193 token = "''.join(["+','.join(outputExprs)+"])"
1194 return token
1196 def _raiseErrorAboutInvalidCheetahVarSyntaxInExpr(self):
1197 match = self.matchCheetahVarStart()
1198 groupdict = match.groupdict()
1199 if groupdict.get('cacheToken'):
1200 raise ParseError(
1201 self,
1202 msg='Cache tokens are not valid inside expressions. '
1203 'Use them in top-level $placeholders only.')
1204 elif groupdict.get('enclosure'):
1205 raise ParseError(
1206 self,
1207 msg='Long-form placeholders - ${}, $(), $[], etc. are not valid inside expressions. '
1208 'Use them in top-level $placeholders only.')
1209 else:
1210 raise ParseError(
1211 self,
1212 msg='This form of $placeholder syntax is not valid here.')
1215 def getPlaceholder(self, allowCacheTokens=False, plain=False, returnEverything=False):
1216 # filtered
1217 for callback in self.setting('preparsePlaceholderHooks'):
1218 callback(parser=self)
1220 startPos = self.pos()
1221 lineCol = self.getRowCol(startPos)
1222 startToken = self.getCheetahVarStartToken()
1223 silentPlaceholderToken = self.getSilentPlaceholderToken()
1224 if silentPlaceholderToken:
1225 isSilentPlaceholder = True
1226 else:
1227 isSilentPlaceholder = False
1230 if allowCacheTokens:
1231 cacheToken = self.getCacheToken()
1232 cacheTokenParts = self.cacheTokenRE.match(cacheToken).groupdict()
1233 else:
1234 cacheTokenParts = {}
1236 if self.peek() in '({[':
1237 pos = self.pos()
1238 enclosureOpenChar = self.getc()
1239 enclosures = [ (enclosureOpenChar, pos) ]
1240 self.getWhiteSpace()
1241 else:
1242 enclosures = []
1244 filterArgs = None
1245 if self.matchIdentifier():
1246 nameChunks = self.getCheetahVarNameChunks()
1247 expr = self._compiler.genCheetahVar(nameChunks[:], plain=plain)
1248 restOfExpr = None
1249 if enclosures:
1250 WS = self.getWhiteSpace()
1251 expr += WS
1252 if self.setting('allowPlaceholderFilterArgs') and self.peek()==',':
1253 filterArgs = self.getCallArgString(enclosures=enclosures)[1:-1]
1254 else:
1255 if self.peek()==closurePairsRev[enclosureOpenChar]:
1256 self.getc()
1257 else:
1258 restOfExpr = self.getExpression(enclosed=True, enclosures=enclosures)
1259 if restOfExpr[-1] == closurePairsRev[enclosureOpenChar]:
1260 restOfExpr = restOfExpr[:-1]
1261 expr += restOfExpr
1262 rawPlaceholder = self[startPos: self.pos()]
1263 else:
1264 expr = self.getExpression(enclosed=True, enclosures=enclosures)
1265 if expr[-1] == closurePairsRev[enclosureOpenChar]:
1266 expr = expr[:-1]
1267 rawPlaceholder=self[startPos: self.pos()]
1269 expr = self._applyExpressionFilters(expr,'placeholder',
1270 rawExpr=rawPlaceholder,startPos=startPos)
1271 for callback in self.setting('postparsePlaceholderHooks'):
1272 callback(parser=self)
1274 if returnEverything:
1275 return (expr, rawPlaceholder, lineCol, cacheTokenParts,
1276 filterArgs, isSilentPlaceholder)
1277 else:
1278 return expr
1281 class _HighLevelParser(_LowLevelParser):
1282 """This class is a StateMachine for parsing Cheetah source and
1283 sending state dependent code generation commands to
1284 Cheetah.Compiler.Compiler.
1286 def __init__(self, src, filename=None, breakPoint=None, compiler=None):
1287 _LowLevelParser.__init__(self, src, filename=filename, breakPoint=breakPoint)
1288 self.setSettingsManager(compiler)
1289 self._compiler = compiler
1290 self.setupState()
1291 self.configureParser()
1293 def setupState(self):
1294 self._macros = {}
1295 self._macroDetails = {}
1296 self._openDirectivesStack = []
1298 def cleanup(self):
1299 """Cleanup to remove any possible reference cycles
1301 self._macros.clear()
1302 for macroname, macroDetails in self._macroDetails.items():
1303 macroDetails.template.shutdown()
1304 del macroDetails.template
1305 self._macroDetails.clear()
1307 def configureParser(self):
1308 _LowLevelParser.configureParser(self)
1309 self._initDirectives()
1311 def _initDirectives(self):
1312 def normalizeParserVal(val):
1313 if isinstance(val, (str,unicode)):
1314 handler = getattr(self, val)
1315 elif type(val) in (ClassType, TypeType):
1316 handler = val(self)
1317 elif callable(val):
1318 handler = val
1319 elif val is None:
1320 handler = val
1321 else:
1322 raise Exception('Invalid parser/handler value %r for %s'%(val, name))
1323 return handler
1325 normalizeHandlerVal = normalizeParserVal
1327 _directiveNamesAndParsers = directiveNamesAndParsers.copy()
1328 customNamesAndParsers = self.setting('directiveNamesAndParsers',{})
1329 _directiveNamesAndParsers.update(customNamesAndParsers)
1331 _endDirectiveNamesAndHandlers = endDirectiveNamesAndHandlers.copy()
1332 customNamesAndHandlers = self.setting('endDirectiveNamesAndHandlers',{})
1333 _endDirectiveNamesAndHandlers.update(customNamesAndHandlers)
1335 self._directiveNamesAndParsers = {}
1336 for name, val in _directiveNamesAndParsers.items():
1337 if val in (False, 0):
1338 continue
1339 self._directiveNamesAndParsers[name] = normalizeParserVal(val)
1341 self._endDirectiveNamesAndHandlers = {}
1342 for name, val in _endDirectiveNamesAndHandlers.items():
1343 if val in (False, 0):
1344 continue
1345 self._endDirectiveNamesAndHandlers[name] = normalizeHandlerVal(val)
1347 self._closeableDirectives = ['def','block','closure','defmacro',
1348 'call',
1349 'capture',
1350 'cache',
1351 'filter',
1352 'if','unless',
1353 'for','while','repeat',
1354 'try',
1356 for directiveName in self.setting('closeableDirectives',[]):
1357 self._closeableDirectives.append(directiveName)
1361 macroDirectives = self.setting('macroDirectives',{})
1362 from Cheetah.Macros.I18n import I18n
1363 macroDirectives['i18n'] = I18n
1366 for macroName, callback in macroDirectives.items():
1367 if type(callback) in (ClassType, TypeType):
1368 callback = callback(parser=self)
1369 assert callback
1370 self._macros[macroName] = callback
1371 self._directiveNamesAndParsers[macroName] = self.eatMacroCall
1373 def _applyExpressionFilters(self, expr, exprType, rawExpr=None, startPos=None):
1374 """Pipes cheetah expressions through a set of optional filter hooks.
1376 The filters are functions which may modify the expressions or raise
1377 a ForbiddenExpression exception if the expression is not allowed. They
1378 are defined in the compiler setting 'expressionFilterHooks'.
1380 Some intended use cases:
1382 - to implement 'restricted execution' safeguards in cases where you
1383 can't trust the author of the template.
1385 - to enforce style guidelines
1387 filter call signature: (parser, expr, exprType, rawExpr=None, startPos=None)
1388 - parser is the Cheetah parser
1389 - expr is the expression to filter. In some cases the parser will have
1390 already modified it from the original source code form. For example,
1391 placeholders will have been translated into namemapper calls. If you
1392 need to work with the original source, see rawExpr.
1393 - exprType is the name of the directive, 'psp', or 'placeholder'. All
1394 lowercase. @@TR: These will eventually be replaced with a set of
1395 constants.
1396 - rawExpr is the original source string that Cheetah parsed. This
1397 might be None in some cases.
1398 - startPos is the character position in the source string/file
1399 where the parser started parsing the current expression.
1401 @@TR: I realize this use of the term 'expression' is a bit wonky as many
1402 of the 'expressions' are actually statements, but I haven't thought of
1403 a better name yet. Suggestions?
1405 for callback in self.setting('expressionFilterHooks'):
1406 expr = callback(parser=self, expr=expr, exprType=exprType,
1407 rawExpr=rawExpr, startPos=startPos)
1408 return expr
1410 def _filterDisabledDirectives(self, directiveName):
1411 directiveName = directiveName.lower()
1412 if (directiveName in self.setting('disabledDirectives')
1413 or (self.setting('enabledDirectives')
1414 and directiveName not in self.setting('enabledDirectives'))):
1415 for callback in self.setting('disabledDirectiveHooks'):
1416 callback(parser=self, directiveName=directiveName)
1417 raise ForbiddenDirective(self, msg='This %r directive is disabled'%directiveName)
1419 ## main parse loop
1421 def parse(self, breakPoint=None, assertEmptyStack=True):
1422 if breakPoint:
1423 origBP = self.breakPoint()
1424 self.setBreakPoint(breakPoint)
1425 assertEmptyStack = False
1427 while not self.atEnd():
1428 if self.matchCommentStartToken():
1429 self.eatComment()
1430 elif self.matchMultiLineCommentStartToken():
1431 self.eatMultiLineComment()
1432 elif self.matchVariablePlaceholderStart():
1433 self.eatPlaceholder()
1434 elif self.matchExpressionPlaceholderStart():
1435 self.eatPlaceholder()
1436 elif self.matchDirective():
1437 self.eatDirective()
1438 elif self.matchPSPStartToken():
1439 self.eatPSP()
1440 elif self.matchEOLSlurpToken():
1441 self.eatEOLSlurpToken()
1442 else:
1443 self.eatPlainText()
1444 if assertEmptyStack:
1445 self.assertEmptyOpenDirectivesStack()
1446 if breakPoint:
1447 self.setBreakPoint(origBP)
1449 ## non-directive eat methods
1451 def eatPlainText(self):
1452 startPos = self.pos()
1453 match = None
1454 while not self.atEnd():
1455 match = self.matchTopLevelToken()
1456 if match:
1457 break
1458 else:
1459 self.advance()
1460 strConst = self.readTo(self.pos(), start=startPos)
1461 self._compiler.addStrConst(strConst)
1462 return match
1464 def eatComment(self):
1465 isLineClearToStartToken = self.isLineClearToStartToken()
1466 if isLineClearToStartToken:
1467 self._compiler.handleWSBeforeDirective()
1468 self.getCommentStartToken()
1469 comm = self.readToEOL(gobble=isLineClearToStartToken)
1470 self._compiler.addComment(comm)
1472 def eatMultiLineComment(self):
1473 isLineClearToStartToken = self.isLineClearToStartToken()
1474 endOfFirstLine = self.findEOL()
1476 self.getMultiLineCommentStartToken()
1477 endPos = startPos = self.pos()
1478 level = 1
1479 while 1:
1480 endPos = self.pos()
1481 if self.atEnd():
1482 break
1483 if self.matchMultiLineCommentStartToken():
1484 self.getMultiLineCommentStartToken()
1485 level += 1
1486 elif self.matchMultiLineCommentEndToken():
1487 self.getMultiLineCommentEndToken()
1488 level -= 1
1489 if not level:
1490 break
1491 self.advance()
1492 comm = self.readTo(endPos, start=startPos)
1494 if not self.atEnd():
1495 self.getMultiLineCommentEndToken()
1497 if (not self.atEnd()) and self.setting('gobbleWhitespaceAroundMultiLineComments'):
1498 restOfLine = self[self.pos():self.findEOL()]
1499 if not restOfLine.strip(): # WS only to EOL
1500 self.readToEOL(gobble=isLineClearToStartToken)
1502 if isLineClearToStartToken and (self.atEnd() or self.pos() > endOfFirstLine):
1503 self._compiler.handleWSBeforeDirective()
1505 self._compiler.addComment(comm)
1507 def eatPlaceholder(self):
1508 (expr, rawPlaceholder,
1509 lineCol, cacheTokenParts,
1510 filterArgs, isSilentPlaceholder) = self.getPlaceholder(
1511 allowCacheTokens=True, returnEverything=True)
1513 self._compiler.addPlaceholder(
1514 expr,
1515 filterArgs=filterArgs,
1516 rawPlaceholder=rawPlaceholder,
1517 cacheTokenParts=cacheTokenParts,
1518 lineCol=lineCol,
1519 silentMode=isSilentPlaceholder)
1520 return
1522 def eatPSP(self):
1523 # filtered
1524 self._filterDisabledDirectives(directiveName='psp')
1525 self.getPSPStartToken()
1526 endToken = self.setting('PSPEndToken')
1527 startPos = self.pos()
1528 while not self.atEnd():
1529 if self.peek() == endToken[0]:
1530 if self.matchPSPEndToken():
1531 break
1532 self.advance()
1533 pspString = self.readTo(self.pos(), start=startPos).strip()
1534 pspString = self._applyExpressionFilters(pspString, 'psp', startPos=startPos)
1535 self._compiler.addPSP(pspString)
1536 self.getPSPEndToken()
1538 ## generic directive eat methods
1539 _simpleIndentingDirectives = '''
1540 else elif for while repeat unless try except finally'''.split()
1541 _simpleExprDirectives = '''
1542 pass continue stop return yield break
1543 del assert raise
1544 silent echo
1545 import from'''.split()
1546 _directiveHandlerNames = {'import':'addImportStatement',
1547 'from':'addImportStatement', }
1548 def eatDirective(self):
1549 directiveName = self.matchDirective()
1550 self._filterDisabledDirectives(directiveName)
1552 for callback in self.setting('preparseDirectiveHooks'):
1553 callback(parser=self, directiveName=directiveName)
1555 # subclasses can override the default behaviours here by providing an
1556 # eater method in self._directiveNamesAndParsers[directiveName]
1557 directiveParser = self._directiveNamesAndParsers.get(directiveName)
1558 if directiveParser:
1559 directiveParser()
1560 elif directiveName in self._simpleIndentingDirectives:
1561 handlerName = self._directiveHandlerNames.get(directiveName)
1562 if not handlerName:
1563 handlerName = 'add'+directiveName.capitalize()
1564 handler = getattr(self._compiler, handlerName)
1565 self.eatSimpleIndentingDirective(directiveName, callback=handler)
1566 elif directiveName in self._simpleExprDirectives:
1567 handlerName = self._directiveHandlerNames.get(directiveName)
1568 if not handlerName:
1569 handlerName = 'add'+directiveName.capitalize()
1570 handler = getattr(self._compiler, handlerName)
1571 if directiveName in ('silent', 'echo'):
1572 includeDirectiveNameInExpr = False
1573 else:
1574 includeDirectiveNameInExpr = True
1575 expr = self.eatSimpleExprDirective(
1576 directiveName,
1577 includeDirectiveNameInExpr=includeDirectiveNameInExpr)
1578 handler(expr)
1580 for callback in self.setting('postparseDirectiveHooks'):
1581 callback(parser=self, directiveName=directiveName)
1583 def _eatRestOfDirectiveTag(self, isLineClearToStartToken, endOfFirstLinePos):
1584 foundComment = False
1585 if self.matchCommentStartToken():
1586 pos = self.pos()
1587 self.advance()
1588 if not self.matchDirective():
1589 self.setPos(pos)
1590 foundComment = True
1591 self.eatComment() # this won't gobble the EOL
1592 else:
1593 self.setPos(pos)
1595 if not foundComment and self.matchDirectiveEndToken():
1596 self.getDirectiveEndToken()
1597 elif isLineClearToStartToken and (not self.atEnd()) and self.peek() in '\r\n':
1598 # still gobble the EOL if a comment was found.
1599 self.readToEOL(gobble=True)
1601 if isLineClearToStartToken and (self.atEnd() or self.pos() > endOfFirstLinePos):
1602 self._compiler.handleWSBeforeDirective()
1604 def _eatToThisEndDirective(self, directiveName):
1605 finalPos = endRawPos = startPos = self.pos()
1606 directiveChar = self.setting('directiveStartToken')[0]
1607 isLineClearToStartToken = False
1608 while not self.atEnd():
1609 if self.peek() == directiveChar:
1610 if self.matchDirective() == 'end':
1611 endRawPos = self.pos()
1612 self.getDirectiveStartToken()
1613 self.advance(len('end'))
1614 self.getWhiteSpace()
1615 if self.startswith(directiveName):
1616 if self.isLineClearToStartToken(endRawPos):
1617 isLineClearToStartToken = True
1618 endRawPos = self.findBOL(endRawPos)
1619 self.advance(len(directiveName)) # to end of directiveName
1620 self.getWhiteSpace()
1621 finalPos = self.pos()
1622 break
1623 self.advance()
1624 finalPos = endRawPos = self.pos()
1626 textEaten = self.readTo(endRawPos, start=startPos)
1627 self.setPos(finalPos)
1629 endOfFirstLinePos = self.findEOL()
1631 if self.matchDirectiveEndToken():
1632 self.getDirectiveEndToken()
1633 elif isLineClearToStartToken and (not self.atEnd()) and self.peek() in '\r\n':
1634 self.readToEOL(gobble=True)
1636 if isLineClearToStartToken and self.pos() > endOfFirstLinePos:
1637 self._compiler.handleWSBeforeDirective()
1638 return textEaten
1641 def eatSimpleExprDirective(self, directiveName, includeDirectiveNameInExpr=True):
1642 # filtered
1643 isLineClearToStartToken = self.isLineClearToStartToken()
1644 endOfFirstLine = self.findEOL()
1645 self.getDirectiveStartToken()
1646 if not includeDirectiveNameInExpr:
1647 self.advance(len(directiveName))
1648 startPos = self.pos()
1649 expr = self.getExpression().strip()
1650 directiveName = expr.split()[0]
1651 expr = self._applyExpressionFilters(expr, directiveName, startPos=startPos)
1652 if directiveName in self._closeableDirectives:
1653 self.pushToOpenDirectivesStack(directiveName)
1654 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
1655 return expr
1657 def eatSimpleIndentingDirective(self, directiveName, callback,
1658 includeDirectiveNameInExpr=False):
1659 # filtered
1660 isLineClearToStartToken = self.isLineClearToStartToken()
1661 endOfFirstLinePos = self.findEOL()
1662 lineCol = self.getRowCol()
1663 self.getDirectiveStartToken()
1664 if directiveName not in 'else elif for while try except finally'.split():
1665 self.advance(len(directiveName))
1666 startPos = self.pos()
1668 self.getWhiteSpace()
1670 expr = self.getExpression(pyTokensToBreakAt=[':'])
1671 expr = self._applyExpressionFilters(expr, directiveName, startPos=startPos)
1672 if self.matchColonForSingleLineShortFormDirective():
1673 self.advance() # skip over :
1674 if directiveName in 'else elif except finally'.split():
1675 callback(expr, dedent=False, lineCol=lineCol)
1676 else:
1677 callback(expr, lineCol=lineCol)
1679 self.getWhiteSpace(max=1)
1680 self.parse(breakPoint=self.findEOL(gobble=True))
1681 self._compiler.commitStrConst()
1682 self._compiler.dedent()
1683 else:
1684 if self.peek()==':':
1685 self.advance()
1686 self.getWhiteSpace()
1687 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
1688 if directiveName in self._closeableDirectives:
1689 self.pushToOpenDirectivesStack(directiveName)
1690 callback(expr, lineCol=lineCol)
1692 def eatEndDirective(self):
1693 isLineClearToStartToken = self.isLineClearToStartToken()
1694 self.getDirectiveStartToken()
1695 self.advance(3) # to end of 'end'
1696 self.getWhiteSpace()
1697 pos = self.pos()
1698 directiveName = False
1699 for key in self._endDirectiveNamesAndHandlers.keys():
1700 if self.find(key, pos) == pos:
1701 directiveName = key
1702 break
1703 if not directiveName:
1704 raise ParseError(self, msg='Invalid end directive')
1706 endOfFirstLinePos = self.findEOL()
1707 self.getExpression() # eat in any extra comment-like crap
1708 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
1709 if directiveName in self._closeableDirectives:
1710 self.popFromOpenDirectivesStack(directiveName)
1712 # subclasses can override the default behaviours here by providing an
1713 # end-directive handler in self._endDirectiveNamesAndHandlers[directiveName]
1714 if self._endDirectiveNamesAndHandlers.get(directiveName):
1715 handler = self._endDirectiveNamesAndHandlers[directiveName]
1716 handler()
1717 elif directiveName in 'block capture cache call filter errorCatcher'.split():
1718 if key == 'block':
1719 self._compiler.closeBlock()
1720 elif key == 'capture':
1721 self._compiler.endCaptureRegion()
1722 elif key == 'cache':
1723 self._compiler.endCacheRegion()
1724 elif key == 'call':
1725 self._compiler.endCallRegion()
1726 elif key == 'filter':
1727 self._compiler.closeFilterBlock()
1728 elif key == 'errorCatcher':
1729 self._compiler.turnErrorCatcherOff()
1730 elif directiveName in 'while for if try repeat unless'.split():
1731 self._compiler.commitStrConst()
1732 self._compiler.dedent()
1733 elif directiveName=='closure':
1734 self._compiler.commitStrConst()
1735 self._compiler.dedent()
1736 # @@TR: temporary hack of useSearchList
1737 self.setSetting('useSearchList', self._useSearchList_orig)
1739 ## specific directive eat methods
1741 def eatBreakPoint(self):
1742 """Tells the parser to stop parsing at this point and completely ignore
1743 everything else.
1745 This is a debugging tool.
1747 self.setBreakPoint(self.pos())
1749 def eatShbang(self):
1750 # filtered
1751 self.getDirectiveStartToken()
1752 self.advance(len('shBang'))
1753 self.getWhiteSpace()
1754 startPos = self.pos()
1755 shBang = self.readToEOL()
1756 shBang = self._applyExpressionFilters(shBang, 'shbang', startPos=startPos)
1757 self._compiler.setShBang(shBang.strip())
1759 def eatEncoding(self):
1760 # filtered
1761 self.getDirectiveStartToken()
1762 self.advance(len('encoding'))
1763 self.getWhiteSpace()
1764 startPos = self.pos()
1765 encoding = self.readToEOL()
1766 encoding = self._applyExpressionFilters(encoding, 'encoding', startPos=startPos)
1767 self._compiler.setModuleEncoding(encoding.strip())
1769 def eatCompiler(self):
1770 # filtered
1771 isLineClearToStartToken = self.isLineClearToStartToken()
1772 endOfFirstLine = self.findEOL()
1773 startPos = self.pos()
1774 self.getDirectiveStartToken()
1775 self.advance(len('compiler')) # to end of 'compiler'
1776 self.getWhiteSpace()
1778 startPos = self.pos()
1779 settingName = self.getIdentifier()
1781 if settingName.lower() == 'reset':
1782 self.getExpression() # gobble whitespace & junk
1783 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
1784 self._initializeSettings()
1785 self.configureParser()
1786 return
1788 self.getWhiteSpace()
1789 if self.peek() == '=':
1790 self.advance()
1791 else:
1792 raise ParserError(self)
1793 valueExpr = self.getExpression()
1794 endPos = self.pos()
1796 # @@TR: it's unlikely that anyone apply filters would have left this
1797 # directive enabled:
1798 # @@TR: fix up filtering, regardless
1799 self._applyExpressionFilters('%s=%r'%(settingName, valueExpr),
1800 'compiler', startPos=startPos)
1802 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
1803 try:
1804 self._compiler.setCompilerSetting(settingName, valueExpr)
1805 except:
1806 out = sys.stderr
1807 print >> out, 'An error occurred while processing the following #compiler directive.'
1808 print >> out, '-'*80
1809 print >> out, self[startPos:endPos]
1810 print >> out, '-'*80
1811 print >> out, 'Please check the syntax of these settings.'
1812 print >> out, 'A full Python exception traceback follows.'
1813 raise
1816 def eatCompilerSettings(self):
1817 # filtered
1818 isLineClearToStartToken = self.isLineClearToStartToken()
1819 endOfFirstLine = self.findEOL()
1820 self.getDirectiveStartToken()
1821 self.advance(len('compiler-settings')) # to end of 'settings'
1823 keywords = self.getTargetVarsList()
1824 self.getExpression() # gobble any garbage
1826 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
1828 if 'reset' in keywords:
1829 self._compiler._initializeSettings()
1830 self.configureParser()
1831 # @@TR: this implies a single-line #compiler-settings directive, and
1832 # thus we should parse forward for an end directive.
1833 # Subject to change in the future
1834 return
1835 startPos = self.pos()
1836 settingsStr = self._eatToThisEndDirective('compiler-settings')
1837 settingsStr = self._applyExpressionFilters(settingsStr, 'compilerSettings',
1838 startPos=startPos)
1839 try:
1840 self._compiler.setCompilerSettings(keywords=keywords, settingsStr=settingsStr)
1841 except:
1842 out = sys.stderr
1843 print >> out, 'An error occurred while processing the following compiler settings.'
1844 print >> out, '-'*80
1845 print >> out, settingsStr.strip()
1846 print >> out, '-'*80
1847 print >> out, 'Please check the syntax of these settings.'
1848 print >> out, 'A full Python exception traceback follows.'
1849 raise
1851 def eatAttr(self):
1852 # filtered
1853 isLineClearToStartToken = self.isLineClearToStartToken()
1854 endOfFirstLinePos = self.findEOL()
1855 startPos = self.pos()
1856 self.getDirectiveStartToken()
1857 self.advance(len('attr'))
1858 self.getWhiteSpace()
1859 startPos = self.pos()
1860 if self.matchCheetahVarStart():
1861 self.getCheetahVarStartToken()
1862 attribName = self.getIdentifier()
1863 self.getWhiteSpace()
1864 self.getAssignmentOperator()
1865 expr = self.getExpression()
1866 expr = self._applyExpressionFilters(expr, 'attr', startPos=startPos)
1867 self._compiler.addAttribute(attribName, expr)
1868 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
1870 def eatDecorator(self):
1871 isLineClearToStartToken = self.isLineClearToStartToken()
1872 endOfFirstLinePos = self.findEOL()
1873 startPos = self.pos()
1874 self.getDirectiveStartToken()
1875 #self.advance() # eat @
1876 startPos = self.pos()
1877 decoratorExpr = self.getExpression()
1878 decoratorExpr = self._applyExpressionFilters(decoratorExpr, 'decorator', startPos=startPos)
1879 self._compiler.addDecorator(decoratorExpr)
1880 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
1881 self.getWhiteSpace()
1883 directiveName = self.matchDirective()
1884 if not directiveName or directiveName not in ('def', 'block', 'closure'):
1885 raise ParseError(self, msg='Expected #def, #block or #closure')
1886 self.eatDirective()
1888 def eatDef(self):
1889 # filtered
1890 self._eatDefOrBlock('def')
1892 def eatBlock(self):
1893 # filtered
1894 startPos = self.pos()
1895 methodName, rawSignature = self._eatDefOrBlock('block')
1896 self._compiler._blockMetaData[methodName] = {
1897 'raw':rawSignature,
1898 'lineCol':self.getRowCol(startPos),
1901 def eatClosure(self):
1902 # filtered
1903 self._eatDefOrBlock('closure')
1905 def _eatDefOrBlock(self, directiveName):
1906 # filtered
1907 assert directiveName in ('def','block','closure')
1908 isLineClearToStartToken = self.isLineClearToStartToken()
1909 endOfFirstLinePos = self.findEOL()
1910 startPos = self.pos()
1911 self.getDirectiveStartToken()
1912 self.advance(len(directiveName))
1913 self.getWhiteSpace()
1914 if self.matchCheetahVarStart():
1915 self.getCheetahVarStartToken()
1916 methodName = self.getIdentifier()
1917 self.getWhiteSpace()
1918 if self.peek() == '(':
1919 argsList = self.getDefArgList()
1920 self.advance() # past the closing ')'
1921 if argsList and argsList[0][0] == 'self':
1922 del argsList[0]
1923 else:
1924 argsList=[]
1926 def includeBlockMarkers():
1927 if self.setting('includeBlockMarkers'):
1928 startMarker = self.setting('blockMarkerStart')
1929 self._compiler.addStrConst(startMarker[0] + methodName + startMarker[1])
1931 # @@TR: fix up filtering
1932 self._applyExpressionFilters(self[startPos:self.pos()], 'def', startPos=startPos)
1934 if self.matchColonForSingleLineShortFormDirective():
1935 isNestedDef = (self.setting('allowNestedDefScopes')
1936 and [name for name in self._openDirectivesStack if name=='def'])
1937 self.getc()
1938 rawSignature = self[startPos:endOfFirstLinePos]
1939 self._eatSingleLineDef(directiveName=directiveName,
1940 methodName=methodName,
1941 argsList=argsList,
1942 startPos=startPos,
1943 endPos=endOfFirstLinePos)
1944 if directiveName == 'def' and not isNestedDef:
1945 #@@TR: must come before _eatRestOfDirectiveTag ... for some reason
1946 self._compiler.closeDef()
1947 elif directiveName == 'block':
1948 includeBlockMarkers()
1949 self._compiler.closeBlock()
1950 elif directiveName == 'closure' or isNestedDef:
1951 self._compiler.dedent()
1953 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
1954 else:
1955 if self.peek()==':':
1956 self.getc()
1957 self.pushToOpenDirectivesStack(directiveName)
1958 rawSignature = self[startPos:self.pos()]
1959 self._eatMultiLineDef(directiveName=directiveName,
1960 methodName=methodName,
1961 argsList=argsList,
1962 startPos=startPos,
1963 isLineClearToStartToken=isLineClearToStartToken)
1964 if directiveName == 'block':
1965 includeBlockMarkers()
1967 return methodName, rawSignature
1969 def _eatMultiLineDef(self, directiveName, methodName, argsList, startPos,
1970 isLineClearToStartToken=False):
1971 # filtered in calling method
1972 self.getExpression() # slurp up any garbage left at the end
1973 signature = self[startPos:self.pos()]
1974 endOfFirstLinePos = self.findEOL()
1975 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
1976 parserComment = ('## CHEETAH: generated from ' + signature +
1977 ' at line %s, col %s' % self.getRowCol(startPos)
1978 + '.')
1980 isNestedDef = (self.setting('allowNestedDefScopes')
1981 and len([name for name in self._openDirectivesStack if name=='def'])>1)
1982 if directiveName=='block' or (directiveName=='def' and not isNestedDef):
1983 self._compiler.startMethodDef(methodName, argsList, parserComment)
1984 else: #closure
1985 self._useSearchList_orig = self.setting('useSearchList')
1986 self.setSetting('useSearchList', False)
1987 self._compiler.addClosure(methodName, argsList, parserComment)
1989 return methodName
1991 def _eatSingleLineDef(self, directiveName, methodName, argsList, startPos, endPos):
1992 # filtered in calling method
1993 fullSignature = self[startPos:endPos]
1994 parserComment = ('## Generated from ' + fullSignature +
1995 ' at line %s, col %s' % self.getRowCol(startPos)
1996 + '.')
1997 isNestedDef = (self.setting('allowNestedDefScopes')
1998 and [name for name in self._openDirectivesStack if name=='def'])
1999 if directiveName=='block' or (directiveName=='def' and not isNestedDef):
2000 self._compiler.startMethodDef(methodName, argsList, parserComment)
2001 else: #closure
2002 # @@TR: temporary hack of useSearchList
2003 useSearchList_orig = self.setting('useSearchList')
2004 self.setSetting('useSearchList', False)
2005 self._compiler.addClosure(methodName, argsList, parserComment)
2007 self.getWhiteSpace(max=1)
2008 self.parse(breakPoint=endPos)
2009 if directiveName=='closure' or isNestedDef: # @@TR: temporary hack of useSearchList
2010 self.setSetting('useSearchList', useSearchList_orig)
2012 def eatExtends(self):
2013 # filtered
2014 isLineClearToStartToken = self.isLineClearToStartToken()
2015 endOfFirstLine = self.findEOL()
2016 self.getDirectiveStartToken()
2017 self.advance(len('extends'))
2018 self.getWhiteSpace()
2019 startPos = self.pos()
2020 if self.setting('allowExpressionsInExtendsDirective'):
2021 baseName = self.getExpression()
2022 else:
2023 baseName = self.getDottedName()
2025 baseName = self._applyExpressionFilters(baseName, 'extends', startPos=startPos)
2026 self._compiler.setBaseClass(baseName) # in compiler
2027 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
2029 def eatImplements(self):
2030 # filtered
2031 isLineClearToStartToken = self.isLineClearToStartToken()
2032 endOfFirstLine = self.findEOL()
2033 self.getDirectiveStartToken()
2034 self.advance(len('implements'))
2035 self.getWhiteSpace()
2036 startPos = self.pos()
2037 methodName = self.getIdentifier()
2038 if not self.atEnd() and self.peek() == '(':
2039 argsList = self.getDefArgList()
2040 self.advance() # past the closing ')'
2041 if argsList and argsList[0][0] == 'self':
2042 del argsList[0]
2043 else:
2044 argsList=[]
2046 # @@TR: need to split up filtering of the methodname and the args
2047 #methodName = self._applyExpressionFilters(methodName, 'implements', startPos=startPos)
2048 self._applyExpressionFilters(self[startPos:self.pos()], 'implements', startPos=startPos)
2050 self._compiler.setMainMethodName(methodName)
2051 self._compiler.setMainMethodArgs(argsList)
2053 self.getExpression() # throw away and unwanted crap that got added in
2054 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
2056 def eatSet(self):
2057 # filtered
2058 isLineClearToStartToken = self.isLineClearToStartToken()
2059 endOfFirstLine = self.findEOL()
2060 self.getDirectiveStartToken()
2061 self.advance(3)
2062 self.getWhiteSpace()
2063 style = SET_LOCAL
2064 if self.startswith('local'):
2065 self.getIdentifier()
2066 self.getWhiteSpace()
2067 elif self.startswith('global'):
2068 self.getIdentifier()
2069 self.getWhiteSpace()
2070 style = SET_GLOBAL
2071 elif self.startswith('module'):
2072 self.getIdentifier()
2073 self.getWhiteSpace()
2074 style = SET_MODULE
2076 startsWithDollar = self.matchCheetahVarStart()
2077 startPos = self.pos()
2078 LVALUE = self.getExpression(pyTokensToBreakAt=assignmentOps, useNameMapper=False).strip()
2079 OP = self.getAssignmentOperator()
2080 RVALUE = self.getExpression()
2081 expr = LVALUE + ' ' + OP + ' ' + RVALUE.strip()
2083 expr = self._applyExpressionFilters(expr, 'set', startPos=startPos)
2084 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
2086 class Components: pass # used for 'set global'
2087 exprComponents = Components()
2088 exprComponents.LVALUE = LVALUE
2089 exprComponents.OP = OP
2090 exprComponents.RVALUE = RVALUE
2091 self._compiler.addSet(expr, exprComponents, style)
2093 def eatSlurp(self):
2094 if self.isLineClearToStartToken():
2095 self._compiler.handleWSBeforeDirective()
2096 self._compiler.commitStrConst()
2097 self.readToEOL(gobble=True)
2099 def eatEOLSlurpToken(self):
2100 if self.isLineClearToStartToken():
2101 self._compiler.handleWSBeforeDirective()
2102 self._compiler.commitStrConst()
2103 self.readToEOL(gobble=True)
2105 def eatRaw(self):
2106 isLineClearToStartToken = self.isLineClearToStartToken()
2107 endOfFirstLinePos = self.findEOL()
2108 self.getDirectiveStartToken()
2109 self.advance(len('raw'))
2110 self.getWhiteSpace()
2111 if self.matchColonForSingleLineShortFormDirective():
2112 self.advance() # skip over :
2113 self.getWhiteSpace(max=1)
2114 rawBlock = self.readToEOL(gobble=False)
2115 else:
2116 if self.peek()==':':
2117 self.advance()
2118 self.getWhiteSpace()
2119 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2120 rawBlock = self._eatToThisEndDirective('raw')
2121 self._compiler.addRawText(rawBlock)
2123 def eatInclude(self):
2124 # filtered
2125 isLineClearToStartToken = self.isLineClearToStartToken()
2126 endOfFirstLinePos = self.findEOL()
2127 self.getDirectiveStartToken()
2128 self.advance(len('include'))
2130 self.getWhiteSpace()
2131 includeFrom = 'file'
2132 isRaw = False
2133 if self.startswith('raw'):
2134 self.advance(3)
2135 isRaw=True
2137 self.getWhiteSpace()
2138 if self.startswith('source'):
2139 self.advance(len('source'))
2140 includeFrom = 'str'
2141 self.getWhiteSpace()
2142 if not self.peek() == '=':
2143 raise ParseError(self)
2144 self.advance()
2145 startPos = self.pos()
2146 sourceExpr = self.getExpression()
2147 sourceExpr = self._applyExpressionFilters(sourceExpr, 'include', startPos=startPos)
2148 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2149 self._compiler.addInclude(sourceExpr, includeFrom, isRaw)
2152 def eatDefMacro(self):
2153 # @@TR: not filtered yet
2154 isLineClearToStartToken = self.isLineClearToStartToken()
2155 endOfFirstLinePos = self.findEOL()
2156 self.getDirectiveStartToken()
2157 self.advance(len('defmacro'))
2159 self.getWhiteSpace()
2160 if self.matchCheetahVarStart():
2161 self.getCheetahVarStartToken()
2162 macroName = self.getIdentifier()
2163 self.getWhiteSpace()
2164 if self.peek() == '(':
2165 argsList = self.getDefArgList(useNameMapper=False)
2166 self.advance() # past the closing ')'
2167 if argsList and argsList[0][0] == 'self':
2168 del argsList[0]
2169 else:
2170 argsList=[]
2172 assert not self._directiveNamesAndParsers.has_key(macroName)
2173 argsList.insert(0, ('src',None))
2174 argsList.append(('parser','None'))
2175 argsList.append(('macros','None'))
2176 argsList.append(('compilerSettings','None'))
2177 argsList.append(('isShortForm','None'))
2178 argsList.append(('EOLCharsInShortForm','None'))
2179 argsList.append(('startPos','None'))
2180 argsList.append(('endPos','None'))
2182 if self.matchColonForSingleLineShortFormDirective():
2183 self.advance() # skip over :
2184 self.getWhiteSpace(max=1)
2185 macroSrc = self.readToEOL(gobble=False)
2186 self.readToEOL(gobble=True)
2187 else:
2188 if self.peek()==':':
2189 self.advance()
2190 self.getWhiteSpace()
2191 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2192 macroSrc = self._eatToThisEndDirective('defmacro')
2194 #print argsList
2195 normalizedMacroSrc = ''.join(
2196 ['%def callMacro('+','.join([defv and '%s=%s'%(n,defv) or n
2197 for n,defv in argsList])
2198 +')\n',
2199 macroSrc,
2200 '%end def'])
2203 from Cheetah.Template import Template
2204 templateAPIClass = self.setting('templateAPIClassForDefMacro', default=Template)
2205 compilerSettings = self.setting('compilerSettingsForDefMacro', default={})
2206 searchListForMacros = self.setting('searchListForDefMacro', default=[])
2207 searchListForMacros = list(searchListForMacros) # copy to avoid mutation bugs
2208 searchListForMacros.append({'macros':self._macros,
2209 'parser':self,
2210 'compilerSettings':self.settings(),
2213 templateAPIClass._updateSettingsWithPreprocessTokens(
2214 compilerSettings, placeholderToken='@', directiveToken='%')
2215 macroTemplateClass = templateAPIClass.compile(source=normalizedMacroSrc,
2216 compilerSettings=compilerSettings)
2217 #print normalizedMacroSrc
2218 #t = macroTemplateClass()
2219 #print t.callMacro('src')
2220 #print t.generatedClassCode()
2222 class MacroDetails: pass
2223 macroDetails = MacroDetails()
2224 macroDetails.macroSrc = macroSrc
2225 macroDetails.argsList = argsList
2226 macroDetails.template = macroTemplateClass(searchList=searchListForMacros)
2228 self._macroDetails[macroName] = macroDetails
2229 self._macros[macroName] = macroDetails.template.callMacro
2230 self._directiveNamesAndParsers[macroName] = self.eatMacroCall
2232 def eatMacroCall(self):
2233 isLineClearToStartToken = self.isLineClearToStartToken()
2234 endOfFirstLinePos = self.findEOL()
2235 startPos = self.pos()
2236 self.getDirectiveStartToken()
2237 macroName = self.getIdentifier()
2238 macro = self._macros[macroName]
2239 if hasattr(macro, 'parse'):
2240 return macro.parse(parser=self, startPos=startPos)
2242 if hasattr(macro, 'parseArgs'):
2243 args = macro.parseArgs(parser=self, startPos=startPos)
2244 else:
2245 self.getWhiteSpace()
2246 args = self.getExpression(useNameMapper=False,
2247 pyTokensToBreakAt=[':']).strip()
2249 if self.matchColonForSingleLineShortFormDirective():
2250 isShortForm = True
2251 self.advance() # skip over :
2252 self.getWhiteSpace(max=1)
2253 srcBlock = self.readToEOL(gobble=False)
2254 EOLCharsInShortForm = self.readToEOL(gobble=True)
2255 #self.readToEOL(gobble=False)
2256 else:
2257 isShortForm = False
2258 if self.peek()==':':
2259 self.advance()
2260 self.getWhiteSpace()
2261 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2262 srcBlock = self._eatToThisEndDirective(macroName)
2265 if hasattr(macro, 'convertArgStrToDict'):
2266 kwArgs = macro.convertArgStrToDict(args, parser=self, startPos=startPos)
2267 else:
2268 def getArgs(*pargs, **kws):
2269 return pargs, kws
2270 exec 'positionalArgs, kwArgs = getArgs(%(args)s)'%locals()
2272 assert not kwArgs.has_key('src')
2273 kwArgs['src'] = srcBlock
2275 if type(macro)==new.instancemethod:
2276 co = macro.im_func.func_code
2277 elif (hasattr(macro, '__call__')
2278 and hasattr(macro.__call__, 'im_func')):
2279 co = macro.__call__.im_func.func_code
2280 else:
2281 co = macro.func_code
2282 availableKwArgs = inspect.getargs(co)[0]
2284 if 'parser' in availableKwArgs:
2285 kwArgs['parser'] = self
2286 if 'macros' in availableKwArgs:
2287 kwArgs['macros'] = self._macros
2288 if 'compilerSettings' in availableKwArgs:
2289 kwArgs['compilerSettings'] = self.settings()
2290 if 'isShortForm' in availableKwArgs:
2291 kwArgs['isShortForm'] = isShortForm
2292 if isShortForm and 'EOLCharsInShortForm' in availableKwArgs:
2293 kwArgs['EOLCharsInShortForm'] = EOLCharsInShortForm
2295 if 'startPos' in availableKwArgs:
2296 kwArgs['startPos'] = startPos
2297 if 'endPos' in availableKwArgs:
2298 kwArgs['endPos'] = self.pos()
2300 srcFromMacroOutput = macro(**kwArgs)
2302 origParseSrc = self._src
2303 origBreakPoint = self.breakPoint()
2304 origPos = self.pos()
2305 # add a comment to the output about the macro src that is being parsed
2306 # or add a comment prefix to all the comments added by the compiler
2307 self._src = srcFromMacroOutput
2308 self.setPos(0)
2309 self.setBreakPoint(len(srcFromMacroOutput))
2311 self.parse(assertEmptyStack=False)
2313 self._src = origParseSrc
2314 self.setBreakPoint(origBreakPoint)
2315 self.setPos(origPos)
2318 #self._compiler.addRawText('end')
2320 def eatCache(self):
2321 isLineClearToStartToken = self.isLineClearToStartToken()
2322 endOfFirstLinePos = self.findEOL()
2323 lineCol = self.getRowCol()
2324 self.getDirectiveStartToken()
2325 self.advance(len('cache'))
2327 startPos = self.pos()
2328 argList = self.getDefArgList(useNameMapper=True)
2329 argList = self._applyExpressionFilters(argList, 'cache', startPos=startPos)
2331 def startCache():
2332 cacheInfo = self._compiler.genCacheInfoFromArgList(argList)
2333 self._compiler.startCacheRegion(cacheInfo, lineCol)
2335 if self.matchColonForSingleLineShortFormDirective():
2336 self.advance() # skip over :
2337 self.getWhiteSpace(max=1)
2338 startCache()
2339 self.parse(breakPoint=self.findEOL(gobble=True))
2340 self._compiler.endCacheRegion()
2341 else:
2342 if self.peek()==':':
2343 self.advance()
2344 self.getWhiteSpace()
2345 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2346 self.pushToOpenDirectivesStack('cache')
2347 startCache()
2349 def eatCall(self):
2350 # @@TR: need to enable single line version of this
2351 isLineClearToStartToken = self.isLineClearToStartToken()
2352 endOfFirstLinePos = self.findEOL()
2353 lineCol = self.getRowCol()
2354 self.getDirectiveStartToken()
2355 self.advance(len('call'))
2356 startPos = self.pos()
2358 useAutocallingOrig = self.setting('useAutocalling')
2359 self.setSetting('useAutocalling', False)
2360 self.getWhiteSpace()
2361 if self.matchCheetahVarStart():
2362 functionName = self.getCheetahVar()
2363 else:
2364 functionName = self.getCheetahVar(plain=True, skipStartToken=True)
2365 self.setSetting('useAutocalling', useAutocallingOrig)
2366 # @@TR: fix up filtering
2367 self._applyExpressionFilters(self[startPos:self.pos()], 'call', startPos=startPos)
2369 self.getWhiteSpace()
2370 args = self.getExpression(pyTokensToBreakAt=[':']).strip()
2371 if self.matchColonForSingleLineShortFormDirective():
2372 self.advance() # skip over :
2373 self._compiler.startCallRegion(functionName, args, lineCol)
2374 self.getWhiteSpace(max=1)
2375 self.parse(breakPoint=self.findEOL(gobble=False))
2376 self._compiler.endCallRegion()
2377 else:
2378 if self.peek()==':':
2379 self.advance()
2380 self.getWhiteSpace()
2381 self.pushToOpenDirectivesStack("call")
2382 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2383 self._compiler.startCallRegion(functionName, args, lineCol)
2385 def eatCallArg(self):
2386 isLineClearToStartToken = self.isLineClearToStartToken()
2387 endOfFirstLinePos = self.findEOL()
2388 lineCol = self.getRowCol()
2389 self.getDirectiveStartToken()
2391 self.advance(len('arg'))
2392 startPos = self.pos()
2393 self.getWhiteSpace()
2394 argName = self.getIdentifier()
2395 self.getWhiteSpace()
2396 argName = self._applyExpressionFilters(argName, 'arg', startPos=startPos)
2397 self._compiler.setCallArg(argName, lineCol)
2398 if self.peek() == ':':
2399 self.getc()
2400 else:
2401 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2403 def eatFilter(self):
2404 isLineClearToStartToken = self.isLineClearToStartToken()
2405 endOfFirstLinePos = self.findEOL()
2407 self.getDirectiveStartToken()
2408 self.advance(len('filter'))
2409 self.getWhiteSpace()
2410 startPos = self.pos()
2411 if self.matchCheetahVarStart():
2412 isKlass = True
2413 theFilter = self.getExpression(pyTokensToBreakAt=[':'])
2414 else:
2415 isKlass = False
2416 theFilter = self.getIdentifier()
2417 self.getWhiteSpace()
2418 theFilter = self._applyExpressionFilters(theFilter, 'filter', startPos=startPos)
2420 if self.matchColonForSingleLineShortFormDirective():
2421 self.advance() # skip over :
2422 self.getWhiteSpace(max=1)
2423 self._compiler.setFilter(theFilter, isKlass)
2424 self.parse(breakPoint=self.findEOL(gobble=False))
2425 self._compiler.closeFilterBlock()
2426 else:
2427 if self.peek()==':':
2428 self.advance()
2429 self.getWhiteSpace()
2430 self.pushToOpenDirectivesStack("filter")
2431 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2432 self._compiler.setFilter(theFilter, isKlass)
2434 def eatErrorCatcher(self):
2435 isLineClearToStartToken = self.isLineClearToStartToken()
2436 endOfFirstLinePos = self.findEOL()
2437 self.getDirectiveStartToken()
2438 self.advance(len('errorCatcher'))
2439 self.getWhiteSpace()
2440 startPos = self.pos()
2441 errorCatcherName = self.getIdentifier()
2442 errorCatcherName = self._applyExpressionFilters(
2443 errorCatcherName, 'errorcatcher', startPos=startPos)
2444 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2445 self._compiler.setErrorCatcher(errorCatcherName)
2447 def eatCapture(self):
2448 # @@TR: this could be refactored to use the code in eatSimpleIndentingDirective
2449 # filtered
2450 isLineClearToStartToken = self.isLineClearToStartToken()
2451 endOfFirstLinePos = self.findEOL()
2452 lineCol = self.getRowCol()
2454 self.getDirectiveStartToken()
2455 self.advance(len('capture'))
2456 startPos = self.pos()
2457 self.getWhiteSpace()
2459 expr = self.getExpression(pyTokensToBreakAt=[':'])
2460 expr = self._applyExpressionFilters(expr, 'capture', startPos=startPos)
2461 if self.matchColonForSingleLineShortFormDirective():
2462 self.advance() # skip over :
2463 self._compiler.startCaptureRegion(assignTo=expr, lineCol=lineCol)
2464 self.getWhiteSpace(max=1)
2465 self.parse(breakPoint=self.findEOL(gobble=False))
2466 self._compiler.endCaptureRegion()
2467 else:
2468 if self.peek()==':':
2469 self.advance()
2470 self.getWhiteSpace()
2471 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos)
2472 self.pushToOpenDirectivesStack("capture")
2473 self._compiler.startCaptureRegion(assignTo=expr, lineCol=lineCol)
2476 def eatIf(self):
2477 # filtered
2478 isLineClearToStartToken = self.isLineClearToStartToken()
2479 endOfFirstLine = self.findEOL()
2480 lineCol = self.getRowCol()
2481 self.getDirectiveStartToken()
2482 startPos = self.pos()
2484 expressionParts = self.getExpressionParts(pyTokensToBreakAt=[':'])
2485 expr = ''.join(expressionParts).strip()
2486 expr = self._applyExpressionFilters(expr, 'if', startPos=startPos)
2488 isTernaryExpr = ('then' in expressionParts and 'else' in expressionParts)
2489 if isTernaryExpr:
2490 conditionExpr = []
2491 trueExpr = []
2492 falseExpr = []
2493 currentExpr = conditionExpr
2494 for part in expressionParts:
2495 if part.strip()=='then':
2496 currentExpr = trueExpr
2497 elif part.strip()=='else':
2498 currentExpr = falseExpr
2499 else:
2500 currentExpr.append(part)
2502 conditionExpr = ''.join(conditionExpr)
2503 trueExpr = ''.join(trueExpr)
2504 falseExpr = ''.join(falseExpr)
2505 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
2506 self._compiler.addTernaryExpr(conditionExpr, trueExpr, falseExpr, lineCol=lineCol)
2507 elif self.matchColonForSingleLineShortFormDirective():
2508 self.advance() # skip over :
2509 self._compiler.addIf(expr, lineCol=lineCol)
2510 self.getWhiteSpace(max=1)
2511 self.parse(breakPoint=self.findEOL(gobble=True))
2512 self._compiler.commitStrConst()
2513 self._compiler.dedent()
2514 else:
2515 if self.peek()==':':
2516 self.advance()
2517 self.getWhiteSpace()
2518 self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLine)
2519 self.pushToOpenDirectivesStack('if')
2520 self._compiler.addIf(expr, lineCol=lineCol)
2522 ## end directive handlers
2523 def handleEndDef(self):
2524 isNestedDef = (self.setting('allowNestedDefScopes')
2525 and [name for name in self._openDirectivesStack if name=='def'])
2526 if not isNestedDef:
2527 self._compiler.closeDef()
2528 else:
2529 # @@TR: temporary hack of useSearchList
2530 self.setSetting('useSearchList', self._useSearchList_orig)
2531 self._compiler.commitStrConst()
2532 self._compiler.dedent()
2535 def pushToOpenDirectivesStack(self, directiveName):
2536 assert directiveName in self._closeableDirectives
2537 self._openDirectivesStack.append(directiveName)
2539 def popFromOpenDirectivesStack(self, directiveName):
2540 if not self._openDirectivesStack:
2541 raise ParseError(self, msg="#end found, but nothing to end")
2543 if self._openDirectivesStack[-1] == directiveName:
2544 del self._openDirectivesStack[-1]
2545 else:
2546 raise ParseError(self, msg="#end %s found, expected #end %s" %(
2547 directiveName, self._openDirectivesStack[-1]))
2549 def assertEmptyOpenDirectivesStack(self):
2550 if self._openDirectivesStack:
2551 errorMsg = (
2552 "Some #directives are missing their corresponding #end ___ tag: %s" %(
2553 ', '.join(self._openDirectivesStack)))
2554 raise ParseError(self, msg=errorMsg)
2556 ##################################################
2557 ## Make an alias to export
2558 Parser = _HighLevelParser