1 ;;; python.wy -- LALR grammar for Python
3 ;; Copyright (C) 2002-2012 Free Software Foundation, Inc.
4 ;; Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008,
5 ;; 2009, 2010 Python Software Foundation; All Rights Reserved
7 ;; Author: Richard Kim <ryk@dspwiz.com>
8 ;; Maintainer: Richard Kim <ryk@dspwiz.com>
12 ;; This file is part of GNU Emacs.
14 ;; GNU Emacs is free software: you can redistribute it and/or modify
15 ;; it under the terms of the GNU General Public License as published by
16 ;; the Free Software Foundation, either version 3 of the License, or
17 ;; (at your option) any later version.
19 ;; GNU Emacs is distributed in the hope that it will be useful,
20 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
21 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 ;; GNU General Public License for more details.
24 ;; You should have received a copy of the GNU General Public License
25 ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
29 ;; This is an LALR python parser that follows the official python
30 ;; grammar closely with very few exceptions. The Python grammar is
31 ;; used and reproduced under the following license:
33 ;; PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
34 ;; --------------------------------------------
35 ;; 1. This LICENSE AGREEMENT is between the Python Software Foundation
36 ;; ("PSF"), and the Individual or Organization ("Licensee") accessing
37 ;; and otherwise using this software ("Python") in source or binary
38 ;; form and its associated documentation.
40 ;; 2. Subject to the terms and conditions of this License Agreement,
41 ;; PSF hereby grants Licensee a nonexclusive, royalty-free, world-wide
42 ;; license to reproduce, analyze, test, perform and/or display
43 ;; publicly, prepare derivative works, distribute, and otherwise use
44 ;; Python alone or in any derivative version, provided, however, that
45 ;; PSF's License Agreement and PSF's notice of copyright, i.e.,
46 ;; "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008,
47 ;; 2009, 2010 Python Software Foundation; All Rights Reserved" are
48 ;; retained in Python alone or in any derivative version prepared by
51 ;; 3. In the event Licensee prepares a derivative work that is based
52 ;; on or incorporates Python or any part thereof, and wants to make
53 ;; the derivative work available to others as provided herein, then
54 ;; Licensee hereby agrees to include in any such work a brief summary
55 ;; of the changes made to Python.
57 ;; 4. PSF is making Python available to Licensee on an "AS IS"
58 ;; basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
59 ;; IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
60 ;; DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
61 ;; FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
62 ;; INFRINGE ANY THIRD PARTY RIGHTS.
64 ;; 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
65 ;; FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A
66 ;; RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR
67 ;; ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
69 ;; 6. This License Agreement will automatically terminate upon a
70 ;; material breach of its terms and conditions.
72 ;; 7. Nothing in this License Agreement shall be deemed to create any
73 ;; relationship of agency, partnership, or joint venture between PSF
74 ;; and Licensee. This License Agreement does not grant permission to
75 ;; use PSF trademarks or trade name in a trademark sense to endorse or
76 ;; promote products or services of Licensee, or any third party.
78 ;; 8. By copying, installing or otherwise using Python, Licensee
79 ;; agrees to be bound by the terms and conditions of this License
84 ;; * Verify that semantic-lex-python-number regexp is correct.
90 %package wisent-python-wy
91 %provide semantic/wisent/python-wy
94 (declare-function wisent-python-reconstitute-function-tag "semantic/wisent/python")
95 (declare-function wisent-python-reconstitute-class-tag "semantic/wisent/python")
98 %languagemode python-mode
100 ;; The default start symbol
102 ;; Alternate entry points
103 ;; - Needed by partial re-parse
104 %start function_parameter
106 %start indented_block
107 ;; - Needed by EXPANDFULL clauses
108 %start function_parameters
110 %start indented_block_body
112 ;; -------------------------------
113 ;; Misc. Python specific terminals
114 ;; -------------------------------
115 ;; The value of these tokens are for documentation only, they are not
116 ;; used by the lexer.
117 %token <charquote> BACKSLASH "\\"
118 %token <newline> NEWLINE "\n"
119 %token <indentation> INDENT "^\\s-+"
120 %token <indentation> DEDENT "[^:INDENT:]"
121 %token <indentation> INDENT_BLOCK "(INDENT DEDENT)"
123 ;; -----------------------------
124 ;; Block & Parenthesis terminals
125 ;; -----------------------------
126 %type <block> ;;syntax "\\s(\\|\\s)" matchdatatype block
128 %token <block> PAREN_BLOCK "(LPAREN RPAREN)"
129 %token <block> BRACE_BLOCK "(LBRACE RBRACE)"
130 %token <block> BRACK_BLOCK "(LBRACK RBRACK)"
132 %token <open-paren> LPAREN "("
133 %token <close-paren> RPAREN ")"
134 %token <open-paren> LBRACE "{"
135 %token <close-paren> RBRACE "}"
136 %token <open-paren> LBRACK "["
137 %token <close-paren> RBRACK "]"
139 ;; ------------------
140 ;; Operator terminals
141 ;; ------------------
142 %type <punctuation> ;;syntax "\\(\\s.\\|\\s$\\|\\s'\\)+" matchdatatype string
144 %token <punctuation> LTLTEQ "<<="
145 %token <punctuation> GTGTEQ ">>="
146 %token <punctuation> EXPEQ "**="
147 %token <punctuation> DIVDIVEQ "//="
148 %token <punctuation> DIVDIV "//"
149 %token <punctuation> LTLT "<<"
150 %token <punctuation> GTGT ">>"
151 %token <punctuation> EXPONENT "**"
152 %token <punctuation> EQ "=="
153 %token <punctuation> GE ">="
154 %token <punctuation> LE "<="
155 %token <punctuation> PLUSEQ "+="
156 %token <punctuation> MINUSEQ "-="
157 %token <punctuation> MULTEQ "*="
158 %token <punctuation> DIVEQ "/="
159 %token <punctuation> MODEQ "%="
160 %token <punctuation> AMPEQ "&="
161 %token <punctuation> OREQ "|="
162 %token <punctuation> HATEQ "^="
163 %token <punctuation> LTGT "<>"
164 %token <punctuation> NE "!="
165 %token <punctuation> HAT "^"
166 %token <punctuation> LT "<"
167 %token <punctuation> GT ">"
168 %token <punctuation> AMP "&"
169 %token <punctuation> MULT "*"
170 %token <punctuation> DIV "/"
171 %token <punctuation> MOD "%"
172 %token <punctuation> PLUS "+"
173 %token <punctuation> MINUS "-"
174 %token <punctuation> PERIOD "."
175 %token <punctuation> TILDE "~"
176 %token <punctuation> BAR "|"
177 %token <punctuation> COLON ":"
178 %token <punctuation> SEMICOLON ";"
179 %token <punctuation> COMMA ","
180 %token <punctuation> ASSIGN "="
181 %token <punctuation> BACKQUOTE "`"
182 %token <punctuation> AT "@"
188 %token <string> STRING_LITERAL
190 %type <number> ;;syntax semantic-lex-number-expression
191 %token <number> NUMBER_LITERAL
193 %type <symbol> ;;syntax "\\(\\sw\\|\\s_\\)+"
199 %type <keyword> ;;syntax "\\(\\sw\\|\\s_\\)+" matchdatatype keyword
203 "Logical AND binary operator ... "
207 "EXPR as NAME makes value of EXPR available as variable NAME"
209 %keyword ASSERT "assert"
211 "Raise AssertionError exception if <expr> is false"
213 %keyword BREAK "break"
215 "Terminate 'for' or 'while' loop"
217 %keyword CLASS "class"
221 %keyword CONTINUE "continue"
222 %put CONTINUE summary
223 "Skip to the next iteration of enclosing 'for' or 'while' loop"
227 "Define a new function"
231 "Delete specified objects, i.e., undo what assignment did"
235 "Shorthand for 'else if' following an 'if' statement"
239 "Start the 'else' clause following an 'if' statement"
241 %keyword EXCEPT "except"
243 "Specify exception handlers along with 'try' keyword"
247 "Dynamically execute Python code"
249 %keyword FINALLY "finally"
251 "Specify code to be executed after 'try' statements whether or not an exception occurred"
259 "Modify behavior of 'import' statement"
261 %keyword GLOBAL "global"
263 "Declare one or more symbols as global symbols"
267 "Start 'if' conditional statement"
269 %keyword IMPORT "import"
271 "Load specified modules"
275 "Part of 'for' statement "
279 "Binary operator that tests for object equality"
281 %keyword LAMBDA "lambda"
283 "Create anonymous function"
287 "Unary boolean negation operator"
291 "Binary logical 'or' operator"
295 "Statement that does nothing"
297 %keyword PRINT "print"
299 "Print each argument to standard output"
301 %keyword RAISE "raise"
305 %keyword RETURN "return"
307 "Return from a function"
311 "Start of statements protected by exception handlers"
313 %keyword WHILE "while"
315 "Start a 'while' loop"
319 "Start statement with an associated context object"
321 %keyword YIELD "yield"
323 "Create a generator function"
327 ;;;****************************************************************************
329 ;;;****************************************************************************
331 ;; simple_stmt are statements that do not involve INDENT tokens
332 ;; compound_stmt are statements that involve INDENT tokens
339 ;;;****************************************************************************
341 ;;;****************************************************************************
343 ;; simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
345 : small_stmt_list semicolon_opt NEWLINE
348 ;; small_stmt (';' small_stmt)*
351 | small_stmt_list SEMICOLON small_stmt
366 ;;;============================================================================
368 ;;;============================================================================
370 ;; print_stmt: 'print' [ test (',' test)* [','] ]
371 ;; | '>>' test [ (',' test)+ [','] ]
373 : PRINT print_stmt_trailer
377 ;; [ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ]
381 | GTGT test trailing_test_list_with_opt_comma_opt
385 ;; [ (',' test)+ [','] ]
386 trailing_test_list_with_opt_comma_opt
388 | trailing_test_list comma_opt
396 | trailing_test_list COMMA test
400 ;;;============================================================================
402 ;;;============================================================================
404 ;; expr_stmt: testlist (augassign testlist | ('=' testlist)*)
406 : testlist expr_stmt_trailer
407 (if (and $2 (stringp $1) (string-match "^\\(\\sw\\|\\s_\\)+$" $1))
408 ;; If this is an assignment statement and left side is a symbol,
409 ;; then generate a 'variable token, else return 'code token.
410 (VARIABLE-TAG $1 nil nil)
414 ;; Could be EMPTY because of eq_testlist_zom.
415 ;; (augassign testlist | ('=' testlist)*)
425 | eq_testlist_zom ASSIGN testlist
429 ;; augassign: '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^='
430 ;; | '<<=' | '>>=' | '**=' | '//='
432 : PLUSEQ | MINUSEQ | MULTEQ | DIVEQ | MODEQ
433 | AMPEQ | OREQ | HATEQ | LTLTEQ
434 | GTGTEQ | EXPEQ | DIVDIVEQ
437 ;;;============================================================================
439 ;;;============================================================================
441 ;; del_stmt: 'del' exprlist
447 ;; exprlist: expr (',' expr)* [',']
449 : expr_list comma_opt
457 | expr_list COMMA expr
461 ;;;============================================================================
463 ;;;============================================================================
471 ;;;============================================================================
473 ;;;============================================================================
483 ;; break_stmt: 'break'
489 ;; continue_stmt: 'continue'
495 ;; return_stmt: 'return' [testlist]
497 : RETURN testlist_opt
508 ;; yield_stmt: 'yield' testlist
516 ;; raise_stmt: 'raise' [test [',' test [',' test]]]
518 : RAISE zero_one_two_or_three_tests
522 ;; [test [',' test [',' test]]]
523 zero_one_two_or_three_tests
525 | test zero_one_or_two_tests
529 ;; [',' test [',' test]]
530 zero_one_or_two_tests
532 | COMMA test zero_or_one_comma_test
537 zero_or_one_comma_test
543 ;;;============================================================================
545 ;;;============================================================================
547 ;; import_stmt : 'import' dotted_as_name (',' dotted_as_name)*
548 ;; | 'from' dotted_name 'import'
549 ;; ('*' | import_as_name (',' import_as_name)*)
551 : IMPORT dotted_as_name_list
553 | FROM dotted_name IMPORT star_or_import_as_name_list
557 ;; dotted_as_name (',' dotted_as_name)*
559 : dotted_as_name_list COMMA dotted_as_name
565 ;; ('*' | import_as_name (',' import_as_name)*)
566 star_or_import_as_name_list
569 | import_as_name_list
573 ;; import_as_name (',' import_as_name)*
577 | import_as_name_list COMMA import_as_name
581 ;; import_as_name: NAME [NAME NAME]
587 ;; dotted_as_name: dotted_name [AS NAME]
589 : dotted_name as_name_opt
599 ;; dotted_name: NAME ('.' NAME)*
602 | dotted_name PERIOD NAME
603 (format "%s.%s" $1 $3)
606 ;;;============================================================================
608 ;;;============================================================================
610 ;; global_stmt: 'global' NAME (',' NAME)*
612 : GLOBAL comma_sep_name_list
619 | comma_sep_name_list COMMA NAME
622 ;;;============================================================================
624 ;;;============================================================================
626 ;; exec_stmt: 'exec' expr ['in' test [',' test]]
628 : EXEC expr exec_trailer
632 ;; ['in' test [',' test]]
635 | IN test comma_test_opt
646 ;;;============================================================================
648 ;;;============================================================================
650 ;; assert_stmt: 'assert' test [',' test]
652 : ASSERT test comma_test_opt
656 ;;;****************************************************************************
658 ;;;****************************************************************************
670 ;;;============================================================================
672 ;;;============================================================================
674 ;; if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
676 : IF test COLON suite elif_suite_pair_list else_suite_pair_opt
680 ;; ('elif' test ':' suite)*
683 | elif_suite_pair_list ELIF test COLON suite
687 ;; ['else' ':' suite]
694 ;; This NT follows the COLON token for most compound statements.
695 ;; suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
699 | NEWLINE indented_block
705 (EXPANDFULL $1 indented_block_body)
717 ;;;============================================================================
719 ;;;============================================================================
721 ;; while_stmt: 'while' test ':' suite ['else' ':' suite]
723 : WHILE test COLON suite else_suite_pair_opt
727 ;;;============================================================================
729 ;;;============================================================================
731 ;; for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
733 : FOR exprlist IN testlist COLON suite else_suite_pair_opt
737 ;;;============================================================================
739 ;;;============================================================================
741 ;; try_stmt: ('try' ':' suite (except_clause ':' suite)+ #diagram:break
742 ;; ['else' ':' suite] | 'try' ':' suite 'finally' ':' suite)
744 : TRY COLON suite except_clause_suite_pair_list else_suite_pair_opt
746 | TRY COLON suite FINALLY COLON suite
750 ;; (except_clause ':' suite)+
751 except_clause_suite_pair_list
752 : except_clause COLON suite
754 | except_clause_suite_pair_list except_clause COLON suite
758 ;; # NB compile.c makes sure that the default except clause is last
759 ;; except_clause: 'except' [test [',' test]]
761 : EXCEPT zero_one_or_two_test
768 | test zero_or_one_comma_test
772 ;;;============================================================================
774 ;;;============================================================================
776 ;; with_stmt: 'with' test [ with_var ] ':' suite
778 : WITH test COLON suite
780 | WITH test with_var COLON suite
781 (CODE-TAG $1 nil) ;; TODO capture variable
789 ;;;============================================================================
791 ;;;============================================================================
794 : AT dotted_name varargslist_opt NEWLINE
795 (FUNCTION-TAG $2 "decorator" $3)
801 | decorator decorators
805 ;; funcdef: [decorators] 'def' NAME parameters ':' suite
807 : DEF NAME function_parameter_list COLON suite
808 (wisent-python-reconstitute-function-tag
809 (FUNCTION-TAG $2 nil $3) $5)
810 | decorators DEF NAME function_parameter_list COLON suite
811 (wisent-python-reconstitute-function-tag
812 (FUNCTION-TAG $3 nil $4 :decorators $1) $6)
815 function_parameter_list
817 (let ((wisent-python-EXPANDING-block t))
818 (EXPANDFULL $1 function_parameters))
821 ;; parameters: '(' [varargslist] ')'
827 | function_parameter COMMA
828 | function_parameter RPAREN
834 ;; (VARIABLE-TAG $1 nil nil)
836 (VARIABLE-TAG $2 nil nil)
838 (VARIABLE-TAG $2 nil nil)
841 ;;;============================================================================
842 ;;;@@ class_declaration
843 ;;;============================================================================
845 ;; classdef: 'class' NAME ['(' testlist ')'] ':' suite
847 : CLASS NAME paren_class_list_opt COLON suite
848 (wisent-python-reconstitute-class-tag
849 (TYPE-TAG $2 $1 ;; Name "class"
851 (cons $3 nil) ;; (SUPERCLASSES . INTERFACES)
855 ;; ['(' testlist ')']
863 (let ((wisent-python-EXPANDING-block t))
864 (mapcar 'semantic-tag-name (EXPANDFULL $1 paren_classes)))
867 ;; parameters: '(' [varargslist] ')'
874 (VARIABLE-TAG $1 nil nil)
876 (VARIABLE-TAG $1 nil nil)
879 ;; In general, the base class can be specified by a general expression
880 ;; which evaluates to a class object, i.e., base classes are not just names!
881 ;; However base classes are names in most cases. Thus the
882 ;; non-terminals below work only with simple names. Even if the
883 ;; parser can parse general expressions, I don't see much benefit in
884 ;; generating a string of expression as base class "name".
889 ;;;****************************************************************************
891 ;;;****************************************************************************
893 ;; test: and_test ('or' and_test)* | lambdef
899 ;; and_test ('or' and_test)*
902 | test_test OR and_test
906 ;; and_test: not_test ('and' not_test)*
909 | and_test AND not_test
913 ;; not_test: 'not' not_test | comparison
920 ;; comparison: expr (comp_op expr)*
923 | comparison comp_op expr
927 ;; comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
929 : LT | GT | EQ | GE | LE | LTGT | NE | IN | NOT IN | IS | IS NOT
932 ;; expr: xor_expr ('|' xor_expr)*
939 ;; xor_expr: and_expr ('^' and_expr)*
942 | xor_expr HAT and_expr
946 ;; and_expr: shift_expr ('&' shift_expr)*
949 | and_expr AMP shift_expr
953 ;; shift_expr: arith_expr (('<<'|'>>') arith_expr)*
956 | shift_expr shift_expr_operators arith_expr
966 ;; arith_expr: term (('+'|'-') term)*
969 | arith_expr plus_or_minus term
979 ;; term: factor (('*'|'/'|'%'|'//') factor)*
982 | term term_operator factor
993 ;; factor: ('+'|'-'|'~') factor | power
995 : prefix_operators factor
1007 ;; power: atom trailer* ('**' factor)*
1009 : atom trailer_zom exponent_zom
1011 (if $2 (concat " " $2 " ") "")
1012 (if $3 (concat " " $3) "")
1018 | trailer_zom trailer
1024 | exponent_zom EXPONENT factor
1028 ;; trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
1038 ;; atom: '(' [testlist] ')' | '[' [listmaker] ']' | '{' [dictmaker] '}'
1039 ;; | '`' testlist '`' | NAME | NUMBER | STRING+
1047 | BACKQUOTE testlist BACKQUOTE
1051 | one_or_more_string
1060 ;; testlist: test (',' test)* [',']
1062 : comma_sep_test_list comma_opt
1068 | comma_sep_test_list COMMA test
1069 (format "%s, %s" $1 $3)
1072 ;; (read $1) and (read $2) were done before to peel away the double quotes.
1073 ;; However that does not work for single quotes, so it was taken out.
1076 | one_or_more_string STRING_LITERAL
1080 ;;;****************************************************************************
1082 ;;;****************************************************************************
1084 ;; lambdef: 'lambda' [varargslist] ':' test
1086 : LAMBDA varargslist_opt COLON test
1087 (format "%s %s" $1 (or $2 ""))
1096 ;; varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] | '**' NAME)
1097 ;; | fpdef ['=' test] (',' fpdef ['=' test])* [',']
1099 : fpdef_opt_test_list_comma_zom rest_args
1101 | fpdef_opt_test_list comma_opt
1104 ;; ('*' NAME [',' '**' NAME] | '**' NAME)
1106 : MULT NAME multmult_name_opt
1107 () ;;(VARIABLE-TAG $2 nil nil)
1109 () ;;(VARIABLE-TAG $2 nil nil)
1115 | COMMA EXPONENT NAME
1116 (VARIABLE-TAG $3 nil nil)
1119 fpdef_opt_test_list_comma_zom
1121 | fpdef_opt_test_list_comma_zom fpdef_opt_test COMMA
1125 ;; fpdef ['=' test] (',' fpdef ['=' test])*
1128 | fpdef_opt_test_list COMMA fpdef_opt_test
1137 ;; fpdef: NAME | '(' fplist ')'
1140 (VARIABLE-TAG $1 nil nil)
1141 ;; Below breaks the parser. Don't know why, but my guess is that
1142 ;; LPAREN/RPAREN clashes with the ones in function_parameters.
1143 ;; | LPAREN fplist RPAREN
1147 ;; fplist: fpdef (',' fpdef)* [',']
1149 : fpdef_list comma_opt
1152 ;; fpdef (',' fpdef)*
1155 | fpdef_list COMMA fpdef
1165 ;;;****************************************************************************
1167 ;;;****************************************************************************
1181 ;;; python.wy ends here